diff --git a/.gitignore b/.gitignore index 215d7c75ef..7055505fec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,37 +1,40 @@ -# Ignore Cuckoo Database -db/cuckoo.db - -# Ignore Cuckoo logs -log/*.log - -# Ignore Cuckoo analyses -storage/* - -# Ignore Python byte code +# Python byte code *.pyc -# Ignore certificates +# Certificates *.pem *.cert -# Ignore OS generated files +# OS generated files .DS_Store* .AppleDouble ehthumbs.db Icon? Thumbs.db -# Ignore development files -docs/book/src/_build/ +# Development files +docs/book/_build/ .idea/ .project .pydevproject -# Ignore Django secret_key -web/web/secret_key.py - -# Ignore yara rules -data/yara/index_*.yar - -# Ignore venv +# Python package stuff +.eggs/ +.cache +Cuckoo.egg-info/ venv +lib/python2.7/ +build/ +dist/ +local/ +.pytest_cache/ + +# Cuckoo package stuff +cuckoo/private/.cwd + +# Frontend compilation services +node_modules/ +bower_components/ +.sass-cache/ +cuckoo/web/src/package-lock.json +*.map diff --git a/.travis.yml b/.travis.yml index e479c99a58..41cea05ee6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,20 +1,107 @@ -branches: - only: - - master language: python -python: - - 2.7 + +matrix: + fast_finish: true + include: + - python: 2.7 + - os: osx + osx_image: xcode9.3beta + language: generic + +services: + - mongodb + - mysql + - postgresql + before_install: - - sudo apt-get update -qq - - sudo apt-get install python-libvirt python-yaml - - wget http://downloads.sourceforge.net/project/ssdeep/ssdeep-2.12/ssdeep-2.12.tar.gz - - tar -zxvf ssdeep-2.12.tar.gz - - cd ssdeep-2.12 - - ./configure && make - - sudo make install - - cd .. + - | + if [[ $TRAVIS_OS_NAME == "linux" ]]; then + sudo apt-get update -qq + sudo apt-get install python-dev python-libvirt libffi-dev libssl-dev + sudo apt-get install swig libpq-dev libmysqlclient-dev rar + sudo apt-get install libcairo2-dev libpng-dev libossp-uuid-dev + sudo apt-get install libfreerdp-dev libjpeg-turbo8-dev + + # Install guacd. + wget http://archive.apache.org/dist/guacamole/0.9.14/source/guacamole-server-0.9.14.tar.gz + tar xvf guacamole-server-0.9.14.tar.gz && cd guacamole-server-0.9.14 + ./configure --with-init-dir=/etc/init.d && make && sudo make install && sudo ldconfig && cd ../ + + sudo mysql -e "CREATE USER cuckoo@localhost IDENTIFIED BY 'cuckoo'" + sudo mysql -e "GRANT ALL PRIVILEGES ON * . * TO cuckoo@localhost" + sudo mysql -e "FLUSH PRIVILEGES" + sudo mysql -e "CREATE DATABASE cuckootest" + sudo mysql -e "CREATE DATABASE cuckootest060" + sudo mysql -e "CREATE DATABASE cuckootest11" + sudo mysql -e "CREATE DATABASE distcuckootest" + psql -U postgres -c "CREATE USER cuckoo WITH PASSWORD 'cuckoo'" + psql -U postgres -c "CREATE DATABASE cuckootest" + psql -U postgres -c "CREATE DATABASE cuckootest060" + psql -U postgres -c "CREATE DATABASE cuckootest11" + psql -U postgres -c "CREATE DATABASE distcuckootest" + + sudo mysql -e "CREATE DATABASE cuckootestimport" + sudo mysql cuckootestimport < tests/files/sql/11my.sql + psql -U postgres -c "CREATE DATABASE cuckootestimport" + psql -U postgres cuckootestimport /dev/null + pip install psycopg2 mysql-python m2crypto==0.24.0 weasyprint==0.36 + else + brew update || brew update + brew install libmagic cairo pango mongodb + brew services start mongodb + # The following wasn't required in the past and therefore may become + # obsolete once again in the future. Let's wait and see. + wget https://bootstrap.pypa.io/get-pip.py + sudo python get-pip.py + sudo pip install virtualenv + virtualenv $HOME + source $HOME/bin/activate + fi + install: - - pip install -r requirements.txt + # Fetch monitor & stuff which is required for CWD migration tests. + - python stuff/monitor.py + + # Bump the version of Cuckoo during tests (e.g., for new Signatures). + - python stuff/patch_version.py setup.py 3.0 + + - pip install -U pip setuptools + - python setup.py sdist + - pip install -e . + - pip install pytest==4.1.1 pytest-cov codecov coveralls distorm3 pycrypto + - pip install flask-testing mock pytest-django pytest-pythonpath responses + + # Install Volatility. + - git clone https://github.com/volatilityfoundation/volatility vol-setup + - cd vol-setup && python setup.py install && cd .. + script: - - python cuckoo.py --debug --test - #- nosetests + # Start guacd. + - | + if [[ $TRAVIS_OS_NAME == "linux" ]]; then + sudo /etc/init.d/guacd start + fi + + # Create the Cuckoo Working Directory. + - cuckoo -d + + # Download Cuckoo Signatures. + - cuckoo community + + # Check the code integrity of the Signatures by running Cuckoo. + - cuckoo -d --ignore-vuln 2>&1|grep "Unable to bind ResultServer" + + # Run tests from the "tests" directory. + - py.test --cov=cuckoo + +after_success: + - coveralls + - codecov + +dist: xenial +addons: + apt: + packages: + - sqlite3 + sources: + - travis-ci/sqlite3 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..142a7040d4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,660 @@ +Cuckoo Sandbox is copyrighted by the Cuckoo Foundation and is licensed under +the following GNU General Public License version 3. + + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + +The file UserDB.txt is copyrighted by BoB / Team PEiD distributed under the following +MIT license. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, +merge, publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be included in all copies +or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +The file jquery.js is copyrighted by John Resig and dual licensed under the MIT or GPL +Version 2 licenses (see: http://jquery.org/license). + +The files lightbox.js and lightbox.css are copyrighted by Lokesh Dhakar and licensed under +the Creative Commons Attribution 2.5 License +(see: http://creativecommons.org/licenses/by/2.5/). + +The files bootstrap-fileupload.js, jasny-bootstrap.js, jasny-bootstrap.min.js, +jasny-bootstrap.css, jasny-bootstrap.min.css, jasny-bootstrap-responsive.css, +jasny-bootstrap-responsive.min.css are copyrighted by Jasny BV and licensed under the Apache +License, Version 2.0. + +The files bootstrap.min.js, bootstrap.min.css, bootstrap-responsive.min.css, +glyphicons-halflings.png, glyphicons-halflings-white.png are copyrighted by Twitter, Inc. +and licensed under the Apache License, Version 2.0. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000..83cafcb624 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +graft cuckoo +recursive-exclude * *.pyc *.pyo *.map +recursive-exclude cuckoo/web/src * diff --git a/README.md b/README.md deleted file mode 100644 index 58fed858d2..0000000000 --- a/README.md +++ /dev/null @@ -1,17 +0,0 @@ -![Cuckoo](http://cuckoosandbox.org/graphic/cuckoo.png) - -In three words, [Cuckoo Sandbox](http://www.cuckoosandbox.org) is a malware analysis system. - -What does that mean? It simply means that you can throw any suspicious file at it and in a matter of seconds Cuckoo will provide you back some detailed results outlining what such file did when executed inside an isolated environment. - -If you want to contribute to development, please read [this](http://www.cuckoosandbox.org/development.html) and [this](http://www.cuckoofoundation.org/contribute) first. Make sure you check our Issues and Pull Requests and that you join our IRC channel. - -
- -This is a development version, we do not recommend its use in production. - -You can find a full documentation of the latest stable release [here](http://docs.cuckoosandbox.org). - -
- -[![Build Status](https://travis-ci.org/cuckoobox/cuckoo.png?branch=master)](https://travis-ci.org/cuckoobox/cuckoo) diff --git a/README.rst b/README.rst new file mode 100644 index 0000000000..35fb0dcfcd --- /dev/null +++ b/README.rst @@ -0,0 +1,50 @@ +.. image:: https://cuckoosandbox.org/assets/images/cuckoo-black.png + :alt: Cuckoo Sandbox + :target: https://cuckoosandbox.org/ + +**PLEASE NOTE: Cuckoo Sandbox 2.x is currently unmaintained. Any open issues +or pull requests will most likely not be processed, as a current full rewrite +of Cuckoo is undergoing and will be announced soon.** + +`Cuckoo Sandbox `_ is the leading open source +automated malware analysis system. + +What does that mean? It simply means that you can throw any suspicious file at +it and in a matter of seconds Cuckoo will provide you back some detailed +results outlining what such file did when executed inside an isolated +environment. + +If you want to contribute to development, report a bug, make a feature request +or ask a question, please first take a look at our `community guidelines`_. +Make sure you check our existing Issues and Pull Requests and that you join +our `IRC or Slack channel `_. + +For setup instructions, please refer +`to `_ +`our `_ +`documentation `_. + +This is a development version, we do not recommend its use in production; the +latest stable version may be installed through :code:`pip install -U cuckoo`. + +You can find the full documentation of the latest stable release +`here `_. + +.. image:: https://travis-ci.org/cuckoosandbox/cuckoo.png?branch=master + :alt: Linux Build Status + :target: https://travis-ci.org/cuckoosandbox/cuckoo + +.. image:: https://ci.appveyor.com/api/projects/status/p892esebjdbhq653/branch/master?svg=true + :alt: Windows Build Status + :target: https://ci.appveyor.com/project/jbremer/cuckoo/branch/master + +.. image:: https://coveralls.io/repos/github/cuckoosandbox/cuckoo/badge.svg?branch=master + :alt: Coverage Coverage Status + :target: https://coveralls.io/github/cuckoosandbox/cuckoo?branch=master + +.. image:: https://codecov.io/gh/cuckoosandbox/cuckoo/branch/master/graph/badge.svg + :alt: Codecov Coverage Status + :target: https://codecov.io/gh/cuckoosandbox/cuckoo + +.. _`community guidelines`: https://docs.cuckoosandbox.org/en/latest/introduction/community.html +.. _`contribution requirements`: http://www.cuckoofoundation.org/contribute.html diff --git a/agent/agent.py b/agent/agent.py deleted file mode 100644 index 038270cf10..0000000000 --- a/agent/agent.py +++ /dev/null @@ -1,209 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import os -import sys -import time -import socket -import string -import random -import platform -import subprocess -import ConfigParser -from StringIO import StringIO -from zipfile import ZipFile -from SimpleXMLRPCServer import SimpleXMLRPCServer - -BIND_IP = "0.0.0.0" -BIND_PORT = 8000 - -STATUS_INIT = 0x0001 -STATUS_RUNNING = 0x0002 -STATUS_COMPLETED = 0x0003 -STATUS_FAILED = 0x0004 - -class Agent(object): - """Cuckoo agent, it runs inside guest.""" - - def __init__(self): - self.system = platform.system().lower() - self.analyzer_path = "" - self.analyzer_pid = 0 - - self.error_message = None - self.current_status = STATUS_INIT - self.analyzer_folder = "" - self.results_folder = "" - - def _initialize(self): - if not self.analyzer_folder: - random.seed(time.time()) - container = "".join(random.choice(string.ascii_lowercase) for x in range(random.randint(5, 10))) - - if self.system == "windows": - system_drive = os.environ["SYSTEMDRIVE"] + os.sep - self.analyzer_folder = os.path.join(system_drive, container) - elif self.system == "linux" or self.system == "darwin": - self.analyzer_folder = \ - os.path.join(os.environ.get("HOME", os.environ.get("PWD", "/tmp")), container) - else: - self.error_message = "Unable to identify operating system" - return False - - try: - os.makedirs(self.analyzer_folder) - except OSError as e: - self.error_message = e - return False - - return True - - def get_status(self): - """Get current status. - @return: status. - """ - return self.current_status - - def get_error(self): - """Get error message. - @return: error message. - """ - return str(self.error_message) - - def add_malware(self, data, name): - """Get analysis data. - @param data: analysis data. - @param name: file name. - @return: operation status. - """ - data = data.data - - if self.system == "windows": - root = os.environ["TEMP"] - elif self.system == "linux" or self.system == "darwin": - root = "/tmp" - else: - self.error_message = \ - "Unable to write malware to disk because the operating " \ - "system could not be identified." - return False - - file_path = os.path.join(root, name) - - try: - with open(file_path, "wb") as sample: - sample.write(data) - except IOError as e: - self.error_message = \ - "Unable to write sample to disk: {0}".format(e) - return False - - return True - - def add_config(self, options): - """Creates analysis.conf file from current analysis options. - @param options: current configuration options, dict format. - @return: operation status. - """ - if not isinstance(options, dict): - return False - - config = ConfigParser.RawConfigParser() - config.add_section("analysis") - - try: - for key, value in options.items(): - # Options can be UTF encoded. - if isinstance(value, basestring): - try: - value = value.encode("utf-8") - except UnicodeEncodeError: - pass - - config.set("analysis", key, value) - - config_path = os.path.join(self.analyzer_folder, "analysis.conf") - - with open(config_path, "wb") as config_file: - config.write(config_file) - except Exception as e: - self.error_message = e - return False - - return True - - def add_analyzer(self, data): - """Add analyzer. - @param data: analyzer data. - @return: operation status. - """ - data = data.data - - if not self._initialize(): - return False - - try: - zip_data = StringIO() - zip_data.write(data) - - with ZipFile(zip_data, "r") as archive: - archive.extractall(self.analyzer_folder) - finally: - zip_data.close() - - self.analyzer_path = os.path.join(self.analyzer_folder, "analyzer.py") - return True - - def execute(self): - """Execute analysis. - @return: analyzer PID. - """ - if not self.analyzer_path or not os.path.exists(self.analyzer_path): - return False - - try: - proc = subprocess.Popen([sys.executable, self.analyzer_path], - cwd=os.path.dirname(self.analyzer_path)) - self.analyzer_pid = proc.pid - except OSError as e: - self.error_message = e - return False - - self.current_status = STATUS_RUNNING - return self.analyzer_pid - - def complete(self, success=True, error="", results=""): - """Complete analysis. - @param success: success status. - @param error: error status. - """ - if success: - self.current_status = STATUS_COMPLETED - else: - self.current_status = STATUS_FAILED - - if error: - self.error_message = error - - self.results_folder = results - return True - -if __name__ == "__main__": - try: - if not BIND_IP: - BIND_IP = socket.gethostbyname(socket.gethostname()) - - print("[+] Starting agent on %s:%s ..." % (BIND_IP, BIND_PORT)) - - # Disable DNS lookup, by Scott D. - def FakeGetFQDN(name=""): - return name - - socket.getfqdn = FakeGetFQDN - - server = SimpleXMLRPCServer((BIND_IP, BIND_PORT), allow_none=True) - server.register_instance(Agent()) - server.serve_forever() - except KeyboardInterrupt: - server.shutdown() diff --git a/analyzer b/analyzer new file mode 120000 index 0000000000..fe414c99a6 --- /dev/null +++ b/analyzer @@ -0,0 +1 @@ +cuckoo/data/analyzer \ No newline at end of file diff --git a/analyzer/android/__init__.py b/analyzer/android/__init__.py deleted file mode 100644 index bf0bac1f5f..0000000000 --- a/analyzer/android/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. -# Originally contributed by Check Point Software Technologies, Ltd. diff --git a/analyzer/android/analyzer.py b/analyzer/android/analyzer.py deleted file mode 100644 index b61899f587..0000000000 --- a/analyzer/android/analyzer.py +++ /dev/null @@ -1,261 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. -# Originally contributed by Check Point Software Technologies, Ltd. - -import os -import logging -import pkgutil -import shutil -import sys -import xmlrpclib -import time - -from lib.core.packages import choose_package -from lib.common.exceptions import CuckooError, CuckooPackageError -from lib.common.abstracts import Package, Auxiliary -from lib.common.constants import PATHS -from lib.core.config import Config -from lib.core.startup import init_logging -from modules import auxiliary - -logging.disable(level=logging.DEBUG) -log = logging.getLogger() - -class Analyzer(object): - def __init__(self): - self.config = None - self.target = None - - def complete(self): - """End analysis.""" - log.info("Analysis completed") - - def get_options(self): - """Get analysis options. - @return: options dict. - """ - # The analysis package can be provided with some options in the - # following format: - # option1=value1,option2=value2,option3=value3 - # - # Here we parse such options and provide a dictionary that will be made - # accessible to the analysis package. - options = {} - if self.config.options: - try: - # Split the options by comma. - fields = self.config.options.strip().split(",") - except ValueError as e: - log.warning("Failed parsing the options: %s", e) - else: - for field in fields: - # Split the name and the value of the option. - try: - key, value = field.strip().split("=") - except ValueError as e: - log.warning("Failed parsing option (%s): %s", field, e) - else: - # If the parsing went good, we add the option to the - # dictionary. - options[key.strip()] = value.strip() - - return options - - def prepare(self): - # Initialize logging. - init_logging() - - # Parse the analysis configuration file generated by the agent. - self.config = Config(cfg="analysis.conf") - - # We update the target according to its category. If it's a file, then - # we store the path. - if self.config.category == "file": - self.target = os.path.join("/data/local/tmp", str(self.config.file_name)) - shutil.copyfile("config/hooks.json", "/data/local/tmp/hooks.json") - # If it's a URL, well.. we store the URL. - else: - self.target = self.config.target - - def run(self): - self.prepare() - - log.info("Starting analyzer from: {0}".format(os.getcwd())) - log.info("Storing results at: {0}".format(PATHS["root"])) - log.info("Target is: {0}".format(self.target)) - - # If no analysis package was specified at submission, we try to select - # one automatically. - if not self.config.package: - log.info("No analysis package specified, trying to detect it automagically") - # If the analysis target is a file, we choose the package according - # to the file format. - if self.config.category == "file": - package = choose_package(self.config.file_type, self.config.file_name) - # If it's an URL, we'll just use the default Internet Explorer - # package. - else: - package = "default_browser" - - # If we weren't able to automatically determine the proper package, - # we need to abort the analysis. - if not package: - raise CuckooError("No valid package available for file type: {0}".format(self.config.file_type)) - - log.info("Automatically selected analysis package \"%s\"", package) - # Otherwise just select the specified package. - else: - package = self.config.package - - # Generate the package path. - package_name = "modules.packages.%s" % package - - # Try to import the analysis package. - try: - __import__(package_name, globals(), locals(), ["dummy"], -1) - # If it fails, we need to abort the analysis. - except ImportError: - raise CuckooError("Unable to import package \"{0}\", does not exist.".format(package_name)) - - # Initialize the package parent abstract. - Package() - - # Enumerate the abstract's subclasses. - try: - package_class = Package.__subclasses__()[0] - except IndexError as e: - raise CuckooError("Unable to select package class (package={0}): {1}".format(package_name, e)) - - # Initialize the analysis package. - pack = package_class(self.get_options()) - - # Initialize Auxiliary modules - Auxiliary() - prefix = auxiliary.__name__ + "." - for loader, name, ispkg in pkgutil.iter_modules(auxiliary.__path__, prefix): - if ispkg: - continue - - # Import the auxiliary module. - try: - __import__(name, globals(), locals(), ["dummy"], -1) - except ImportError as e: - log.warning("Unable to import the auxiliary module " - "\"%s\": %s", name, e) - - # Walk through the available auxiliary modules. - aux_enabled = [] - for module in Auxiliary.__subclasses__(): - # Try to start the auxiliary module. - try: - aux = module() - aux.start() - except (NotImplementedError, AttributeError): - log.warning("Auxiliary module %s was not implemented", - aux.__class__.__name__) - continue - except Exception as e: - log.warning("Cannot execute auxiliary module %s: %s", - aux.__class__.__name__, e) - continue - finally: - log.info("Started auxiliary module %s", - aux.__class__.__name__) - aux_enabled.append(aux) - - # Start analysis package. If for any reason, the execution of the - # analysis package fails, we have to abort the analysis. - try: - pack.start(self.target) - except NotImplementedError: - raise CuckooError("The package \"{0}\" doesn't contain a run " - "function.".format(package_name)) - except CuckooPackageError as e: - raise CuckooError("The package \"{0}\" start function raised an " - "error: {1}".format(package_name, e)) - except Exception as e: - raise CuckooError("The package \"{0}\" start function encountered " - "an unhandled exception: " - "{1}".format(package_name, e)) - - time_counter = 0 - while True: - time_counter += 1 - if time_counter == int(self.config.timeout): - log.info("Analysis timeout hit, terminating analysis") - break - - try: - # The analysis packages are provided with a function that - # is executed at every loop's iteration. If such function - # returns False, it means that it requested the analysis - # to be terminate. - if not pack.check(): - log.info("The analysis package requested the " - "termination of the analysis...") - break - - # If the check() function of the package raised some exception - # we don't care, we can still proceed with the analysis but we - # throw a warning. - except Exception as e: - log.warning("The package \"%s\" check function raised " - "an exception: %s", package_name, e) - finally: - # Zzz. - time.sleep(1) - - try: - # Before shutting down the analysis, the package can perform some - # final operations through the finish() function. - pack.finish() - except Exception as e: - log.warning("The package \"%s\" finish function raised an " - "exception: %s", package_name, e) - - # Terminate the Auxiliary modules. - for aux in aux_enabled: - try: - aux.stop() - except (NotImplementedError, AttributeError): - continue - except Exception as e: - log.warning("Cannot terminate auxiliary module %s: %s", - aux.__class__.__name__, e) - - # Let's invoke the completion procedure. - self.complete() - return True - -if __name__ == "__main__": - success = False - error = "" - - try: - # Initialize the main analyzer class. - analyzer = Analyzer() - # Run it and wait for the response. - success = analyzer.run() - # This is not likely to happen. - except KeyboardInterrupt: - error = "Keyboard Interrupt" - # If the analysis process encountered a critical error, it will raise a - # CuckooError exception, which will force the termination of the analysis - # weill notify the agent of the failure. Also catched unexpected - # exceptions. - except Exception as e: - # Store the error. - error = str(e) - - # Just to be paranoid. - if len(log.handlers) > 0: - log.critical(error) - else: - sys.stderr.write("{0}\n".format(e)) - # Once the analysis is completed or terminated for any reason, we report - # back to the agent, notifying that it can report back to the host. - finally: - # Establish connection with the agent XMLRPC server. - server = xmlrpclib.Server("http://127.0.0.1:8000") - server.complete(success, error, PATHS["root"]) diff --git a/analyzer/android/lib/__init__.py b/analyzer/android/lib/__init__.py deleted file mode 100644 index bf0bac1f5f..0000000000 --- a/analyzer/android/lib/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. -# Originally contributed by Check Point Software Technologies, Ltd. diff --git a/analyzer/android/lib/api/__init__.py b/analyzer/android/lib/api/__init__.py deleted file mode 100644 index bf0bac1f5f..0000000000 --- a/analyzer/android/lib/api/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. -# Originally contributed by Check Point Software Technologies, Ltd. diff --git a/analyzer/android/lib/api/screenshot.py b/analyzer/android/lib/api/screenshot.py deleted file mode 100644 index 2833e29525..0000000000 --- a/analyzer/android/lib/api/screenshot.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. -# Originally contributed by Check Point Software Technologies, Ltd. - -import math -import filecmp - -try: - import ImageChops - from PIL import Image - HAVE_PIL = True -except: - try: - from PIL import ImageChops - from PIL import Image - HAVE_PIL = True - except: - HAVE_PIL = False - -class Screenshot: - """Get screenshots.""" - - def have_pil(self): - """Is Python Image Library installed? - @return: installed status. - """ - return HAVE_PIL - - def equal_old(self, img1, img2): - """Compares two screenshots using Root-Mean-Square Difference (RMS). - @param img1: screenshot to compare. - @param img2: screenshot to compare. - @return: equal status. - """ - if not HAVE_PIL: - return None - image1 = Image.open(img1) - image2 = Image.open(img2) - # To get a measure of how similar two images are, we use - # root-mean-square (RMS). If the images are exactly identical, - # this value is zero. - diff = ImageChops.difference(image1, image2) - h = diff.histogram() - sq = (value*((idx % 256)**2) for idx, value in enumerate(h)) - sum_of_squares = sum(sq) - rms = math.sqrt(sum_of_squares/float(image1.size[0] * image2.size[1])) - - # Might need to tweak the threshold. - return rms < 8 - - def equal(self, img1, img2): - return filecmp.cmp(img1, img2) diff --git a/analyzer/android/lib/common/__init__.py b/analyzer/android/lib/common/__init__.py deleted file mode 100644 index bf0bac1f5f..0000000000 --- a/analyzer/android/lib/common/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. -# Originally contributed by Check Point Software Technologies, Ltd. diff --git a/analyzer/android/lib/common/abstracts.py b/analyzer/android/lib/common/abstracts.py deleted file mode 100644 index de9800fab0..0000000000 --- a/analyzer/android/lib/common/abstracts.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. -# Originally contributed by Check Point Software Technologies, Ltd. - -class Package(object): - """Base abstact analysis package.""" - - def __init__(self, options={}): - """@param options: options dict.""" - self.options = options - self.pids = [] - - def set_pids(self, pids): - """Update list of monitored PIDs in the package context. - @param pids: list of pids. - """ - self.pids = pids - - def start(self): - """Run analysis packege. - @param path: sample path. - @raise NotImplementedError: this method is abstract. - """ - raise NotImplementedError - - def check(self): - """Check. - @raise NotImplementedError: this method is abstract. - """ - raise NotImplementedError - - def finish(self): - """Finish run. - @raise NotImplementedError: this method is abstract. - """ - raise NotImplementedError - -class Auxiliary(object): - pass diff --git a/analyzer/android/lib/common/constants.py b/analyzer/android/lib/common/constants.py deleted file mode 100644 index 6a7b8da7dd..0000000000 --- a/analyzer/android/lib/common/constants.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. -# Originally contributed by Check Point Software Technologies, Ltd. - -import os -import string -import random - -def _rand_string(a, b): - return "".join(random.choice(string.ascii_lowercase) for x in xrange(random.randint(a, b))) - -ROOT = os.path.join("/data/local/tmp", _rand_string(6, 10)) - -PATHS = { - "root" : ROOT, - "logs" : os.path.join(ROOT, "logs"), - "files" : os.path.join(ROOT, "files"), - "shots" : os.path.join(ROOT, "shots"), - "memory" : os.path.join(ROOT, "memory"), - "drop" : os.path.join(ROOT, "drop") -} diff --git a/analyzer/android/lib/common/exceptions.py b/analyzer/android/lib/common/exceptions.py deleted file mode 100644 index a41ef35da2..0000000000 --- a/analyzer/android/lib/common/exceptions.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. -# Originally contributed by Check Point Software Technologies, Ltd. - -class CuckooError(Exception): - pass - -class CuckooPackageError(Exception): - pass diff --git a/analyzer/android/lib/common/results.py b/analyzer/android/lib/common/results.py deleted file mode 100644 index a9ba198a82..0000000000 --- a/analyzer/android/lib/common/results.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. -# Originally contributed by Check Point Software Technologies, Ltd. - -import logging -import socket -import time - -from lib.core.config import Config - -log = logging.getLogger(__name__) - -BUFSIZE = 1024*1024 - -def upload_to_host(file_path, dump_path): - nc = infd = None - try: - nc = NetlogFile(dump_path) - - infd = open(file_path, "rb") - buf = infd.read(BUFSIZE) - while buf: - nc.send(buf, retry=False) - buf = infd.read(BUFSIZE) - except Exception as e: - log.error("Exception uploading file %s to host: %s", file_path, e) - finally: - if infd: - infd.close() - if nc: - nc.close() - -class NetlogConnection(object): - def __init__(self, proto=""): - config = Config(cfg="analysis.conf") - self.hostip, self.hostport = config.ip, config.port - self.sock = None - self.proto = proto - - def connect(self): - # Try to connect as quickly as possible. Just sort of force it to - # connect with a short timeout. - while not self.sock: - try: - s = socket.create_connection((self.hostip, self.hostport), 0.1) - s.sendall(self.proto) - except socket.error: - time.sleep(0.1) - continue - - self.sock = s - - def send(self, data, retry=True): - if not self.sock: - self.connect() - - try: - self.sock.sendall(data) - except socket.error as e: - if retry: - self.connect() - self.send(data, retry=False) - else: - raise - except Exception as e: - log.error("Unhandled exception in NetlogConnection: %s", str(e)) - # We really have nowhere to log this, if the netlog connection - # does not work, we can assume that any logging won't work either. - # So we just fail silently. - self.close() - - def close(self): - try: - self.sock.close() - except Exception: - pass - -class NetlogFile(NetlogConnection): - def __init__(self, filepath): - self.filepath = filepath - NetlogConnection.__init__(self, proto="FILE\n{0}\n".format(self.filepath)) - self.connect() - -class NetlogHandler(logging.Handler, NetlogConnection): - def __init__(self): - logging.Handler.__init__(self) - NetlogConnection.__init__(self, proto="LOG\n") - self.connect() - - def emit(self, record): - msg = self.format(record) - self.send("{0}\n".format(msg)) diff --git a/analyzer/android/lib/common/utils.py b/analyzer/android/lib/common/utils.py deleted file mode 100644 index b363ba512d..0000000000 --- a/analyzer/android/lib/common/utils.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. -# Originally contributed by Check Point Software Technologies, Ltd. - -from lib.common.results import NetlogFile - -def send_file(name, data): - """Send file to result server""" - nf = NetlogFile(name) - nf.sock.sendall(data) - nf.close() diff --git a/analyzer/android/lib/core/__init__.py b/analyzer/android/lib/core/__init__.py deleted file mode 100644 index bf0bac1f5f..0000000000 --- a/analyzer/android/lib/core/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. -# Originally contributed by Check Point Software Technologies, Ltd. diff --git a/analyzer/android/lib/core/config.py b/analyzer/android/lib/core/config.py deleted file mode 100644 index f640645454..0000000000 --- a/analyzer/android/lib/core/config.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. -# Originally contributed by Check Point Software Technologies, Ltd. - -import ConfigParser - -class Config: - def __init__(self, cfg): - """@param cfg: configuration file.""" - config = ConfigParser.ConfigParser(allow_no_value=True) - config.read(cfg) - - for section in config.sections(): - for name, raw_value in config.items(section): - try: - value = config.getboolean(section, name) - except ValueError: - try: - value = config.getint(section, name) - except ValueError: - value = config.get(section, name) - - setattr(self, name, value) diff --git a/analyzer/android/lib/core/packages.py b/analyzer/android/lib/core/packages.py deleted file mode 100644 index 017475e024..0000000000 --- a/analyzer/android/lib/core/packages.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. -# Originally contributed by Check Point Software Technologies, Ltd. - -def choose_package(file_type, file_name): - """Choose analysis package due to file type and file extension. - @param file_type: file type. - @return: package or None. - """ - if not file_type: - return None - - file_type = file_type.lower() - file_name = file_name.lower() - - if "apk" in file_name: - return "apk" - elif "zip" in file_type: - return "apk" - # elif "DEX" in file_type: - # return "dex" - else: - return "apk" diff --git a/analyzer/android/lib/core/startup.py b/analyzer/android/lib/core/startup.py deleted file mode 100644 index f5b6ac1afa..0000000000 --- a/analyzer/android/lib/core/startup.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. -# Originally contributed by Check Point Software Technologies, Ltd. - -import os -import logging - -from lib.common.constants import PATHS -from lib.common.results import NetlogHandler - -log = logging.getLogger() - -def create_folders(): - """Create folders in PATHS.""" - for name, folder in PATHS.items(): - if os.path.exists(folder): - continue - - try: - os.makedirs(folder) - except OSError: - pass - -def init_logging(): - """Initialize logger.""" - formatter = logging.Formatter("%(asctime)s [%(name)s] %(levelname)s: %(message)s") - sh = logging.StreamHandler() - sh.setFormatter(formatter) - log.addHandler(sh) - - nh = NetlogHandler() - nh.setFormatter(formatter) - log.addHandler(nh) - - log.setLevel(logging.DEBUG) diff --git a/analyzer/android/modules/__init__.py b/analyzer/android/modules/__init__.py deleted file mode 100644 index bf0bac1f5f..0000000000 --- a/analyzer/android/modules/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. -# Originally contributed by Check Point Software Technologies, Ltd. diff --git a/analyzer/android/modules/auxiliary/__init__.py b/analyzer/android/modules/auxiliary/__init__.py deleted file mode 100644 index bf0bac1f5f..0000000000 --- a/analyzer/android/modules/auxiliary/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. -# Originally contributed by Check Point Software Technologies, Ltd. diff --git a/analyzer/android/modules/auxiliary/screenshots.py b/analyzer/android/modules/auxiliary/screenshots.py deleted file mode 100644 index 0057e8751d..0000000000 --- a/analyzer/android/modules/auxiliary/screenshots.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. -# Originally contributed by Check Point Software Technologies, Ltd. - -import time -import logging -import StringIO -from threading import Thread -from lib.common.abstracts import Auxiliary -from lib.common.results import NetlogFile -from lib.api.adb import take_screenshot -from lib.api.screenshot import Screenshot - -log = logging.getLogger(__name__) -SHOT_DELAY = 2 - -class Screenshots(Auxiliary, Thread): - """Take screenshots.""" - - def __init__(self): - Thread.__init__(self) - self.do_run = True - - def stop(self): - """Stop screenshotting.""" - self.do_run = False - - def run(self): - """Run screenshotting. - @return: operation status. - """ - img_counter = 0 - img_last = None - - while self.do_run: - time.sleep(SHOT_DELAY) - - try: - filename = "screenshot%s.jpg" % str(img_counter) - img_current = take_screenshot(filename) - if img_last: - if Screenshot().equal(img_last, img_current): - continue - - file = open(img_current, 'r') - tmpio = StringIO.StringIO(file.read()) - # now upload to host from the StringIO - nf = NetlogFile("shots/%s.jpg" % str(img_counter).rjust(4, "0")) - - for chunk in tmpio: - nf.sock.sendall(chunk) - - nf.close() - file.close() - img_counter += 1 - img_last = img_current - - except IOError as e: - log.error("Cannot take screenshot: %s", e) - continue - - return True diff --git a/analyzer/android/modules/packages/__init__.py b/analyzer/android/modules/packages/__init__.py deleted file mode 100644 index bf0bac1f5f..0000000000 --- a/analyzer/android/modules/packages/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. -# Originally contributed by Check Point Software Technologies, Ltd. diff --git a/analyzer/darwin/analyzer.py b/analyzer/darwin/analyzer.py deleted file mode 100644 index af80a71a40..0000000000 --- a/analyzer/darwin/analyzer.py +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2015 Dmitry Rodionov -# This software may be modified and distributed under the terms -# of the MIT license. See the LICENSE file for details. - -import logging -from sys import stderr -from hashlib import sha256 -from xmlrpclib import Server -from traceback import format_exc -from os import path, getcwd, makedirs - -from lib.common.config import Config -from lib.common.hashing import hash_file -from lib.common.results import NetlogHandler, upload_to_host -from lib.core.constants import PATHS -from lib.core.packages import choose_package_class -from lib.core.osx import set_wallclock -from lib.core.host import CuckooHost - -class Macalyzer(object): - """Cuckoo OS X analyser. - """ - - log = logging.getLogger() - target = None - - files_to_upload = [] - uploaded_hashes = [] - - def __init__(self, host, configuration=None): - self.config = configuration - self.host = host - - def bootstrap(self): - _create_result_folders() - _setup_logging() - self._detect_target() - - def run(self): - """Run analysis. - """ - self.bootstrap() - - self.log.debug("Starting analyzer from %s", getcwd()) - self.log.debug("Storing results at: %s", PATHS["root"]) - - package = self._setup_analysis_package() - - if self.config.clock: - set_wallclock(self.config.clock) - self._analysis(package) - - return self._complete() - - def _complete(self): - for f in self.files_to_upload: - self._upload_file(f) - return True - - # - # Implementation details - # - - def _detect_target(self): - if self.config.category == "file": - self.target = path.join("/tmp/", str(self.config.file_name)) - else: # It's not a file, but a URL - self.target = self.config.target - - def _setup_analysis_package(self): - # Do we have a suggestion about an analysis package? - if self.config.package: - suggestion = self.config.package - elif self.config.category != "file": - suggestion = "url" - else: - suggestion = None - # Try to figure out what analysis package to use with this target - kwargs = {"suggestion" : suggestion} - package_class = choose_package_class(self.config.file_type, - self.config.file_name, **kwargs) - if not package_class: - raise Exception("Could not find an appropriate analysis package") - # Package initialization - kwargs = { - "options" : self.config.get_options(), - "timeout" : self.config.timeout - } - return package_class(self.target, self.host, **kwargs) - - def _analysis(self, package): - package.start() - self.files_to_upload = package.touched_files - - def _upload_file(self, filepath): - if not path.isfile(filepath): - return - # Check whether we've already dumped this file - in that case skip it - try: - hashsum = hash_file(sha256, filepath) - if sha256 in self.uploaded_hashes: - return - except IOError as e: - self.log.info("Error dumping file from path \"%s\": %s", filepath, e) - return - filename = "%s_%s" % (hashsum[:16], path.basename(filepath)) - upload_path = path.join("files", filename) - - try: - upload_to_host(filepath, upload_path) - self.uploaded_hashes.append(hashsum) - except IOError as e: - self.log.error("Unable to upload dropped file at path \"%s\": %s", filepath, e) - -def _create_result_folders(): - for _, folder in PATHS.items(): - if path.exists(folder): - continue - try: - makedirs(folder) - except OSError: - pass - - -def _setup_logging(): - """ Initialize logger. """ - logger = logging.getLogger() - formatter = logging.Formatter("%(asctime)s [%(name)s] %(levelname)s: %(message)s") - - stream = logging.StreamHandler() - stream.setFormatter(formatter) - logger.addHandler(stream) - - netlog = NetlogHandler() - netlog.setFormatter(formatter) - logger.addHandler(netlog) - logger.setLevel(logging.DEBUG) - - - -if __name__ == "__main__": - success = False - error = "" - - try: - config = Config(cfg="analysis.conf") - cuckoo = CuckooHost(config.ip, config.port) - analyzer = Macalyzer(cuckoo, config) - success = analyzer.run() - - except KeyboardInterrupt: - error = "Keyboard Interrupt" - - except Exception as err: - error_exc = format_exc() - error = str(err) - if len(analyzer.log.handlers): - analyzer.log.exception(error_exc) - else: - stderr.write("{0}\n".format(error_exc)) - # Once the analysis is completed or terminated for any reason, we report - # back to the agent, notifying that it can report back to the host. - finally: - # Establish connection with the agent XMLRPC server. - server = Server("http://127.0.0.1:8000") - server.complete(success, error, PATHS["root"]) diff --git a/analyzer/darwin/lib/common/__init__.py b/analyzer/darwin/lib/common/__init__.py deleted file mode 100644 index 13fb1d219c..0000000000 --- a/analyzer/darwin/lib/common/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. diff --git a/analyzer/darwin/lib/common/config.py b/analyzer/darwin/lib/common/config.py deleted file mode 100644 index 51416269a9..0000000000 --- a/analyzer/darwin/lib/common/config.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import ConfigParser - -class Config: - def __init__(self, cfg): - """@param cfg: configuration file.""" - config = ConfigParser.ConfigParser(allow_no_value=True) - config.read(cfg) - - for section in config.sections(): - for name, raw_value in config.items(section): - if name == "file_name": - value = config.get(section, name) - else: - try: - value = config.getboolean(section, name) - except ValueError: - try: - value = config.getint(section, name) - except ValueError: - value = config.get(section, name) - setattr(self, name, value) - - def get_options(self): - """Get analysis options. - @return: options dict. - """ - # The analysis package can be provided with some options in the - # following format: - # option1=value1,option2=value2,option3=value3 - # - # Here we parse such options and provide a dictionary that will be made - # accessible to the analysis package. - options = {} - if hasattr(self, "options") and len(self.options) > 0: - try: - # Split the options by comma. - fields = self.options.split(",") - except ValueError: - pass - else: - for field in fields: - # Split the name and the value of the option. - try: - # Sometimes, we have a key without a value (i.e. it's a - # command line argument), so we can't use the - # `key, value = field.split("=", 1)` style here - parts = field.split("=", 1) - except ValueError: - pass - else: - key = parts[0].strip() - arg_prefix = "arg-" - if not key.startswith(arg_prefix): - # If the parsing went good, we add the option to the - # dictionary. - value = parts[1].strip() - options[key] = value - elif len(key) > len(arg_prefix): - # Remove "arg-" prefix from the key - key = key[4:]; parts[0] = key - # Add this key (with a value maybe) to the args - if "args" not in options: options["args"] = [] - options["args"] += parts - return options diff --git a/analyzer/darwin/lib/common/hashing.py b/analyzer/darwin/lib/common/hashing.py deleted file mode 100644 index 04b32e7e0e..0000000000 --- a/analyzer/darwin/lib/common/hashing.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -BUFSIZE = 1024*1024 - - -def hash_file(method, path): - """Calculates an hash on a file by path. - @param method: callable hashing method - @param path: file path - @return: computed hash string - """ - f = open(path, "rb") - h = method() - while True: - buf = f.read(BUFSIZE) - if not buf: - break - h.update(buf) - return h.hexdigest() diff --git a/analyzer/darwin/lib/common/results.py b/analyzer/darwin/lib/common/results.py deleted file mode 100644 index c222845d60..0000000000 --- a/analyzer/darwin/lib/common/results.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import time -import socket -import logging -from config import Config - -log = logging.getLogger(__name__) - -BUFSIZE = 1024*1024 - -def upload_to_host(file_path, dump_path): - nc = infd = None - try: - nc = NetlogFile(dump_path) - - infd = open(file_path, "rb") - buf = infd.read(BUFSIZE) - while buf: - nc.send(buf, retry=False) - buf = infd.read(BUFSIZE) - except Exception as e: - log.error("Exception uploading file %s to host: %s", file_path, e) - finally: - if infd: - infd.close() - if nc: - nc.close() - -class NetlogConnection(object): - def __init__(self, proto=""): - config = Config(cfg="analysis.conf") - self.hostip, self.hostport = config.ip, config.port - self.sock, self.file = None, None - self.proto = proto - - def connect(self): - i = 1 - # this can loop forever, if we can't connect the whole analysis is useless anyways - while True: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - s.connect((self.hostip, self.hostport)) - s.sendall(self.proto) - except: - time.sleep(i) - i = min(i + 1, 60) - else: - self.sock = s - self.file = s.makefile() - break - - def send(self, data, retry=True): - if not self.sock: self.connect() - - try: - self.sock.sendall(data) - except socket.error as e: - if retry: - self.connect() - self.send(data, retry=False) - else: - raise - except Exception as e: - log.error("Unhandled exception in NetlogConnection: %s", str(e)) - # We really have nowhere to log this, if the netlog connection - # does not work, we can assume that any logging won't work either. - # So we just fail silently. - self.close() - - def close(self): - try: - self.file.close() - self.sock.close() - except Exception: - pass - -class NetlogFile(NetlogConnection): - def __init__(self, filepath): - self.filepath = filepath - NetlogConnection.__init__(self, proto="FILE\n{0}\n".format(self.filepath)) - self.connect() - -class NetlogHandler(logging.Handler, NetlogConnection): - def __init__(self): - logging.Handler.__init__(self) - NetlogConnection.__init__(self, proto="LOG\n") - self.connect() - - def emit(self, record): - msg = self.format(record) - self.send("{0}\n".format(msg)) diff --git a/analyzer/darwin/lib/core/constants.py b/analyzer/darwin/lib/core/constants.py deleted file mode 100644 index 24fef1c274..0000000000 --- a/analyzer/darwin/lib/core/constants.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import os -from tempfile import gettempdir -from ..common.rand import random_string - -ROOT = os.path.join(gettempdir() + os.sep, random_string(6, 10)) - -PATHS = { - "root" : ROOT, - "logs" : os.path.join(ROOT, "logs"), - "files" : os.path.join(ROOT, "files"), - "shots" : os.path.join(ROOT, "shots"), - "memory" : os.path.join(ROOT, "memory"), - "drop" : os.path.join(ROOT, "drop") -} diff --git a/analyzer/darwin/modules/packages/zip.py b/analyzer/darwin/modules/packages/zip.py deleted file mode 100644 index eaf475c6da..0000000000 --- a/analyzer/darwin/modules/packages/zip.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2015 Dmitry Rodionov -# This software may be modified and distributed under the terms -# of the MIT license. See the LICENSE file for details. - -import logging -from shutil import move -from os import path, environ -from random import SystemRandom -from string import ascii_letters -from subprocess import check_output -from zipfile import ZipFile, BadZipfile -from lib.core.packages import Package, choose_package_class - -log = logging.getLogger(__name__) - -class Zip(Package): - - real_package = None - - def prepare(self): - password = self.options.get("password") - files = self._extract(self.target, password) - if not files or len(files) == 0: - raise Exception("Invalid (or empty) zip archive: %s" % self.target) - # Look for a file to analyse - target_name = self.options.get("file") - if not target_name: - # If no file name is provided via option, take the first file - target_name = files[0] - log.debug("Missing file option, auto executing: %s", target_name) - - filepath = path.join(environ.get("TEMP", "/tmp"), target_name) - # Remove the trailing slash (if any) - if filepath.endswith("/"): - self.target = filepath[:-1] - else: - self.target = filepath - - # Since we don't know what kind of file we're going to analyse, let's - # detect it automatically and create an appropriate analysis package - # for this file - file_info = _fileinfo(self.target) - pkg_class = choose_package_class(file_info, target_name) - - if not pkg_class: - raise Exception("Unable to detect analysis package for the file %s" % target_name) - else: - log.debug("Analysing file \"%s\" using package \"%s\"", target_name, pkg_class.__name__) - - kwargs = { - "options" : self.options, - "timeout" : self.timeout - } - # We'll forward start() method invocation to the proper package later - self.real_package = pkg_class(self.target, self.host, **kwargs) - - def start(self): - # We have nothing to do here; let the proper package do it's job - self.prepare() - if not self.real_package: - raise Exception("Invalid analysis package, aborting") - self.real_package.start() - - def _extract(self, filename, password): - archive_path = _prepare_archive_at_path(filename) - if not archive_path: - return None - # Extraction. - extract_path = environ.get("TEMP", "/tmp") - with ZipFile(archive_path, "r") as archive: - try: - archive.extractall(path=extract_path, pwd=password) - except BadZipfile: - raise Exception("Invalid Zip file") - # Try to extract it again, but with a default password - except RuntimeError: - try: - archive.extractall(path=extract_path, pwd="infected") - except RuntimeError as err: - raise Exception("Unable to extract Zip file: %s" % err) - finally: - self._extract_nested_archives(archive, extract_path, password) - return archive.namelist() - - def _extract_nested_archives(self, archive, where, password): - for name in archive.namelist(): - if name.endswith(".zip"): - self._extract(path.join(where, name), password) - - -def _prepare_archive_at_path(filename): - """ Verifies that there's a readable zip archive at the given path. - - This function returns a new name for the archive (for most cases it's - the same as the original one; but if an archive named "foo.zip" contains - a file named "foo" this archive will be renamed to avoid being overwrite. - """ - # Verify that the archive is actually readable - try: - with ZipFile(filename, "r") as archive: - archive.close() - except BadZipfile: - return None - # Test if zip file contains a file named as itself - if _is_overwritten(filename): - log.debug("ZIP file contains a file with the same name, original is \ - going to be overwrite") - # In this case we just change the file name - new_zip_path = filename + _random_extension() - move(filename, new_zip_path) - filename = new_zip_path - return filename - - -def _is_overwritten(zip_path): - archive = ZipFile(zip_path, "r") - try: - # Test if zip file contains a file named as itself - return any(n == path.basename(zip_path) for n in archive.namelist()) - except BadZipfile: - raise Exception("Invalid Zip file") - - -def _random_extension(length=5): - return '.' + ''.join(SystemRandom().choice(ascii_letters) for _ in range(length)) - - -def _fileinfo(target): - raw = check_output(["file", target]) - # The utility has the following output format: "%filename%: %description%", - # so we just skip everything before the actual description - return raw[raw.index(":")+2:] diff --git a/analyzer/linux/analyzer.py b/analyzer/linux/analyzer.py deleted file mode 100644 index 3227a2015a..0000000000 --- a/analyzer/linux/analyzer.py +++ /dev/null @@ -1,367 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import os -import sys -import pkgutil -import logging -import tempfile -import xmlrpclib -import traceback -import time -import datetime - -from lib.api.process import Process -from lib.common.abstracts import Package, Auxiliary -from lib.common.constants import PATHS -from lib.common.exceptions import CuckooError, CuckooPackageError -from lib.common.results import upload_to_host -from lib.core.config import Config -from lib.core.startup import create_folders, init_logging -from modules import auxiliary - -log = logging.getLogger() - -PID = os.getpid() -FILES_LIST = set() -DUMPED_LIST = set() -PROCESS_LIST = set() -SEEN_LIST = set() -PPID = Process(pid=PID).get_parent_pid() - -def add_pids(pids): - """Add PID.""" - if not isinstance(pids, (tuple, list, set)): - pids = [pids,] - - for pid in pids: - log.info("Added new process to list with pid: %s", pid) - pid = int(pid) - if not pid in SEEN_LIST: PROCESS_LIST.add(pid) - SEEN_LIST.add(pid) - -def dump_files(): - """Dump all the dropped files.""" - for file_path in FILES_LIST: - log.info("PLS IMPLEMENT DUMP, want to dump %s", file_path) - -class Analyzer: - """Cuckoo Linux Analyzer. - - This class handles the initialization and execution of the analysis - procedure, including the auxiliary modules and the analysis packages. - """ - - def __init__(self): - self.config = None - self.target = None - - def prepare(self): - """Prepare env for analysis.""" - - # Create the folders used for storing the results. - create_folders() - - # Initialize logging. - init_logging() - - # Parse the analysis configuration file generated by the agent. - self.config = Config(cfg="analysis.conf") - - if self.config.get("clock", None): - # Set virtual machine clock. - clock = datetime.datetime.strptime(self.config.clock, "%Y%m%dT%H:%M:%S") - # Setting date and time. - os.system("date -s \"{0}\"".format(clock.strftime("%y-%m-%d %H:%M:%S"))) - - # We update the target according to its category. If it's a file, then - # we store the path. - if self.config.category == "file": - self.target = os.path.join(tempfile.gettempdir(), self.config.file_name) - # If it's a URL, well.. we store the URL. - else: - self.target = self.config.target - - def complete(self): - """End analysis.""" - # Dump all the notified files. - dump_files() - - # Hell yeah. - log.info("Analysis completed.") - - def run(self): - """Run analysis. - @return: operation status. - """ - self.prepare() - - log.debug("Starting analyzer from: %s", os.getcwd()) - log.debug("Storing results at: %s", PATHS["root"]) - - # If no analysis package was specified at submission, we try to select - # one automatically. - if not self.config.package: - log.debug("No analysis package specified, trying to detect " - "it automagically.") - - if self.config.category == "file": - package = "generic" - else: - package = "wget" - - # If we weren't able to automatically determine the proper package, - # we need to abort the analysis. - if not package: - raise CuckooError("No valid package available for file " - "type: {0}".format(self.config.file_type)) - - log.info("Automatically selected analysis package \"%s\"", package) - # Otherwise just select the specified package. - else: - package = self.config.package - - # Generate the package path. - package_name = "modules.packages.%s" % package - - # Try to import the analysis package. - try: - __import__(package_name, globals(), locals(), ["dummy"], -1) - # If it fails, we need to abort the analysis. - except ImportError: - raise CuckooError("Unable to import package \"{0}\", does " - "not exist.".format(package_name)) - - # Initialize the package parent abstract. - Package() - - # Enumerate the abstract subclasses. - try: - package_class = Package.__subclasses__()[0] - except IndexError as e: - raise CuckooError("Unable to select package class " - "(package={0}): {1}".format(package_name, e)) - - # Initialize the analysis package. - pack = package_class(self.config.get_options()) - - # Initialize Auxiliary modules - Auxiliary() - prefix = auxiliary.__name__ + "." - for loader, name, ispkg in pkgutil.iter_modules(auxiliary.__path__, prefix): - if ispkg: - continue - - # Import the auxiliary module. - try: - __import__(name, globals(), locals(), ["dummy"], -1) - except ImportError as e: - log.warning("Unable to import the auxiliary module " - "\"%s\": %s", name, e) - - # Walk through the available auxiliary modules. - aux_enabled, aux_avail = [], [] - for module in sorted(Auxiliary.__subclasses__(), key=lambda x: x.priority, reverse=True): - # Try to start the auxiliary module. - try: - aux = module() - aux_avail.append(aux) - aux.start() - except (NotImplementedError, AttributeError): - log.warning("Auxiliary module %s was not implemented", - aux.__class__.__name__) - continue - except Exception as e: - log.warning("Cannot execute auxiliary module %s: %s", - aux.__class__.__name__, e) - continue - finally: - log.debug("Started auxiliary module %s", - aux.__class__.__name__) - aux_enabled.append(aux) - - # Start analysis package. If for any reason, the execution of the - # analysis package fails, we have to abort the analysis. - try: - pids = pack.start(self.target) - except NotImplementedError: - raise CuckooError("The package \"{0}\" doesn't contain a run " - "function.".format(package_name)) - except CuckooPackageError as e: - raise CuckooError("The package \"{0}\" start function raised an " - "error: {1}".format(package_name, e)) - except Exception as e: - raise CuckooError("The package \"{0}\" start function encountered " - "an unhandled exception: " - "{1}".format(package_name, e)) - - # If the analysis package returned a list of process IDs, we add them - # to the list of monitored processes and enable the process monitor. - if pids: - add_pids(pids) - pid_check = True - - # If the package didn't return any process ID (for example in the case - # where the package isn't enabling any behavioral analysis), we don't - # enable the process monitor. - else: - log.info("No process IDs returned by the package, running " - "for the full timeout.") - pid_check = False - - # Check in the options if the user toggled the timeout enforce. If so, - # we need to override pid_check and disable process monitor. - if self.config.enforce_timeout: - log.info("Enabled timeout enforce, running for the full timeout.") - pid_check = False - - time_counter = 0 - - while True: - time_counter += 1 - if time_counter == int(self.config.timeout): - log.info("Analysis timeout hit, terminating analysis.") - break - - try: - # If the process monitor is enabled we start checking whether - # the monitored processes are still alive. - if pid_check: - for pid in list(PROCESS_LIST): - if not Process(pid=pid).is_alive(): - log.info("Process with pid %s has terminated", pid) - PROCESS_LIST.remove(pid) - - # ask the package if it knows any new pids - add_pids(pack.get_pids()) - - # also ask the auxiliaries - for aux in aux_avail: - add_pids(aux.get_pids()) - - # If none of the monitored processes are still alive, we - # can terminate the analysis. - if not PROCESS_LIST: - log.info("Process list is empty, " - "terminating analysis.") - break - - # Update the list of monitored processes available to the - # analysis package. It could be used for internal - # operations within the module. - pack.set_pids(PROCESS_LIST) - - try: - # The analysis packages are provided with a function that - # is executed at every loop's iteration. If such function - # returns False, it means that it requested the analysis - # to be terminate. - if not pack.check(): - log.info("The analysis package requested the " - "termination of the analysis.") - break - - # If the check() function of the package raised some exception - # we don't care, we can still proceed with the analysis but we - # throw a warning. - except Exception as e: - log.warning("The package \"%s\" check function raised " - "an exception: %s", package_name, e) - except Exception as e: - log.exception("The PID watching loop raised an exception: %s", e) - finally: - # Zzz. - time.sleep(1) - - try: - # Before shutting down the analysis, the package can perform some - # final operations through the finish() function. - pack.finish() - except Exception as e: - log.warning("The package \"%s\" finish function raised an " - "exception: %s", package_name, e) - - try: - # Upload files the package created to package_files in the results folder - package_files = pack.package_files() - if package_files != None: - for package in package_files: - upload_to_host(package[0], os.path.join("package_files", package[1])); - except Exception as e: - log.warning("The package \"%s\" package_files function raised an " - "exception: %s", package_name, e) - - # Terminate the Auxiliary modules. - for aux in sorted(aux_enabled, key=lambda x: x.priority): - try: - aux.stop() - except (NotImplementedError, AttributeError): - continue - except Exception as e: - log.warning("Cannot terminate auxiliary module %s: %s", - aux.__class__.__name__, e) - - if self.config.terminate_processes: - # Try to terminate remaining active processes. We do this to make sure - # that we clean up remaining open handles (sockets, files, etc.). - log.info("Terminating remaining processes before shutdown.") - - for pid in PROCESS_LIST: - proc = Process(pid=pid) - if proc.is_alive(): - try: - proc.terminate() - except: - continue - - # Run the finish callback of every available Auxiliary module. - for aux in aux_avail: - try: - aux.finish() - except (NotImplementedError, AttributeError): - continue - except Exception as e: - log.warning("Exception running finish callback of auxiliary " - "module %s: %s", aux.__class__.__name__, e) - - # Let's invoke the completion procedure. - self.complete() - - return True - -if __name__ == "__main__": - success = False - error = "" - - try: - # Initialize the main analyzer class. - analyzer = Analyzer() - - # Run it and wait for the response. - success = analyzer.run() - - # This is not likely to happen. - except KeyboardInterrupt: - error = "Keyboard Interrupt" - - # If the analysis process encountered a critical error, it will raise a - # CuckooError exception, which will force the termination of the analysis. - # Notify the agent of the failure. Also catch unexpected exceptions. - except Exception as e: - # Store the error. - error_exc = traceback.format_exc() - error = str(e) - - # Just to be paranoid. - if len(log.handlers): - log.exception(error_exc) - else: - sys.stderr.write("{0}\n".format(error_exc)) - - # Once the analysis is completed or terminated for any reason, we report - # back to the agent, notifying that it can report back to the host. - finally: - # Establish connection with the agent XMLRPC server. - server = xmlrpclib.Server("http://127.0.0.1:8000") - server.complete(success, error, PATHS["root"]) diff --git a/analyzer/linux/lib/api/process.py b/analyzer/linux/lib/api/process.py deleted file mode 100644 index 503679bb1c..0000000000 --- a/analyzer/linux/lib/api/process.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import os -import subprocess -import logging - -log = logging.getLogger(__name__) - -class Process: - """Linux process.""" - first_process = True - first_process_pid = None - - def __init__(self, pid=0): - """@param pid: PID. - """ - self.pid = pid - - def is_alive(self): - if not os.path.exists("/proc/%u" % self.pid): return False - status = self.get_proc_status() - if not status: return False - if "zombie" in status.get("State:", ""): return False - return True - - def get_parent_pid(self): - return self.get_proc_status().get("PPid", None) - - def get_proc_status(self): - try: - status = open("/proc/%u/status" % self.pid).readlines() - status_values = dict((i[0], i[1]) for i in [j.strip().split(None, 1) for j in status]) - return status_values - except: - log.critical("could not get process status for pid %u", self.pid) - return {} - - def execute(self, cmd): - self.proc = proc = subprocess.Popen(cmd) - self.pid = proc.pid - return True diff --git a/analyzer/linux/lib/common/abstracts.py b/analyzer/linux/lib/common/abstracts.py deleted file mode 100644 index 8e1473fbad..0000000000 --- a/analyzer/linux/lib/common/abstracts.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import os - -from lib.api.process import Process -from lib.common.exceptions import CuckooPackageError - -class Package(object): - """Base abstract analysis package.""" - PATHS = [] - - def __init__(self, options={}): - """@param options: options dict.""" - self.options = options - self.pids = [] - - def set_pids(self, pids): - """Update list of monitored PIDs in the package context. - @param pids: list of pids. - """ - self.pids = pids - - def start(self): - """Run analysis package. - @raise NotImplementedError: this method is abstract. - """ - raise NotImplementedError - - def check(self): - """Check.""" - return True - - def execute(self, cmd): - """Starts an executable for analysis. - @param path: executable path - @param args: executable arguments - @return: process pid - """ - p = Process() - if not p.execute(cmd): - raise CuckooPackageError("Unable to execute the initial process, " - "analysis aborted.") - - return p.pid - - def package_files(self): - """A list of files to upload to host. - The list should be a list of tuples (, ). - (package_files is a folder that will be created in analysis folder). - """ - return None - - def finish(self): - """Finish run. - If specified to do so, this method dumps the memory of - all running processes. - """ - if self.options.get("procmemdump"): - for pid in self.pids: - p = Process(pid=pid) - p.dump_memory() - - return True - - def get_pids(self): - return [] - -class Auxiliary(object): - priority = 0 - - def get_pids(self): - return [] diff --git a/analyzer/linux/lib/common/constants.py b/analyzer/linux/lib/common/constants.py deleted file mode 100644 index b503af894b..0000000000 --- a/analyzer/linux/lib/common/constants.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import os -import tempfile -from lib.common.utils import random_string - -ROOT = os.path.join(tempfile.gettempdir(), random_string(6, 10)) - -PATHS = {"root" : ROOT, - "logs" : os.path.join(ROOT, "logs"), - "files" : os.path.join(ROOT, "files"), - "shots" : os.path.join(ROOT, "shots"), - "memory" : os.path.join(ROOT, "memory"), - "drop" : os.path.join(ROOT, "drop")} diff --git a/analyzer/linux/lib/common/exceptions.py b/analyzer/linux/lib/common/exceptions.py deleted file mode 100644 index d37f525b40..0000000000 --- a/analyzer/linux/lib/common/exceptions.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -class CuckooError(Exception): - pass - -class CuckooPackageError(Exception): - pass \ No newline at end of file diff --git a/analyzer/linux/lib/common/hashing.py b/analyzer/linux/lib/common/hashing.py deleted file mode 100644 index ec76024b37..0000000000 --- a/analyzer/linux/lib/common/hashing.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import hashlib - -BUFSIZE = 1024*1024 - -def sha256_file(path): - return hash_file(hashlib.sha256, path) - -def hash_file(method, path): - """Calculates an hash on a file by path. - @param method: callable hashing method - @param path: file path - @return: computed hash string - """ - f = open(path, "rb") - h = method() - while True: - buf = f.read(BUFSIZE) - if not buf: - break - h.update(buf) - return h.hexdigest() diff --git a/analyzer/linux/lib/common/results.py b/analyzer/linux/lib/common/results.py deleted file mode 100644 index 02c6f52ad6..0000000000 --- a/analyzer/linux/lib/common/results.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import logging -import socket -import time - -from lib.core.config import Config - -log = logging.getLogger(__name__) - -BUFSIZE = 1024*1024 - -def upload_to_host(file_path, dump_path): - nc = infd = None - try: - nc = NetlogFile(dump_path) - - infd = open(file_path, "rb") - buf = infd.read(BUFSIZE) - while buf: - nc.send(buf, retry=False) - buf = infd.read(BUFSIZE) - except Exception as e: - log.error("Exception uploading file %s to host: %s", file_path, e) - finally: - if infd: - infd.close() - if nc: - nc.close() - -class NetlogConnection(object): - def __init__(self, proto=""): - config = Config(cfg="analysis.conf") - self.hostip, self.hostport = config.ip, config.port - self.sock, self.file = None, None - self.proto = proto - - def connect(self): - i = 1 - # this can loop forever, if we can't connect the whole analysis is useless anyways - while True: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - s.connect((self.hostip, self.hostport)) - s.sendall(self.proto) - except: - time.sleep(i) - i = min(i + 1, 60) - else: - self.sock = s - self.file = s.makefile() - break - - def send(self, data, retry=True): - if not self.sock: self.connect() - - try: - self.sock.sendall(data) - except socket.error as e: - if retry: - self.connect() - self.send(data, retry=False) - else: - raise - except Exception as e: - log.error("Unhandled exception in NetlogConnection: %s", str(e)) - # We really have nowhere to log this, if the netlog connection - # does not work, we can assume that any logging won't work either. - # So we just fail silently. - self.close() - - def close(self): - try: - self.file.close() - self.sock.close() - except Exception: - pass - -class NetlogFile(NetlogConnection): - def __init__(self, filepath): - self.filepath = filepath - NetlogConnection.__init__(self, proto="FILE\n{0}\n".format(self.filepath)) - self.connect() - -class NetlogHandler(logging.Handler, NetlogConnection): - def __init__(self): - logging.Handler.__init__(self) - NetlogConnection.__init__(self, proto="LOG\n") - self.connect() - - def emit(self, record): - msg = self.format(record) - self.send("{0}\n".format(msg)) diff --git a/analyzer/linux/lib/common/utils.py b/analyzer/linux/lib/common/utils.py deleted file mode 100644 index ad9c368fcc..0000000000 --- a/analyzer/linux/lib/common/utils.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import random -import string - -def random_string(minimum, maximum=None): - if maximum is None: - maximum = minimum - - count = random.randint(minimum, maximum) - return "".join(random.choice(string.ascii_letters) for x in xrange(count)) - -def random_integer(digits): - start = 10 ** (digits - 1) - end = (10 ** digits) - 1 - return random.randint(start, end) diff --git a/analyzer/linux/lib/core/config.py b/analyzer/linux/lib/core/config.py deleted file mode 100644 index 35b8e772c7..0000000000 --- a/analyzer/linux/lib/core/config.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import ConfigParser - -class Config: - def __init__(self, cfg): - """@param cfg: configuration file.""" - config = ConfigParser.ConfigParser(allow_no_value=True) - config.read(cfg) - - for section in config.sections(): - for name, raw_value in config.items(section): - if name == "file_name": - value = config.get(section, name) - else: - try: - value = config.getboolean(section, name) - except ValueError: - try: - value = config.getint(section, name) - except ValueError: - value = config.get(section, name) - setattr(self, name, value) - - def get(self, name, default=None): - if hasattr(self, name): - return getattr(self, name) - return default - - def get_options(self): - """Get analysis options. - @return: options dict. - """ - # The analysis package can be provided with some options in the - # following format: - # option1=value1,option2=value2,option3=value3 - # - # Here we parse such options and provide a dictionary that will be made - # accessible to the analysis package. - options = {} - if hasattr(self, "options"): - try: - # Split the options by comma. - fields = self.options.split(",") - except ValueError as e: - pass - else: - for field in fields: - # Split the name and the value of the option. - try: - key, value = field.split("=", 1) - except ValueError as e: - pass - else: - # If the parsing went good, we add the option to the - # dictionary. - options[key.strip()] = value.strip() - - return options diff --git a/analyzer/linux/lib/core/startup.py b/analyzer/linux/lib/core/startup.py deleted file mode 100644 index 7373bd886e..0000000000 --- a/analyzer/linux/lib/core/startup.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import os -import logging - -from lib.common.constants import PATHS -from lib.common.results import NetlogHandler - -log = logging.getLogger() - -def create_folders(): - """Create folders in PATHS.""" - for name, folder in PATHS.items(): - if os.path.exists(folder): - continue - - try: - os.makedirs(folder) - except OSError: - pass - -def init_logging(): - """Initialize logger.""" - formatter = logging.Formatter("%(asctime)s [%(name)s] %(levelname)s: %(message)s") - sh = logging.StreamHandler() - sh.setFormatter(formatter) - log.addHandler(sh) - - nh = NetlogHandler() - nh.setFormatter(formatter) - log.addHandler(nh) - - log.setLevel(logging.DEBUG) diff --git a/analyzer/linux/modules/auxiliary/lkm.py b/analyzer/linux/modules/auxiliary/lkm.py deleted file mode 100644 index 710aa7291a..0000000000 --- a/analyzer/linux/modules/auxiliary/lkm.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import os -import subprocess -import logging -import platform - -from lib.common.abstracts import Auxiliary -from lib.common.results import NetlogFile -from lib.core.config import Config - -log = logging.getLogger(__name__) - -class LKM(Auxiliary): - """helper LKM for sleep skipping etc""" - - def __init__(self): - self.config = Config(cfg="analysis.conf") - self.pids_reported = set() - - def start(self): - # highest priority: if the vm config specifies the path - if self.config.get("analyzer_lkm_path", None) and os.path.exists(self.config.get("analyzer_lkm_path")): - path = self.config.get("analyzer_lkm_path") - # next: if the analyzer was uploaded with a module for our platform - elif os.path.exists(os.path.join(platform.machine(), "probelkm.ko")): - path = os.path.join(platform.machine(), "probelkm.ko") - # next: default path inside the machine - elif os.path.exists("/root/.cuckoo/probelkm.ko"): - path = "/root/.cuckoo/probelkm.ko" - # next: generic module uploaded with the analyzer (single arch setup maybe?) - elif os.path.exists("probelkm.ko"): - path = "probelkm.ko" - else: - log.warning("Could not find probelkm :(") - return False - - os.system("insmod %s trace_descendants=1 target_pid=%u" % (path, os.getpid())) - return True - - def get_pids(self): - new = [] - - fd = open("/var/log/kern.log") - for line in fd: - if not "[probelkm]" in line: continue - pos1 = line.find("forked to ") - pos2 = line.find("@", pos1+10) - if pos1 == -1 or pos2 == -1: continue - - forked_pid = int(line[pos1+10:pos2]) - - if forked_pid in self.pids_reported: - continue - - self.pids_reported.add(forked_pid) - new.append(forked_pid) - - return new - - def stop(self): - # i guess we don't need to unload at all - #os.system("rmmod probelkm") - - # now upload the logfile - nf = NetlogFile("logs/all.lkm") - - fd = open("/var/log/kern.log") - for line in fd: - if not "[probelkm]" in line: continue - nf.sock.sendall(line) # dirty direct send, no reconnecting - - fd.close() - nf.close() diff --git a/analyzer/linux/modules/auxiliary/stap.py b/analyzer/linux/modules/auxiliary/stap.py deleted file mode 100644 index 95341605e0..0000000000 --- a/analyzer/linux/modules/auxiliary/stap.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import os -import subprocess -import time -import logging -import platform - -from lib.common.abstracts import Auxiliary -from lib.common.results import NetlogFile -from lib.core.config import Config - -log = logging.getLogger(__name__) - -class STAP(Auxiliary): - """system-wide syscall trace with stap.""" - priority = -10 # low prio to wrap tightly around the analysis - - def __init__(self): - self.config = Config(cfg="analysis.conf") - self.fallback_strace = False - - def start(self): - # helper function locating the stap module - def has_stap(p): - files = os.listdir(p) - only_stap = [fn for fn in os.listdir(p) if fn.startswith("stap_") and fn.endswith(".ko")] - if only_stap: return os.path.join(p, only_stap[0]) - return False - - # highest priority: if the vm config specifies the path - if self.config.get("analyzer_stap_path", None) and os.path.exists(self.config.get("analyzer_stap_path")): - path = self.config.get("analyzer_lkm_path") - # next: if a module was uploaded with the analyzer for our platform - elif os.path.exists(platform.machine()) and has_stap(platform.machine()): - path = has_stap(platform.machine()) - # next: default path inside the machine - elif os.path.exists("/root/.cuckoo") and has_stap("/root/.cuckoo"): - path = has_stap("/root/.cuckoo") - # next: generic module uploaded with the analyzer (single arch setup maybe?) - elif has_stap("."): - path = has_stap(".") - else: - # we can't find the stap module, fallback to strace - log.warning("Could not find STAP LKM, falling back to strace.") - return self.start_strace() - - stap_start = time.time() - stderrfd = open("stap.stderr", "wb") - self.proc = subprocess.Popen(["staprun", "-v", "-x", str(os.getpid()), "-o", "stap.log", path], stderr=stderrfd) - - # read from stderr until the tap script is compiled - # while True: - # if not self.proc.poll() is None: - # break - # line = self.proc.stderr.readline() - # print "DBG LINE", line - # if "Pass 5: starting run." in line: - # break - - time.sleep(10) - stap_stop = time.time() - log.info("STAP aux module startup took %.2f seconds" % (stap_stop - stap_start)) - return True - - def start_strace(self): - try: os.mkdir("strace") - except: pass # don't worry, it exists - - stderrfd = open("strace/strace.stderr", "wb") - self.proc = subprocess.Popen(["strace", "-ff", "-o", "strace/straced", "-p", str(os.getpid())], stderr=stderrfd) - self.fallback_strace = True - return True - - def get_pids(self): - if self.fallback_strace: - return [self.proc.pid, ] - return [] - - def stop(self): - try: - r = self.proc.poll() - log.debug("stap subprocess retval %r", r) - self.proc.kill() - except Exception as e: - log.warning("Exception killing stap: %s", e) - - if os.path.exists("stap.log"): - # now upload the logfile - nf = NetlogFile("logs/all.stap") - - fd = open("stap.log", "rb") - for chunk in fd: - nf.sock.sendall(chunk) # dirty direct send, no reconnecting - - fd.close() - nf.close() - - # in case we fell back to strace - if os.path.exists("strace"): - for fn in os.listdir("strace"): - # we don't need the logs from the analyzer python process itself - if fn == "straced.%u" % os.getpid(): continue - - fp = os.path.join("strace", fn) - - # now upload the logfile - nf = NetlogFile("logs/%s" % fn) - - fd = open(fp, "rb") - for chunk in fd: - nf.sock.sendall(chunk) # dirty direct send, no reconnecting - - fd.close() - nf.close() diff --git a/analyzer/linux/modules/packages/generic.py b/analyzer/linux/modules/packages/generic.py deleted file mode 100644 index 44d662b537..0000000000 --- a/analyzer/linux/modules/packages/generic.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import os -from random import randint - -from lib.common.abstracts import Package - -class Generic(Package): - """Generic analysis package. Uses shell based execution. - """ - - def __init__(self, *args, **kwargs): - Package.__init__(self, *args, **kwargs) - self.seen_pids = set() - - def start(self, path): - os.chmod(path, 0o755) - return self.execute(["sh", "-c", path]) - - def get_pids(self): - probelkm_pids = set() - - fd = open("/var/log/kern.log") - for line in fd: - if not "[probelkm]" in line: continue - if "forked to" in line: - # [probelkm] task 2102@0x00007fa5d0b8b576 forked to 2107@0xffffffff81352f6d - parts = line[line.find("[probelkm]"):].split() - newtask = parts[-1] - pid, rip = newtask.split("@") - probelkm_pids.add(int(pid)) - - new_pids = probelkm_pids - self.seen_pids - self.seen_pids |= new_pids - return list(new_pids) diff --git a/analyzer/windows/analyzer.py b/analyzer/windows/analyzer.py deleted file mode 100644 index 85a6cd7586..0000000000 --- a/analyzer/windows/analyzer.py +++ /dev/null @@ -1,724 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import os -import sys -import socket -import struct -import pkgutil -import logging -import hashlib -import threading -import traceback -import urllib -import urllib2 -import xmlrpclib -from datetime import datetime - -from lib.api.process import Process -from lib.common.abstracts import Package, Auxiliary -from lib.common.constants import PATHS, SHUTDOWN_MUTEX -from lib.common.defines import KERNEL32 -from lib.common.exceptions import CuckooError, CuckooPackageError -from lib.common.hashing import hash_file -from lib.common.rand import random_string -from lib.common.results import upload_to_host -from lib.core.config import Config -from lib.core.packages import choose_package -from lib.core.pipe import PipeServer, PipeForwarder, PipeDispatcher -from lib.core.privileges import grant_debug_privilege -from lib.core.startup import create_folders, init_logging -from modules import auxiliary - -log = logging.getLogger("analyzer") - -class Files(object): - PROTECTED_NAMES = () - - def __init__(self): - self.files = [] - self.dumped = [] - - def is_protected_filename(self, file_name): - """Do we want to inject into a process with this name?""" - return file_name.lower() in self.PROTECTED_NAMES - - def add_file(self, filepath): - """Add filepath to the list of files.""" - if filepath.lower() not in self.files: - log.info("Added new file to list with path: %s", filepath) - self.files.append(filepath.lower()) - - def dump_file(self, filepath): - """Dump a file to the host.""" - if not os.path.isfile(filepath): - log.warning("File at path \"%r\" does not exist, skip.", filepath) - return False - - # Check whether we've already dumped this file - in that case skip it. - try: - sha256 = hash_file(hashlib.sha256, filepath) - if sha256 in self.dumped: - return - except IOError as e: - log.info("Error dumping file from path \"%s\": %s", filepath, e) - return - - filename = "%s_%s" % (sha256[:16], os.path.basename(filepath)) - upload_path = os.path.join("files", filename) - - try: - upload_to_host(filepath, upload_path) - self.dumped.append(sha256) - except (IOError, socket.error) as e: - log.error("Unable to upload dropped file at path \"%s\": %s", - filepath, e) - - def delete_file(self, filepath): - """A file is about to removed and thus should be dumped right away.""" - self.dump_file(filepath) - - # Remove the filepath from the files list. - if filepath.lower() in self.files: - self.files.remove(filepath.lower()) - - def move_file(self, oldfilepath, newfilepath): - """A file will be moved - track this change.""" - if oldfilepath.lower() in self.files: - # Replace the entry with the new filepath. - index = self.files.index(oldfilepath.lower()) - self.files[index] = newfilepath.lower() - - def dump_files(self): - """Dump all pending files.""" - for filepath in self.files: - self.dump_file(filepath) - -class ProcessList(object): - def __init__(self): - self.pids = [] - self.pids_notrack = [] - - def add_pid(self, pid, track=True): - """Add a process identifier to the process list. - - Track determines whether the analyzer should be monitoring this - process, i.e., whether Cuckoo should wait for this process to finish. - """ - if int(pid) not in self.pids and int(pid) not in self.pids_notrack: - if track: - self.pids.append(int(pid)) - else: - self.pids_notrack.append(int(pid)) - - def add_pids(self, pids): - """Add one or more process identifiers to the process list.""" - if isinstance(pids, (tuple, list)): - for pid in pids: - self.add_pid(pid) - else: - self.add_pid(pids) - - def has_pid(self, pid, notrack=True): - """Is this process identifier being tracked?""" - if int(pid) in self.pids: - return True - - if notrack and int(pid) in self.pids_notrack: - return True - - return False - - def remove_pid(self, pid): - """Remove a process identifier from being tracked.""" - if pid in self.pids: - self.pids.remove(pid) - - if pid in self.pids_notrack: - self.pids_notrack.remove(pid) - -class CommandPipeHandler(object): - """Pipe Handler. - - This class handles the notifications received through the Pipe Server and - decides what to do with them. - """ - ignore_list = dict(pid=[]) - - def __init__(self, analyzer): - self.analyzer = analyzer - - def _handle_debug(self, data): - """Debug message from the monitor.""" - log.debug(data) - - def _handle_info(self, data): - """Regular message from the monitor.""" - log.info(data) - - def _handle_warning(self, data): - """Warning message from the monitor.""" - log.warning(data) - - def _handle_critical(self, data): - """Critical message from the monitor.""" - log.critical(data) - - def _handle_loaded(self, data): - """The monitor has loaded into a particular process.""" - if not data or data.count(",") != 1: - log.warning("Received loaded command with incorrect parameters, " - "skipping it.") - return - - pid, track = data.split(",") - if not pid.isdigit() or not track.isdigit(): - log.warning("Received loaded command with incorrect parameters, " - "skipping it.") - return - - self.analyzer.process_lock.acquire() - self.analyzer.process_list.add_pid(int(pid), track=int(track)) - self.analyzer.process_lock.release() - - log.debug("Loaded monitor into process with pid %s", pid) - - def _handle_getpids(self, data): - """Return the process identifiers of the agent and its parent - process.""" - return struct.pack("II", self.analyzer.pid, self.analyzer.ppid) - - def _inject_process(self, process_id, thread_id, mode): - """Helper function for injecting the monitor into a process.""" - # We acquire the process lock in order to prevent the analyzer to - # terminate the analysis while we are operating on the new process. - self.analyzer.process_lock.acquire() - - # Set the current DLL to the default one provided at submission. - dll = self.analyzer.default_dll - - if process_id in (self.analyzer.pid, self.analyzer.ppid): - if process_id not in self.ignore_list["pid"]: - log.warning("Received request to inject Cuckoo processes, " - "skipping it.") - self.ignore_list["pid"].append(process_id) - self.analyzer.process_lock.release() - return - - # We inject the process only if it's not being monitored already, - # otherwise we would generated polluted logs (if it wouldn't crash - # horribly to start with). - if self.analyzer.process_list.has_pid(process_id): - # This pid is already on the notrack list, move it to the - # list of tracked pids. - if not self.analyzer.process_list.has_pid(process_id, notrack=False): - log.debug("Received request to inject pid=%d. It was already " - "on our notrack list, moving it to the track list.") - - self.analyzer.process_list.remove_pid(process_id) - self.analyzer.process_list.add_pid(process_id) - self.ignore_list["pid"].append(process_id) - # Spit out an error once and just ignore it further on. - elif process_id not in self.ignore_list["pid"]: - log.debug("Received request to inject pid=%d, but we are " - "already injected there.", process_id) - self.ignore_list["pid"].append(process_id) - - # We're done operating on the processes list, release the lock. - self.analyzer.process_lock.release() - return - - # Open the process and inject the DLL. Hope it enjoys it. - proc = Process(pid=process_id, tid=thread_id) - - filename = os.path.basename(proc.get_filepath()) - - if not self.analyzer.files.is_protected_filename(filename): - # Add the new process ID to the list of monitored processes. - self.analyzer.process_list.add_pid(process_id) - - # We're done operating on the processes list, - # release the lock. Let the injection do its thing. - self.analyzer.process_lock.release() - - # If we have both pid and tid, then we can use APC to inject. - if process_id and thread_id: - proc.inject(dll, apc=True, mode="%s" % mode) - else: - proc.inject(dll, apc=False, mode="%s" % mode) - - log.info("Injected into process with pid %s and name %s", - proc.pid, filename) - - def _handle_process(self, data): - """Request for injection into a process.""" - # Parse the process identifier. - if not data or not data.isdigit(): - log.warning("Received PROCESS command from monitor with an " - "incorrect argument.") - return - - return self._inject_process(int(data), None, 0) - - def _handle_process2(self, data): - """Request for injection into a process using APC.""" - # Parse the process and thread identifier. - if not data or data.count(",") != 2: - log.warning("Received PROCESS2 command from monitor with an " - "incorrect argument.") - return - - pid, tid, mode = data.split(",") - if not pid.isdigit() or not tid.isdigit() or not mode.isdigit(): - log.warning("Received PROCESS2 command from monitor with an " - "incorrect argument.") - return - - return self._inject_process(int(pid), int(tid), int(mode)) - - def _handle_file_new(self, data): - """Notification of a new dropped file.""" - # Extract the file path and add it to the list. - self.analyzer.files.add_file(data.decode("utf8")) - - def _handle_file_del(self, data): - """Notification of a file being removed - we have to dump it before - it's being removed.""" - self.analyzer.files.delete_file(data.decode("utf8")) - - def _handle_file_move(self, data): - """A file is being moved - track these changes.""" - if "::" not in data: - log.warning("Received FILE_MOVE command from monitor with an " - "incorrect argument.") - return - - old_filepath, new_filepath = data.split("::", 1) - self.analyzer.files.move_file(old_filepath.decode("utf8"), - new_filepath.decode("utf8")) - - def dispatch(self, data): - response = "NOPE" - - if not data or ":" not in data: - log.critical("Unknown command received from the monitor: %r", - data.strip()) - else: - command, arguments = data.strip().split(":", 1) - - if not hasattr(self, "_handle_%s" % command.lower()): - log.critical("Unknown command received from the monitor: %r", - data.strip()) - else: - fn = getattr(self, "_handle_%s" % command.lower()) - response = fn(arguments) - - return response - -class Analyzer(object): - """Cuckoo Windows Analyzer. - - This class handles the initialization and execution of the analysis - procedure, including handling of the pipe server, the auxiliary modules and - the analysis packages. - """ - - def __init__(self): - self.config = None - self.target = None - self.do_run = True - self.time_counter = 0 - - self.process_lock = threading.Lock() - self.default_dll = None - self.pid = os.getpid() - self.ppid = Process(pid=self.pid).get_parent_pid() - self.files = Files() - self.process_list = ProcessList() - - def prepare(self): - """Prepare env for analysis.""" - # Get SeDebugPrivilege for the Python process. It will be needed in - # order to perform the injections. - grant_debug_privilege() - - # Create the folders used for storing the results. - create_folders() - - # Initialize logging. - init_logging() - - # Parse the analysis configuration file generated by the agent. - self.config = Config(cfg="analysis.conf") - - # Pass the configuration through to the Process class. - Process.set_config(self.config) - - # Set virtual machine clock. - clock = datetime.strptime(self.config.clock, "%Y%m%dT%H:%M:%S") - - # Setting date and time. - # NOTE: Windows system has only localized commands with date format - # following localization settings, so these commands for english date - # format cannot work in other localizations. - # In addition DATE and TIME commands are blocking if an incorrect - # syntax is provided, so an echo trick is used to bypass the input - # request and not block analysis. - os.system("echo:|date {0}".format(clock.strftime("%m-%d-%y"))) - os.system("echo:|time {0}".format(clock.strftime("%H:%M:%S"))) - - # Set the default DLL to be used for this analysis. - self.default_dll = self.config.options.get("dll") - - # If a pipe name has not set, then generate a random one. - if "pipe" in self.config.options: - self.config.pipe = "\\\\.\\PIPE\\%s" % self.config.options["pipe"] - else: - self.config.pipe = "\\\\.\\PIPE\\%s" % random_string(16, 32) - - # Generate a random name for the logging pipe server. - self.config.logpipe = "\\\\.\\PIPE\\%s" % random_string(16, 32) - - # Initialize and start the Command Handler pipe server. This is going - # to be used for communicating with the monitored processes. - self.command_pipe = PipeServer(PipeDispatcher, self.config.pipe, - message=True, - dispatcher=CommandPipeHandler(self)) - self.command_pipe.daemon = True - self.command_pipe.start() - - # Initialize and start the Log Pipe Server - the log pipe server will - # open up a pipe that monitored processes will use to send logs to - # before they head off to the host machine. - destination = self.config.ip, self.config.port - self.log_pipe_server = PipeServer(PipeForwarder, self.config.logpipe, - destination=destination) - self.log_pipe_server.daemon = True - self.log_pipe_server.start() - - # We update the target according to its category. If it's a file, then - # we store the target path. - if self.config.category == "file": - self.target = os.path.join(os.environ["TEMP"] + os.sep, - self.config.file_name) - # If it's a URL, well.. we store the URL. - else: - self.target = self.config.target - - def stop(self): - """Allows an auxiliary module to stop the analysis.""" - self.do_run = False - - def complete(self): - """End analysis.""" - # Stop the Pipe Servers. - self.command_pipe.stop() - self.log_pipe_server.stop() - - # Dump all the notified files. - self.files.dump_files() - - # Hell yeah. - log.info("Analysis completed.") - - def run(self): - """Run analysis. - @return: operation status. - """ - self.prepare() - - log.debug("Starting analyzer from: %s", os.getcwd()) - log.debug("Storing results at: %s", PATHS["root"]) - log.debug("Pipe server name: %s", self.config.pipe) - log.debug("Log pipe server name: %s", self.config.logpipe) - - # If no analysis package was specified at submission, we try to select - # one automatically. - if not self.config.package: - log.debug("No analysis package specified, trying to detect " - "it automagically.") - - # If the analysis target is a file, we choose the package according - # to the file format. - if self.config.category == "file": - package = choose_package(self.config.file_type, - self.config.file_name, - self.config.pe_exports.split(",")) - # If it's an URL, we'll just use the default Internet Explorer - # package. - else: - package = "ie" - - # If we weren't able to automatically determine the proper package, - # we need to abort the analysis. - if not package: - raise CuckooError("No valid package available for file " - "type: {0}".format(self.config.file_type)) - - log.info("Automatically selected analysis package \"%s\"", package) - # Otherwise just select the specified package. - else: - package = self.config.package - - # Generate the package path. - package_name = "modules.packages.%s" % package - - # Try to import the analysis package. - try: - __import__(package_name, globals(), locals(), ["dummy"], -1) - # If it fails, we need to abort the analysis. - except ImportError: - raise CuckooError("Unable to import package \"{0}\", does " - "not exist.".format(package_name)) - - # Initialize the package parent abstract. - Package() - - # Enumerate the abstract subclasses. - try: - package_class = Package.__subclasses__()[0] - except IndexError as e: - raise CuckooError("Unable to select package class " - "(package={0}): {1}".format(package_name, e)) - - # Initialize the analysis package. - package = package_class(self.config.options) - - # Move the sample to the current working directory as provided by the - # task - one is able to override the starting path of the sample. - # E.g., for some samples it might be useful to run from %APPDATA% - # instead of %TEMP%. - if self.config.category == "file": - self.target = package.move_curdir(self.target) - - # Initialize Auxiliary modules - Auxiliary() - prefix = auxiliary.__name__ + "." - for loader, name, ispkg in pkgutil.iter_modules(auxiliary.__path__, prefix): - if ispkg: - continue - - # Import the auxiliary module. - try: - __import__(name, globals(), locals(), ["dummy"], -1) - except ImportError as e: - log.warning("Unable to import the auxiliary module " - "\"%s\": %s", name, e) - - # Walk through the available auxiliary modules. - aux_enabled, aux_avail = [], [] - for module in Auxiliary.__subclasses__(): - # Try to start the auxiliary module. - try: - aux = module(options=self.config.options, analyzer=self) - aux_avail.append(aux) - aux.start() - except (NotImplementedError, AttributeError): - log.warning("Auxiliary module %s was not implemented", - aux.__class__.__name__) - except Exception as e: - log.warning("Cannot execute auxiliary module %s: %s", - aux.__class__.__name__, e) - else: - log.debug("Started auxiliary module %s", - aux.__class__.__name__) - aux_enabled.append(aux) - - # Start analysis package. If for any reason, the execution of the - # analysis package fails, we have to abort the analysis. - try: - pids = package.start(self.target) - except NotImplementedError: - raise CuckooError("The package \"{0}\" doesn't contain a run " - "function.".format(package_name)) - except CuckooPackageError as e: - raise CuckooError("The package \"{0}\" start function raised an " - "error: {1}".format(package_name, e)) - except Exception as e: - raise CuckooError("The package \"{0}\" start function encountered " - "an unhandled exception: " - "{1}".format(package_name, e)) - - # If the analysis package returned a list of process identifiers, we - # add them to the list of monitored processes and enable the process monitor. - if pids: - self.process_list.add_pids(pids) - pid_check = True - - # If the package didn't return any process ID (for example in the case - # where the package isn't enabling any behavioral analysis), we don't - # enable the process monitor. - else: - log.info("No process IDs returned by the package, running " - "for the full timeout.") - pid_check = False - - # Check in the options if the user toggled the timeout enforce. If so, - # we need to override pid_check and disable process monitor. - if self.config.enforce_timeout: - log.info("Enabled timeout enforce, running for the full timeout.") - pid_check = False - - while self.do_run: - self.time_counter += 1 - if self.time_counter == int(self.config.timeout): - log.info("Analysis timeout hit, terminating analysis.") - break - - # If the process lock is locked, it means that something is - # operating on the list of monitored processes. Therefore we - # cannot proceed with the checks until the lock is released. - if self.process_lock.locked(): - KERNEL32.Sleep(1000) - continue - - try: - # If the process monitor is enabled we start checking whether - # the monitored processes are still alive. - if pid_check: - for pid in self.process_list.pids: - if not Process(pid=pid).is_alive(): - log.info("Process with pid %s has terminated", pid) - self.process_list.remove_pid(pid) - - # If none of the monitored processes are still alive, we - # can terminate the analysis. - if not self.process_list.pids: - log.info("Process list is empty, " - "terminating analysis.") - break - - # Update the list of monitored processes available to the - # analysis package. It could be used for internal - # operations within the module. - package.set_pids(self.process_list.pids) - - try: - # The analysis packages are provided with a function that - # is executed at every loop's iteration. If such function - # returns False, it means that it requested the analysis - # to be terminate. - if not package.check(): - log.info("The analysis package requested the " - "termination of the analysis.") - break - - # If the check() function of the package raised some exception - # we don't care, we can still proceed with the analysis but we - # throw a warning. - except Exception as e: - log.warning("The package \"%s\" check function raised " - "an exception: %s", package_name, e) - finally: - # Zzz. - KERNEL32.Sleep(1000) - - if not self.do_run: - log.debug("The analyzer has been stopped on request by an " - "auxiliary module.") - - # Create the shutdown mutex. - KERNEL32.CreateMutexA(None, False, SHUTDOWN_MUTEX) - - try: - # Before shutting down the analysis, the package can perform some - # final operations through the finish() function. - package.finish() - except Exception as e: - log.warning("The package \"%s\" finish function raised an " - "exception: %s", package_name, e) - - try: - # Upload files the package created to package_files in the - # results folder. - for path, name in package.package_files() or []: - upload_to_host(path, os.path.join("package_files", name)) - except Exception as e: - log.warning("The package \"%s\" package_files function raised an " - "exception: %s", package_name, e) - - # Terminate the Auxiliary modules. - for aux in aux_enabled: - try: - aux.stop() - except (NotImplementedError, AttributeError): - continue - except Exception as e: - log.warning("Cannot terminate auxiliary module %s: %s", - aux.__class__.__name__, e) - - if self.config.terminate_processes: - # Try to terminate remaining active processes. We do this to make sure - # that we clean up remaining open handles (sockets, files, etc.). - log.info("Terminating remaining processes before shutdown.") - - for pid in self.process_list.pids: - proc = Process(pid=pid) - if proc.is_alive(): - try: - proc.terminate() - except: - continue - - # Run the finish callback of every available Auxiliary module. - for aux in aux_avail: - try: - aux.finish() - except (NotImplementedError, AttributeError): - continue - except Exception as e: - log.warning("Exception running finish callback of auxiliary " - "module %s: %s", aux.__class__.__name__, e) - - # Let's invoke the completion procedure. - self.complete() - - return True - -if __name__ == "__main__": - success = False - error = "" - - try: - # Initialize the main analyzer class. - analyzer = Analyzer() - - # Run it and wait for the response. - success = analyzer.run() - - data = { - "status": "complete", - "description": success, - } - # This is not likely to happen. - except KeyboardInterrupt: - error = "Keyboard Interrupt" - - # If the analysis process encountered a critical error, it will raise a - # CuckooError exception, which will force the termination of the analysis. - # Notify the agent of the failure. Also catch unexpected exceptions. - except Exception as e: - # Store the error. - error_exc = traceback.format_exc() - error = str(e) - - # Just to be paranoid. - if len(log.handlers): - log.exception(error_exc) - else: - sys.stderr.write("{0}\n".format(error_exc)) - - data = { - "status": "exception", - "description": error_exc, - } - finally: - # Report that we're finished. First try with the XML RPC thing and - # if that fails, attempt the new Agent. - try: - server = xmlrpclib.Server("http://127.0.0.1:8000") - server.complete(success, error, PATHS["root"]) - except xmlrpclib.ProtocolError: - urllib2.urlopen("http://127.0.0.1:8000/status", - urllib.urlencode(data)).read() diff --git a/analyzer/windows/bin/cuckoomon.dll b/analyzer/windows/bin/cuckoomon.dll deleted file mode 100644 index 9a6346270f..0000000000 Binary files a/analyzer/windows/bin/cuckoomon.dll and /dev/null differ diff --git a/analyzer/windows/bin/cuckoomon_bson.dll b/analyzer/windows/bin/cuckoomon_bson.dll deleted file mode 100644 index 0b25a4b33e..0000000000 Binary files a/analyzer/windows/bin/cuckoomon_bson.dll and /dev/null differ diff --git a/analyzer/windows/bin/cuckoomon_netlog.dll b/analyzer/windows/bin/cuckoomon_netlog.dll deleted file mode 100644 index 6fa67dd2e3..0000000000 Binary files a/analyzer/windows/bin/cuckoomon_netlog.dll and /dev/null differ diff --git a/analyzer/windows/bin/execsc.exe b/analyzer/windows/bin/execsc.exe deleted file mode 100755 index cc267585b5..0000000000 Binary files a/analyzer/windows/bin/execsc.exe and /dev/null differ diff --git a/analyzer/windows/lib/__init__.py b/analyzer/windows/lib/__init__.py deleted file mode 100644 index 13fb1d219c..0000000000 --- a/analyzer/windows/lib/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. diff --git a/analyzer/windows/lib/api/__init__.py b/analyzer/windows/lib/api/__init__.py deleted file mode 100644 index 13fb1d219c..0000000000 --- a/analyzer/windows/lib/api/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. diff --git a/analyzer/windows/lib/api/process.py b/analyzer/windows/lib/api/process.py deleted file mode 100644 index a03970cca1..0000000000 --- a/analyzer/windows/lib/api/process.py +++ /dev/null @@ -1,449 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import os -import logging -import random -import subprocess -import tempfile -import time -from ctypes import byref, c_ulong, create_string_buffer, c_int, sizeof -from ctypes import c_uint, c_wchar_p, create_unicode_buffer - -from lib.common.constants import PATHS, SHUTDOWN_MUTEX -from lib.common.defines import KERNEL32, NTDLL, SYSTEM_INFO, STILL_ACTIVE -from lib.common.defines import THREAD_ALL_ACCESS, PROCESS_ALL_ACCESS -from lib.common.defines import MEM_COMMIT, MEMORY_BASIC_INFORMATION -from lib.common.defines import MEM_IMAGE, MEM_MAPPED, MEM_PRIVATE -from lib.common.errors import get_error_string -from lib.common.exceptions import CuckooError -from lib.common.results import NetlogFile - -log = logging.getLogger(__name__) - -class Process(object): - """Windows process.""" - first_process = True - config = None - - def __init__(self, pid=None, tid=None, process_name=None): - """ - @param pid: process identifier. - @param tid: thread identifier. - @param process_name: process name. - """ - self.pid = pid - self.tid = tid - self.process_name = process_name - - @staticmethod - def set_config(config): - """Sets the analyzer configuration once.""" - Process.config = config - - def get_system_info(self): - """Get system information.""" - self.system_info = SYSTEM_INFO() - KERNEL32.GetSystemInfo(byref(self.system_info)) - - def open_process(self): - """Open a process handle.""" - return KERNEL32.OpenProcess(PROCESS_ALL_ACCESS, False, self.pid) - - def open_thread(self): - """Open a thread handle.""" - return KERNEL32.OpenThread(THREAD_ALL_ACCESS, False, self.tid) - - def exit_code(self): - """Get process exit code. - @return: exit code value. - """ - process_handle = self.open_process() - - exit_code = c_ulong(0) - KERNEL32.GetExitCodeProcess(process_handle, byref(exit_code)) - KERNEL32.CloseHandle(process_handle) - - return exit_code.value - - def get_filepath(self): - """Get process image file path. - @return: decoded file path. - """ - process_handle = self.open_process() - - NT_SUCCESS = lambda val: val >= 0 - - pbi = create_string_buffer(200) - size = c_int() - - # Set return value to signed 32bit integer. - NTDLL.NtQueryInformationProcess.restype = c_int - - ret = NTDLL.NtQueryInformationProcess(process_handle, - 27, - byref(pbi), - sizeof(pbi), - byref(size)) - - KERNEL32.CloseHandle(process_handle) - - if NT_SUCCESS(ret) and size.value > 8: - try: - fbuf = pbi.raw[8:] - fbuf = fbuf[:fbuf.find("\x00\x00")+1] - return fbuf.decode("utf16", errors="ignore") - except: - return "" - - return "" - - def is_alive(self): - """Process is alive? - @return: process status. - """ - return self.exit_code() == STILL_ACTIVE - - def get_parent_pid(self): - """Get the Parent Process ID.""" - process_handle = self.open_process() - - NT_SUCCESS = lambda val: val >= 0 - - pbi = (c_int * 6)() - size = c_int() - - # Set return value to signed 32bit integer. - NTDLL.NtQueryInformationProcess.restype = c_int - - ret = NTDLL.NtQueryInformationProcess(process_handle, - 0, - byref(pbi), - sizeof(pbi), - byref(size)) - - KERNEL32.CloseHandle(process_handle) - - if NT_SUCCESS(ret) and size.value == sizeof(pbi): - return pbi[5] - - return None - - def shortpath(self, path): - """Returns the shortpath for a file. - - As Python 2.7 does not support passing along unicode strings in - subprocess.Popen() and alike this will have to do. See also: - http://stackoverflow.com/questions/2595448/unicode-filename-to-python-subprocess-call - """ - KERNEL32.GetShortPathNameW.restype = c_uint - KERNEL32.GetShortPathNameW.argtypes = c_wchar_p, c_wchar_p, c_uint - - buf = create_unicode_buffer(0x8000) - KERNEL32.GetShortPathNameW(path, buf, len(buf)) - return buf.value - - def _encode_args(self, args): - """Convert a list of arguments to a string that can be passed along - on the command-line. - @param args: list of arguments - @return: the command-line equivalent - """ - ret = [] - for line in args: - if " " in line or '"' in line: - ret.append("\"%s\"" % line.replace('"', '\\"')) - else: - ret.append(line) - return " ".join(ret) - - def is32bit(self, pid=None, process_name=None, path=None): - """Is a PE file 32-bit or does a process identifier belong to a - 32-bit process. - @param pid: process identifier. - @param process_name: process name. - @param path: path to a PE file. - @return: boolean or exception. - """ - count = (pid is None) + (process_name is None) + (path is None) - if count != 2: - raise CuckooError("Invalid usage of is32bit, only one identifier " - "should be specified") - - is32bit_exe = os.path.join("bin", "is32bit.exe") - - if pid: - args = [is32bit_exe, "-p", "%s" % pid] - elif process_name: - args = [is32bit_exe, "-n", process_name] - # If we're running a 32-bit Python in a 64-bit Windows system and the - # path points to System32, then we hardcode it as being a 64-bit - # binary. (To be fair, a 64-bit Python on 64-bit Windows would also - # make the System32 binary 64-bit). - elif os.path.isdir("C:\\Windows\\Sysnative") and \ - path.lower().startswith("c:\\windows\\system32"): - return False - else: - args = [is32bit_exe, "-f", self.shortpath(path)] - - try: - bitsize = int(subprocess.check_output(args)) - except subprocess.CalledProcessError as e: - raise CuckooError("Error returned by is32bit: %s" % e) - - return bitsize == 32 - - def execute(self, path, args=None, dll=None, free=False, curdir=None, - source=None, mode=None, maximize=False): - """Execute sample process. - @param path: sample path. - @param args: process args. - @param dll: dll path. - @param free: do not inject our monitor. - @param curdir: current working directory. - @param source: process identifier or process name which will - become the parent process for the new process. - @param mode: monitor mode - which functions to instrument. - @param maximize: whether the GUI should be maximized. - @return: operation status. - """ - if not os.access(path, os.X_OK): - log.error("Unable to access file at path \"%s\", " - "execution aborted", path) - return False - - is32bit = self.is32bit(path=path) - - if not dll: - if is32bit: - dll = "monitor-x86.dll" - else: - dll = "monitor-x64.dll" - - dllpath = os.path.abspath(os.path.join("bin", dll)) - - if not os.path.exists(dllpath): - log.warning("No valid DLL specified to be injected, " - "injection aborted.") - return False - - if is32bit: - inject_exe = os.path.join("bin", "inject-x86.exe") - else: - inject_exe = os.path.join("bin", "inject-x64.exe") - - argv = [inject_exe, "--app", self.shortpath(path)] - - if args: - argv += ["--args", self._encode_args(args)] - - if free: - argv += ["--free"] - else: - argv += ["--apc", "--dll", dllpath, - "--config", self.drop_config(mode=mode)] - - if curdir: - argv += ["--curdir", self.shortpath(curdir)] - - if source: - if isinstance(source, (int, long)) or source.isdigit(): - argv += ["--from", "%s" % source] - else: - argv += ["--from-process", source] - - if maximize: - argv += ["--maximize"] - - try: - self.pid = int(subprocess.check_output(argv)) - except Exception: - log.error("Failed to execute process from path %r with " - "arguments %r (Error: %s)", path, argv, - get_error_string(KERNEL32.GetLastError())) - return False - - log.info("Successfully executed process from path %r with " - "arguments %r and pid %d", path, args or "", self.pid) - return True - - def terminate(self): - """Terminate process. - @return: operation status. - """ - process_handle = self.open_process() - - ret = KERNEL32.TerminateProcess(process_handle, 1) - KERNEL32.CloseHandle(process_handle) - - if ret: - log.info("Successfully terminated process with pid %d.", self.pid) - return True - else: - log.error("Failed to terminate process with pid %d.", self.pid) - return False - - def inject(self, dll=None, apc=False, track=True, mode=None): - """Inject our monitor into the specified process. - @param dll: Cuckoo DLL path. - @param apc: Use APC injection. - @param track: Track this process in the analyzer. - @param mode: Monitor mode - which functions to instrument. - """ - if not self.pid and not self.process_name: - log.warning("No valid pid or process name specified, " - "injection aborted.") - return False - - # Only check whether the process is still alive when it's identified - # by a process identifier. Not when it's identified by a process name. - if not self.process_name and not self.is_alive(): - log.warning("The process with pid %s is not alive, " - "injection aborted", self.pid) - return False - - if self.process_name: - is32bit = self.is32bit(process_name=self.process_name) - elif self.pid: - is32bit = self.is32bit(pid=self.pid) - - if not dll: - if is32bit: - dll = "monitor-x86.dll" - else: - dll = "monitor-x64.dll" - - dllpath = os.path.abspath(os.path.join("bin", dll)) - - if not os.path.exists(dllpath): - log.warning("No valid DLL specified to be injected in process " - "with pid %s / process name %s, injection aborted.", - self.pid, self.process_name) - return False - - if is32bit: - inject_exe = os.path.join("bin", "inject-x86.exe") - else: - inject_exe = os.path.join("bin", "inject-x64.exe") - - args = [ - inject_exe, "--dll", dllpath, - "--config", self.drop_config(track=track, mode=mode), - ] - - if self.pid: - args += ["--pid", "%s" % self.pid] - elif self.process_name: - args += ["--process-name", self.process_name] - - if apc: - args += ["--apc", "--tid", "%s" % self.tid] - else: - args += ["--crt"] - - try: - subprocess.check_call(args) - except Exception: - log.error("Failed to inject %s-bit process with pid %s and " - "process name %s", 32 if is32bit else 64, self.pid, - self.process_name) - return False - - log.info("Successfully injected process with pid %s", self.pid) - return True - - def drop_config(self, track=True, mode=None): - """Helper function to drop the configuration for a new process.""" - fd, config_path = tempfile.mkstemp() - - # The first time we come up with a random startup-time. - if Process.first_process: - # This adds 1 up to 30 times of 20 minutes to the startup - # time of the process, therefore bypassing anti-vm checks - # which check whether the VM has only been up for <10 minutes. - Process.startup_time = random.randint(1, 30) * 20 * 60 * 1000 - - lines = { - "host-ip": self.config.ip, - "host-port": self.config.port, - "pipe": self.config.pipe, - "logpipe": self.config.logpipe, - "results": PATHS["root"], - "analyzer": os.getcwd(), - "first-process": "1" if Process.first_process else "0", - "startup-time": Process.startup_time, - "shutdown-mutex": SHUTDOWN_MUTEX, - "force-sleepskip": self.config.options.get("force-sleepskip", "0"), - "hashes-path": os.path.join(os.getcwd(), "hashes.bin"), - "track": "1" if track else "0", - "mode": mode or "", - "disguise": self.config.options.get("disguise", "0"), - } - - for key, value in lines.items(): - os.write(fd, "%s=%s\n" % (key, value)) - - os.close(fd) - Process.first_process = False - return config_path - - def dump_memory(self): - """Dump process memory. - @return: operation status. - """ - if not self.pid: - log.warning("No valid pid specified, memory dump aborted") - return False - - if not self.is_alive(): - log.warning("The process with pid %d is not alive, memory " - "dump aborted", self.pid) - return False - - self.get_system_info() - - page_size = self.system_info.dwPageSize - min_addr = self.system_info.lpMinimumApplicationAddress - max_addr = self.system_info.lpMaximumApplicationAddress - mem = min_addr - - root = os.path.join(PATHS["memory"], str(int(time.time()))) - - if not os.path.exists(root): - os.makedirs(root) - - # Now upload to host from the StringIO. - nf = NetlogFile(os.path.join("memory", "%s.dmp" % str(self.pid))) - - process_handle = self.open_process() - - while mem < max_addr: - mbi = MEMORY_BASIC_INFORMATION() - count = c_ulong(0) - - if KERNEL32.VirtualQueryEx(process_handle, - mem, - byref(mbi), - sizeof(mbi)) < sizeof(mbi): - mem += page_size - continue - - if mbi.State & MEM_COMMIT and \ - mbi.Type & (MEM_IMAGE | MEM_MAPPED | MEM_PRIVATE): - buf = create_string_buffer(mbi.RegionSize) - if KERNEL32.ReadProcessMemory(process_handle, - mem, - buf, - mbi.RegionSize, - byref(count)): - nf.sock.sendall(buf.raw) - mem += mbi.RegionSize - else: - mem += page_size - - KERNEL32.CloseHandle(process_handle) - nf.close() - - log.info("Memory dump of process with pid %d completed", self.pid) - return True diff --git a/analyzer/windows/lib/api/screenshot.py b/analyzer/windows/lib/api/screenshot.py deleted file mode 100644 index 09f667d25d..0000000000 --- a/analyzer/windows/lib/api/screenshot.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import math - -try: - import ImageChops - import ImageGrab - import ImageDraw - HAVE_PIL = True -except: - try: - from PIL import ImageChops - from PIL import ImageGrab - from PIL import ImageDraw - HAVE_PIL = True - except: - HAVE_PIL = False - -class Screenshot: - """Get screenshots.""" - - def _draw_rectangle(self, img, xy): - """Draw a black rectangle. - @param img: PIL Image object - @param xy: Coordinates as refined in PIL rectangle() doc - @return: Image with black rectangle - """ - dr = ImageDraw.Draw(img) - dr.rectangle(xy, fill="black", outline="black") - return img - - def have_pil(self): - """Is Python Image Library installed? - @return: installed status. - """ - return HAVE_PIL - - def equal(self, img1, img2, skip_area=None): - """Compares two screenshots using Root-Mean-Square Difference (RMS). - @param img1: screenshot to compare. - @param img2: screenshot to compare. - @return: equal status. - """ - if not HAVE_PIL: - return None - - # Trick to avoid getting a lot of screen shots only because the time in the windows - # clock is changed. - # We draw a black rectangle on the coordinates where the clock is locates, and then - # run the comparison. - # NOTE: the coordinates are changing with VM screen resolution. - if skip_area: - # Copying objects to draw in another object. - img1 = img1.copy() - img2 = img2.copy() - # Draw a rectangle to cover windows clock. - for img in (img1, img2): - self._draw_rectangle(img, skip_area) - - # To get a measure of how similar two images are, we use - # root-mean-square (RMS). If the images are exactly identical, - # this value is zero. - diff = ImageChops.difference(img1, img2) - h = diff.histogram() - sq = (value * ((idx % 256)**2) for idx, value in enumerate(h)) - sum_of_squares = sum(sq) - rms = math.sqrt(sum_of_squares/float(img1.size[0] * img1.size[1])) - - # Might need to tweak the threshold. - return rms < 8 - - def take(self): - """Take a screenshot. - @return: screenshot or None. - """ - if not HAVE_PIL: - return None - - return ImageGrab.grab() diff --git a/analyzer/windows/lib/common/__init__.py b/analyzer/windows/lib/common/__init__.py deleted file mode 100644 index 13fb1d219c..0000000000 --- a/analyzer/windows/lib/common/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. diff --git a/analyzer/windows/lib/common/abstracts.py b/analyzer/windows/lib/common/abstracts.py deleted file mode 100644 index 0bb1ca1ab2..0000000000 --- a/analyzer/windows/lib/common/abstracts.py +++ /dev/null @@ -1,174 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import glob -import os - -from _winreg import CreateKey, SetValueEx, CloseKey, REG_DWORD, REG_SZ - -from lib.api.process import Process -from lib.common.exceptions import CuckooPackageError - -class Package(object): - """Base abstract analysis package.""" - PATHS = [] - REGKEYS = [] - - def __init__(self, options={}): - """@param options: options dict.""" - self.options = options - self.pids = [] - - # Fetch the current working directory, defaults to $TEMP. - if "curdir" in options: - self.curdir = os.path.expandvars(options["curdir"]) - else: - self.curdir = os.getenv("TEMP") - - def set_pids(self, pids): - """Update list of monitored PIDs in the package context. - @param pids: list of pids. - """ - self.pids = pids - - def start(self, target): - """Run analysis package. - @raise NotImplementedError: this method is abstract. - """ - raise NotImplementedError - - def check(self): - """Check.""" - return True - - def enum_paths(self): - """Enumerate available paths.""" - basepaths = { - "System32": [ - os.path.join(os.getenv("SystemRoot"), "System32"), - os.path.join(os.getenv("SystemRoot"), "SysWOW64"), - ], - "ProgramFiles": [ - os.getenv("ProgramFiles").replace(" (x86)", ""), - os.getenv("ProgramFiles(x86)"), - ], - "HomeDrive": [ - # os.path.join() doesn't work well if you give it just "C:" - # so manually append a backslash. - os.getenv("HomeDrive") + "\\", - ], - } - - for path in self.PATHS: - basedir = path[0] - for basepath in basepaths.get(basedir, [basedir]): - if not basepath or not os.path.isdir(basepath): - continue - - yield os.path.join(basepath, *path[1:]) - - def get_path(self, application): - """Search for the application in all available paths. - @param applicaiton: application executable name - @return: executable path - """ - for path in self.enum_paths(): - if os.path.isfile(path): - return path - - raise CuckooPackageError("Unable to find any %s executable." % - application) - - def get_path_glob(self, application): - """Search for the application in all available paths with glob support. - @param applicaiton: application executable name - @return: executable path - """ - for path in self.enum_paths(): - for path in glob.iglob(path): - if os.path.isfile(path): - return path - - raise CuckooPackageError("Unable to find any %s executable." % - application) - - def move_curdir(self, filepath): - """Move a file to the current working directory so it can be executed - from there. - @param filepath: the file to be moved - @return: the new filepath - """ - outpath = os.path.join(self.curdir, os.path.basename(filepath)) - os.rename(filepath, outpath) - return outpath - - def init_regkeys(self, regkeys): - """Initializes the registry to avoid annoying popups, configure - settings, etc. - @param regkeys: the root keys, subkeys, and key/value pairs. - """ - for rootkey, subkey, values in regkeys: - key_handle = CreateKey(rootkey, subkey) - - for key, value in values.items(): - if isinstance(value, str): - SetValueEx(key_handle, key, 0, REG_SZ, value) - elif isinstance(value, int): - SetValueEx(key_handle, key, 0, REG_DWORD, value) - elif isinstance(value, dict): - self.init_regkeys([ - [rootkey, "%s\\%s" % (subkey, key), value], - ]) - else: - raise CuckooPackageError("Invalid value type: %r" % value) - - CloseKey(key_handle) - - def execute(self, path, args, mode=None, maximize=False): - """Starts an executable for analysis. - @param path: executable path - @param args: executable arguments - @param mode: monitor mode - which functions to instrument - @param maximize: whether the GUI should start maximized - @return: process pid - """ - dll = self.options.get("dll") - free = self.options.get("free") - source = self.options.get("from") - - # Setup pre-defined registry keys. - self.init_regkeys(self.REGKEYS) - - p = Process() - if not p.execute(path=path, args=args, dll=dll, free=free, - curdir=self.curdir, source=source, mode=mode, - maximize=maximize): - raise CuckooPackageError("Unable to execute the initial process, " - "analysis aborted.") - - return p.pid - - def package_files(self): - """A list of files to upload to host. - The list should be a list of tuples (, ). - (package_files is a folder that will be created in analysis folder). - """ - return None - - def finish(self): - """Finish run. - If specified to do so, this method dumps the memory of - all running processes. - """ - if self.options.get("procmemdump"): - for pid in self.pids: - p = Process(pid=pid) - p.dump_memory() - - return True - -class Auxiliary(object): - def __init__(self, options={}, analyzer=None): - self.options = options - self.analyzer = analyzer diff --git a/analyzer/windows/lib/common/constants.py b/analyzer/windows/lib/common/constants.py deleted file mode 100644 index 40e396e2a7..0000000000 --- a/analyzer/windows/lib/common/constants.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import os -from lib.common.rand import random_string - - -ROOT = os.path.join(os.getenv("SystemDrive"), "\\", random_string(6, 10)) - -PATHS = {"root" : ROOT, - "logs" : os.path.join(ROOT, "logs"), - "files" : os.path.join(ROOT, "files"), - "shots" : os.path.join(ROOT, "shots"), - "memory" : os.path.join(ROOT, "memory"), - "drop" : os.path.join(ROOT, "drop")} - -SHUTDOWN_MUTEX = "Global\\" + random_string(6, 10) diff --git a/analyzer/windows/lib/common/defines.py b/analyzer/windows/lib/common/defines.py deleted file mode 100644 index 8f366d13df..0000000000 --- a/analyzer/windows/lib/common/defines.py +++ /dev/null @@ -1,170 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -from ctypes import * - -NTDLL = windll.ntdll -KERNEL32 = windll.kernel32 -ADVAPI32 = windll.advapi32 -USER32 = windll.user32 - -BYTE = c_ubyte -WORD = c_ushort -DWORD = c_ulong -LONG = c_ulong -LPBYTE = POINTER(c_ubyte) -LPTSTR = POINTER(c_char) -HANDLE = c_void_p -PVOID = c_void_p -LPVOID = c_void_p -UINT_PTR = c_ulong -SIZE_T = c_ulong -HMODULE = c_void_p -NULL = c_int(0) - -DEBUG_PROCESS = 0x00000001 -CREATE_NEW_CONSOLE = 0x00000010 -CREATE_SUSPENDED = 0x00000004 -DBG_CONTINUE = 0x00010002 -INFINITE = 0xFFFFFFFF -PROCESS_ALL_ACCESS = 0x001F0FFF -THREAD_ALL_ACCESS = 0x001f03ff -TOKEN_ALL_ACCESS = 0x000F01FF -SE_PRIVILEGE_ENABLED = 0x00000002 -STILL_ACTIVE = 0x00000103 - -PAGE_EXECUTE_READWRITE = 0x00000040 -PAGE_EXECUTE = 0x00000010 -PAGE_EXECUTE_READ = 0x00000020 -PAGE_READONLY = 0x00000002 -PAGE_READWRITE = 0x00000004 - -MEM_COMMIT = 0x00001000 -MEM_RESERVE = 0x00002000 -MEM_DECOMMIT = 0x00004000 -MEM_RELEASE = 0x00008000 -MEM_RESET = 0x00080000 - -MEM_IMAGE = 0x01000000 -MEM_MAPPED = 0x00040000 -MEM_PRIVATE = 0x00020000 - -PAGE_NOACCESS = 0x00000001 -PAGE_READONLY = 0x00000002 -PAGE_READWRITE = 0x00000004 -PAGE_WRITECOPY = 0x00000008 -PAGE_EXECUTE = 0x00000010 -PAGE_EXECUTE_READ = 0x00000020 -PAGE_EXECUTE_READWRITE = 0x00000040 -PAGE_EXECUTE_WRITECOPY = 0x00000080 -PAGE_GUARD = 0x00000100 -PAGE_NOCACHE = 0x00000200 -PAGE_WRITECOMBINE = 0x00000400 - -PIPE_ACCESS_INBOUND = 0x00000001 -PIPE_ACCESS_DUPLEX = 0x00000003 -PIPE_TYPE_MESSAGE = 0x00000004 -PIPE_READMODE_MESSAGE = 0x00000002 -PIPE_WAIT = 0x00000000 -PIPE_UNLIMITED_INSTANCES = 0x000000ff -PIPE_TYPE_BYTE = 0x00000000 -PIPE_READMODE_BYTE = 0x00000000 -FILE_FLAG_WRITE_THROUGH = 0x80000000 -INVALID_HANDLE_VALUE = 0xffffffff -ERROR_BROKEN_PIPE = 0x0000006d -ERROR_MORE_DATA = 0x000000EA -ERROR_PIPE_CONNECTED = 0x00000217 - -WAIT_TIMEOUT = 0x00000102 - -FILE_ATTRIBUTE_HIDDEN = 0x00000002 - -WM_GETTEXT = 0x0000000D -WM_GETTEXTLENGTH = 0x0000000E -BM_CLICK = 0x000000F5 - -class STARTUPINFO(Structure): - _fields_ = [ - ("cb", DWORD), - ("lpReserved", LPTSTR), - ("lpDesktop", LPTSTR), - ("lpTitle", LPTSTR), - ("dwX", DWORD), - ("dwY", DWORD), - ("dwXSize", DWORD), - ("dwYSize", DWORD), - ("dwXCountChars", DWORD), - ("dwYCountChars", DWORD), - ("dwFillAttribute",DWORD), - ("dwFlags", DWORD), - ("wShowWindow", WORD), - ("cbReserved2", WORD), - ("lpReserved2", LPBYTE), - ("hStdInput", HANDLE), - ("hStdOutput", HANDLE), - ("hStdError", HANDLE), - ] - -class PROCESS_INFORMATION(Structure): - _fields_ = [ - ("hProcess", HANDLE), - ("hThread", HANDLE), - ("dwProcessId", DWORD), - ("dwThreadId", DWORD), - ] - -class LUID(Structure): - _fields_ = [ - ("LowPart", DWORD), - ("HighPart", LONG), - ] - -class LUID_AND_ATTRIBUTES(Structure): - _fields_ = [ - ("Luid", LUID), - ("Attributes", DWORD), - ] - -class TOKEN_PRIVILEGES(Structure): - _fields_ = [ - ("PrivilegeCount", DWORD), - ("Privileges", LUID_AND_ATTRIBUTES), - ] - -class MEMORY_BASIC_INFORMATION(Structure): - _fields_ = [ - ("BaseAddress", PVOID), - ("AllocationBase", PVOID), - ("AllocationProtect", DWORD), - ("RegionSize", SIZE_T), - ("State", DWORD), - ("Protect", DWORD), - ("Type", DWORD), - ] - -class PROC_STRUCT(Structure): - _fields_ = [ - ("wProcessorArchitecture", WORD), - ("wReserved", WORD), - ] - -class SYSTEM_INFO_UNION(Union): - _fields_ = [ - ("dwOemId", DWORD), - ("sProcStruc", PROC_STRUCT), - ] - -class SYSTEM_INFO(Structure): - _fields_ = [ - ("uSysInfo", SYSTEM_INFO_UNION), - ("dwPageSize", DWORD), - ("lpMinimumApplicationAddress", LPVOID), - ("lpMaximumApplicationAddress", LPVOID), - ("dwActiveProcessorMask", DWORD), - ("dwNumberOfProcessors", DWORD), - ("dwProcessorType", DWORD), - ("dwAllocationGranularity", DWORD), - ("wProcessorLevel", WORD), - ("wProcessorRevision", WORD), - ] diff --git a/analyzer/windows/lib/common/errors.py b/analyzer/windows/lib/common/errors.py deleted file mode 100644 index c7b875f9f8..0000000000 --- a/analyzer/windows/lib/common/errors.py +++ /dev/null @@ -1,485 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -ERRORS = { - 0: {"description": "The operation completed successfully", - "name": "ERROR_SUCCESS"}, - 1: {"description": "Incorrect function", "name": "ERROR_INVALID_FUNCTION"}, - 2: {"description": "The system cannot find the file specified", - "name": "ERROR_FILE_NOT_FOUND"}, - 3: {"description": "The system cannot find the path specified", - "name": "ERROR_PATH_NOT_FOUND"}, - 4: {"description": "The system cannot open the file", - "name": "ERROR_TOO_MANY_OPEN_FILES"}, - 5: {"description": "Access is denied", "name": "ERROR_ACCESS_DENIED"}, - 6: {"description": "The handle is invalid", "name": "ERROR_INVALID_HANDLE"}, - 7: {"description": "The storage control blocks were destroyed", - "name": "ERROR_ARENA_TRASHED"}, - 8: {"description": "Not enough storage is available to process this command", - "name": "ERROR_NOT_ENOUGH_MEMORY"}, - 9: {"description": "The storage control block address is invalid", - "name": "ERROR_INVALID_BLOCK"}, - 10: {"description": "The environment is incorrect", - "name": "ERROR_BAD_ENVIRONMENT"}, - 11: {"description": "An attempt was made to load a program with an incorrect format", - "name": "ERROR_BAD_FORMAT"}, - 12: {"description": "The access code is invalid", - "name": "ERROR_INVALID_ACCESS"}, - 13: {"description": "The data is invalid", "name": "ERROR_INVALID_DATA"}, - 14: {"description": "Not enough storage is available to complete this operation", - "name": "ERROR_OUTOFMEMORY"}, - 15: {"description": "The system cannot find the drive specified", - "name": "ERROR_INVALID_DRIVE"}, - 16: {"description": "The directory cannot be removed", - "name": "ERROR_CURRENT_DIRECTORY"}, - 17: {"description": "The system cannot move the file to a different disk drive", - "name": "ERROR_NOT_SAME_DEVICE"}, - 18: {"description": "There are no more files", "name": "ERROR_NO_MORE_FILES"}, - 19: {"description": "The media is write protected", - "name": "ERROR_WRITE_PROTECT"}, - 20: {"description": "The system cannot find the device specified", - "name": "ERROR_BAD_UNIT"}, - 21: {"description": "The device is not ready", "name": "ERROR_NOT_READY"}, - 22: {"description": "The device does not recognize the command", - "name": "ERROR_BAD_COMMAND"}, - 23: {"description": "Data error (cyclic redundancy check)", - "name": "ERROR_CRC"}, - 24: {"description": "The program issued a command but the command length is incorrect", - "name": "ERROR_BAD_LENGTH"}, - 25: {"description": "The drive cannot locate a specific area or track on the disk", - "name": "ERROR_SEEK"}, - 26: {"description": "The specified disk or diskette cannot be accessed", - "name": "ERROR_NOT_DOS_DISK"}, - 27: {"description": "The drive cannot find the sector requested", - "name": "ERROR_SECTOR_NOT_FOUND"}, - 28: {"description": "The printer is out of paper", - "name": "ERROR_OUT_OF_PAPER"}, - 29: {"description": "The system cannot write to the specified device", - "name": "ERROR_WRITE_FAULT"}, - 30: {"description": "The system cannot read from the specified device", - "name": "ERROR_READ_FAULT"}, - 31: {"description": "A device attached to the system is not functioning", - "name": "ERROR_GEN_FAILURE"}, - 32: {"description": "The process cannot access the file because it is being used by another process", - "name": "ERROR_SHARING_VIOLATION"}, - 33: {"description": "The process cannot access the file because another process has locked a portion of the file", - "name": "ERROR_LOCK_VIOLATION"}, - 34: {"description": "The wrong diskette is in the drive. Insert %2 (Volume Serial Number: %3) into drive %1", - "name": "ERROR_WRONG_DISK"}, - 36: {"description": "Too many files opened for sharing", - "name": "ERROR_SHARING_BUFFER_EXCEEDED"}, - 38: {"description": "Reached the end of the file", - "name": "ERROR_HANDLE_EOF"}, - 39: {"description": "The disk is full", "name": "ERROR_HANDLE_DISK_FULL"}, - 50: {"description": "The request is not supported", - "name": "ERROR_NOT_SUPPORTED"}, - 51: {"description": "Windows cannot find the network path. Verify that the network path is correct and the destination computer is not busy or turned off. If Windows still cannot find the network path, contact your network administrator", - "name": "ERROR_REM_NOT_LIST"}, - 52: {"description": "You were not connected because a duplicate name exists on the network. If joining a domain, go to System in Control Panel to change the computer name and try again. If joining a workgroup, choose another workgroup name", - "name": "ERROR_DUP_NAME"}, - 53: {"description": "The network path was not found", - "name": "ERROR_BAD_NETPATH"}, - 54: {"description": "The network is busy", "name": "ERROR_NETWORK_BUSY"}, - 55: {"description": "The specified network resource or device is no longer available", - "name": "ERROR_DEV_NOT_EXIST"}, - 56: {"description": "The network BIOS command limit has been reached", - "name": "ERROR_TOO_MANY_CMDS"}, - 57: {"description": "A network adapter hardware error occurred", - "name": "ERROR_ADAP_HDW_ERR"}, - 58: {"description": "The specified server cannot perform the requested operation", - "name": "ERROR_BAD_NET_RESP"}, - 59: {"description": "An unexpected network error occurred", - "name": "ERROR_UNEXP_NET_ERR"}, - 60: {"description": "The remote adapter is not compatible", - "name": "ERROR_BAD_REM_ADAP"}, - 61: {"description": "The printer queue is full", "name": "ERROR_PRINTQ_FULL"}, - 62: {"description": "Space to store the file waiting to be printed is not available on the server", - "name": "ERROR_NO_SPOOL_SPACE"}, - 63: {"description": "Your file waiting to be printed was deleted", - "name": "ERROR_PRINT_CANCELLED"}, - 64: {"description": "The specified network name is no longer available", - "name": "ERROR_NETNAME_DELETED"}, - 65: {"description": "Network access is denied", - "name": "ERROR_NETWORK_ACCESS_DENIED"}, - 66: {"description": "The network resource type is not correct", - "name": "ERROR_BAD_DEV_TYPE"}, - 67: {"description": "The network name cannot be found", - "name": "ERROR_BAD_NET_NAME"}, - 68: {"description": "The name limit for the local computer network adapter card was exceeded", - "name": "ERROR_TOO_MANY_NAMES"}, - 69: {"description": "The network BIOS session limit was exceeded", - "name": "ERROR_TOO_MANY_SESS"}, - 70: {"description": "The remote server has been paused or is in the process of being started", - "name": "ERROR_SHARING_PAUSED"}, - 71: {"description": "No more connections can be made to this remote computer at this time because there are already as many connections as the computer can accept", - "name": "ERROR_REQ_NOT_ACCEP"}, - 72: {"description": "The specified printer or disk device has been paused", - "name": "ERROR_REDIR_PAUSED"}, - 80: {"description": "The file exists", "name": "ERROR_FILE_EXISTS"}, - 82: {"description": "The directory or file cannot be created", - "name": "ERROR_CANNOT_MAKE"}, - 83: {"description": "Fail on INT 24", "name": "ERROR_FAIL_I24"}, - 84: {"description": "Storage to process this request is not available", - "name": "ERROR_OUT_OF_STRUCTURES"}, - 85: {"description": "The local device name is already in use", - "name": "ERROR_ALREADY_ASSIGNED"}, - 86: {"description": "The specified network password is not correct", - "name": "ERROR_INVALID_PASSWORD"}, - 87: {"description": "The parameter is incorrect", - "name": "ERROR_INVALID_PARAMETER"}, - 88: {"description": "A write fault occurred on the network", - "name": "ERROR_NET_WRITE_FAULT"}, - 89: {"description": "The system cannot start another process at this time", - "name": "ERROR_NO_PROC_SLOTS"}, - 100: {"description": "Cannot create another system semaphore", - "name": "ERROR_TOO_MANY_SEMAPHORES"}, - 101: {"description": "The exclusive semaphore is owned by another process", - "name": "ERROR_EXCL_SEM_ALREADY_OWNED"}, - 102: {"description": "The semaphore is set and cannot be closed", - "name": "ERROR_SEM_IS_SET"}, - 103: {"description": "The semaphore cannot be set again", - "name": "ERROR_TOO_MANY_SEM_REQUESTS"}, - 104: {"description": "Cannot request exclusive semaphores at interrupt time", - "name": "ERROR_INVALID_AT_INTERRUPT_TIME"}, - 105: {"description": "The previous ownership of this semaphore has ended", - "name": "ERROR_SEM_OWNER_DIED"}, - 106: {"description": "Insert the diskette for drive %1", - "name": "ERROR_SEM_USER_LIMIT"}, - 107: {"description": "The program stopped because an alternate diskette was not inserted", - "name": "ERROR_DISK_CHANGE"}, - 108: {"description": "The disk is in use or locked by another process", - "name": "ERROR_DRIVE_LOCKED"}, - 109: {"description": "The pipe has been ended", "name": "ERROR_BROKEN_PIPE"}, - 110: {"description": "The system cannot open the device or file specified", - "name": "ERROR_OPEN_FAILED"}, - 111: {"description": "The file name is too long", - "name": "ERROR_BUFFER_OVERFLOW"}, - 112: {"description": "There is not enough space on the disk", - "name": "ERROR_DISK_FULL"}, - 113: {"description": "No more internal file identifiers available", - "name": "ERROR_NO_MORE_SEARCH_HANDLES"}, - 114: {"description": "The target internal file identifier is incorrect", - "name": "ERROR_INVALID_TARGET_HANDLE"}, - 117: {"description": "The IOCTL call made by the application program is not correct", - "name": "ERROR_INVALID_CATEGORY"}, - 118: {"description": "The verify-on-write switch parameter value is not correct", - "name": "ERROR_INVALID_VERIFY_SWITCH"}, - 119: {"description": "The system does not support the command requested", - "name": "ERROR_BAD_DRIVER_LEVEL"}, - 120: {"description": "This function is not supported on this system", - "name": "ERROR_CALL_NOT_IMPLEMENTED"}, - 121: {"description": "The semaphore timeout period has expired", - "name": "ERROR_SEM_TIMEOUT"}, - 122: {"description": "The data area passed to a system call is too small", - "name": "ERROR_INSUFFICIENT_BUFFER"}, - 123: {"description": "The filename, directory name, or volume label syntax is incorrect", - "name": "ERROR_INVALID_NAME"}, - 124: {"description": "The system call level is not correct", - "name": "ERROR_INVALID_LEVEL"}, - 125: {"description": "The disk has no volume label", - "name": "ERROR_NO_VOLUME_LABEL"}, - 126: {"description": "The specified module could not be found", - "name": "ERROR_MOD_NOT_FOUND"}, - 127: {"description": "The specified procedure could not be found", - "name": "ERROR_PROC_NOT_FOUND"}, - 128: {"description": "There are no child processes to wait for", - "name": "ERROR_WAIT_NO_CHILDREN"}, - 129: {"description": "The %1 application cannot be run in Win32 mode", - "name": "ERROR_CHILD_NOT_COMPLETE"}, - 130: {"description": "Attempt to use a file handle to an open disk partition for an operation other than raw disk I/O", - "name": "ERROR_DIRECT_ACCESS_HANDLE"}, - 131: {"description": "An attempt was made to move the file pointer before the beginning of the file", - "name": "ERROR_NEGATIVE_SEEK"}, - 132: {"description": "The file pointer cannot be set on the specified device or file", - "name": "ERROR_SEEK_ON_DEVICE"}, - 133: {"description": "A JOIN or SUBST command cannot be used for a drive that contains previously joined drives", - "name": "ERROR_IS_JOIN_TARGET"}, - 134: {"description": "An attempt was made to use a JOIN or SUBST command on a drive that has already been joined", - "name": "ERROR_IS_JOINED"}, - 135: {"description": "An attempt was made to use a JOIN or SUBST command on a drive that has already been substituted", - "name": "ERROR_IS_SUBSTED"}, - 136: {"description": "The system tried to delete the JOIN of a drive that is not joined", - "name": "ERROR_NOT_JOINED"}, - 137: {"description": "The system tried to delete the substitution of a drive that is not substituted", - "name": "ERROR_NOT_SUBSTED"}, - 138: {"description": "The system tried to join a drive to a directory on a joined drive", - "name": "ERROR_JOIN_TO_JOIN"}, - 139: {"description": "The system tried to substitute a drive to a directory on a substituted drive", - "name": "ERROR_SUBST_TO_SUBST"}, - 140: {"description": "The system tried to join a drive to a directory on a substituted drive", - "name": "ERROR_JOIN_TO_SUBST"}, - 141: {"description": "The system tried to SUBST a drive to a directory on a joined drive", - "name": "ERROR_SUBST_TO_JOIN"}, - 142: {"description": "The system cannot perform a JOIN or SUBST at this time", - "name": "ERROR_BUSY_DRIVE"}, - 143: {"description": "The system cannot join or substitute a drive to or for a directory on the same drive", - "name": "ERROR_SAME_DRIVE"}, - 144: {"description": "The directory is not a subdirectory of the root directory", - "name": "ERROR_DIR_NOT_ROOT"}, - 145: {"description": "The directory is not empty", - "name": "ERROR_DIR_NOT_EMPTY"}, - 146: {"description": "The path specified is being used in a substitute", - "name": "ERROR_IS_SUBST_PATH"}, - 147: {"description": "Not enough resources are available to process this command", - "name": "ERROR_IS_JOIN_PATH"}, - 148: {"description": "The path specified cannot be used at this time", - "name": "ERROR_PATH_BUSY"}, - 149: {"description": "An attempt was made to join or substitute a drive for which a directory on the drive is the target of a previous substitute", - "name": "ERROR_IS_SUBST_TARGET"}, - 150: {"description": "System trace information was not specified in your CONFIG.SYS file, or tracing is disallowed", - "name": "ERROR_SYSTEM_TRACE"}, - 151: {"description": "The number of specified semaphore events for DosMuxSemWait is not correct", - "name": "ERROR_INVALID_EVENT_COUNT"}, - 152: {"description": "DosMuxSemWait did not execute; too many semaphores are already set", - "name": "ERROR_TOO_MANY_MUXWAITERS"}, - 153: {"description": "The DosMuxSemWait list is not correct", - "name": "ERROR_INVALID_LIST_FORMAT"}, - 154: {"description": "The volume label you entered exceeds the label character limit of the target file system", - "name": "ERROR_LABEL_TOO_LONG"}, - 155: {"description": "Cannot create another thread", - "name": "ERROR_TOO_MANY_TCBS"}, - 156: {"description": "The recipient process has refused the signal", - "name": "ERROR_SIGNAL_REFUSED"}, - 157: {"description": "The segment is already discarded and cannot be locked", - "name": "ERROR_DISCARDED"}, - 158: {"description": "The segment is already unlocked", - "name": "ERROR_NOT_LOCKED"}, - 159: {"description": "The address for the thread ID is not correct", - "name": "ERROR_BAD_THREADID_ADDR"}, - 160: {"description": "One or more arguments are not correct", - "name": "ERROR_BAD_ARGUMENTS"}, - 161: {"description": "The specified path is invalid", - "name": "ERROR_BAD_PATHNAME"}, - 162: {"description": "A signal is already pending", - "name": "ERROR_SIGNAL_PENDING"}, - 164: {"description": "No more threads can be created in the system", - "name": "ERROR_MAX_THRDS_REACHED"}, - 167: {"description": "Unable to lock a region of a file", - "name": "ERROR_LOCK_FAILED"}, - 170: {"description": "The requested resource is in use", - "name": "ERROR_BUSY"}, - 171: {"description": "Device's command support detection is in progress", - "name": "ERROR_DEVICE_SUPPORT_IN_PROGRESS"}, - 173: {"description": "A lock request was not outstanding for the supplied cancel region", - "name": "ERROR_CANCEL_VIOLATION"}, - 174: {"description": "The file system does not support atomic changes to the lock type", - "name": "ERROR_ATOMIC_LOCKS_NOT_SUPPORTED"}, - 180: {"description": "The system detected a segment number that was not correct", - "name": "ERROR_INVALID_SEGMENT_NUMBER"}, - 182: {"description": "The operating system cannot run %1", - "name": "ERROR_INVALID_ORDINAL"}, - 183: {"description": "Cannot create a file when that file already exists", - "name": "ERROR_ALREADY_EXISTS"}, - 186: {"description": "The flag passed is not correct", - "name": "ERROR_INVALID_FLAG_NUMBER"}, - 187: {"description": "The specified system semaphore name was not found", - "name": "ERROR_SEM_NOT_FOUND"}, - 188: {"description": "The operating system cannot run %1", - "name": "ERROR_INVALID_STARTING_CODESEG"}, - 189: {"description": "The operating system cannot run %1", - "name": "ERROR_INVALID_STACKSEG"}, - 190: {"description": "The operating system cannot run %1", - "name": "ERROR_INVALID_MODULETYPE"}, - 191: {"description": "Cannot run %1 in Win32 mode", - "name": "ERROR_INVALID_EXE_SIGNATURE"}, - 192: {"description": "The operating system cannot run %1", - "name": "ERROR_EXE_MARKED_INVALID"}, - 193: {"description": "%1 is not a valid Win32 application", - "name": "ERROR_BAD_EXE_FORMAT"}, - 194: {"description": "The operating system cannot run %1", - "name": "ERROR_ITERATED_DATA_EXCEEDS_64k"}, - 195: {"description": "The operating system cannot run %1", - "name": "ERROR_INVALID_MINALLOCSIZE"}, - 196: {"description": "The operating system cannot run this application program", - "name": "ERROR_DYNLINK_FROM_INVALID_RING"}, - 197: {"description": "The operating system is not presently configured to run this application", - "name": "ERROR_IOPL_NOT_ENABLED"}, - 198: {"description": "The operating system cannot run %1", - "name": "ERROR_INVALID_SEGDPL"}, - 199: {"description": "The operating system cannot run this application program", - "name": "ERROR_AUTODATASEG_EXCEEDS_64k"}, - 200: {"description": "The code segment cannot be greater than or equal to 64K", - "name": "ERROR_RING2SEG_MUST_BE_MOVABLE"}, - 201: {"description": "The operating system cannot run %1", - "name": "ERROR_RELOC_CHAIN_XEEDS_SEGLIM"}, - 202: {"description": "The operating system cannot run %1", - "name": "ERROR_INFLOOP_IN_RELOC_CHAIN"}, - 203: {"description": "The system could not find the environment option that was entered", - "name": "ERROR_ENVVAR_NOT_FOUND"}, - 205: {"description": "No process in the command subtree has a signal handler", - "name": "ERROR_NO_SIGNAL_SENT"}, - 206: {"description": "The filename or extension is too long", - "name": "ERROR_FILENAME_EXCED_RANGE"}, - 207: {"description": "The ring 2 stack is in use", - "name": "ERROR_RING2_STACK_IN_USE"}, - 208: {"description": "The global filename characters, * or ?, are entered incorrectly or too many global filename characters are specified", - "name": "ERROR_META_EXPANSION_TOO_LONG"}, - 209: {"description": "The signal being posted is not correct", - "name": "ERROR_INVALID_SIGNAL_NUMBER"}, - 210: {"description": "The signal handler cannot be set", - "name": "ERROR_THREAD_1_INACTIVE"}, - 212: {"description": "The segment is locked and cannot be reallocated", - "name": "ERROR_LOCKED"}, - 214: {"description": "Too many dynamic-link modules are attached to this program or dynamic-link module", - "name": "ERROR_TOO_MANY_MODULES"}, - 215: {"description": "Cannot nest calls to LoadModule", - "name": "ERROR_NESTING_NOT_ALLOWED"}, - 216: {"description": "This version of %1 is not compatible with the version of Windows you're running. Check your computer's system information to see whether you need a x86 (32-bit) or x64 (64-bit) version of the program, and then contact the software publisher", - "name": "ERROR_EXE_MACHINE_TYPE_MISMATCH"}, - 217: {"description": "The image file %1 is signed, unable to modify", - "name": "ERROR_EXE_CANNOT_MODIFY_SIGNED_BINARY"}, - 218: {"description": "The image file %1 is strong signed, unable to modify", - "name": "ERROR_EXE_CANNOT_MODIFY_STRONG_SIGNED_BINARY"}, - 220: {"description": "This file is checked out or locked for editing by another user", - "name": "ERROR_FILE_CHECKED_OUT"}, - 221: {"description": "The file must be checked out before saving changes", - "name": "ERROR_CHECKOUT_REQUIRED"}, - 222: {"description": "The file type being saved or retrieved has been blocked", - "name": "ERROR_BAD_FILE_TYPE"}, - 223: {"description": "The file size exceeds the limit allowed and cannot be saved", - "name": "ERROR_FILE_TOO_LARGE"}, - 224: {"description": "Access Denied. Before opening files in this location, you must first add the web site to your trusted sites list, browse to the web site, and select the option to login automatically", - "name": "ERROR_FORMS_AUTH_REQUIRED"}, - 225: {"description": "Operation did not complete successfully because the file contains a virus or potentially unwanted software", - "name": "ERROR_VIRUS_INFECTED"}, - 226: {"description": "This file contains a virus or potentially unwanted software and cannot be opened. Due to the nature of this virus or potentially unwanted software, the file has been removed from this location", - "name": "ERROR_VIRUS_DELETED"}, - 229: {"description": "The pipe is local", "name": "ERROR_PIPE_LOCAL"}, - 230: {"description": "The pipe state is invalid", "name": "ERROR_BAD_PIPE"}, - 231: {"description": "All pipe instances are busy", - "name": "ERROR_PIPE_BUSY"}, - 232: {"description": "The pipe is being closed", "name": "ERROR_NO_DATA"}, - 233: {"description": "No process is on the other end of the pipe", - "name": "ERROR_PIPE_NOT_CONNECTED"}, - 234: {"description": "More data is available", "name": "ERROR_MORE_DATA"}, - 240: {"description": "The session was canceled", - "name": "ERROR_VC_DISCONNECTED"}, - 254: {"description": "The specified extended attribute name was invalid", - "name": "ERROR_INVALID_EA_NAME"}, - 255: {"description": "The extended attributes are inconsistent", - "name": "ERROR_EA_LIST_INCONSISTENT"}, - 258: {"description": "The wait operation timed out", "name": "WAIT_TIMEOUT"}, - 259: {"description": "No more data is available", - "name": "ERROR_NO_MORE_ITEMS"}, - 266: {"description": "The copy functions cannot be used", - "name": "ERROR_CANNOT_COPY"}, - 267: {"description": "The directory name is invalid", - "name": "ERROR_DIRECTORY"}, - 275: {"description": "The extended attributes did not fit in the buffer", - "name": "ERROR_EAS_DIDNT_FIT"}, - 276: {"description": "The extended attribute file on the mounted file system is corrupt", - "name": "ERROR_EA_FILE_CORRUPT"}, - 277: {"description": "The extended attribute table file is full", - "name": "ERROR_EA_TABLE_FULL"}, - 278: {"description": "The specified extended attribute handle is invalid", - "name": "ERROR_INVALID_EA_HANDLE"}, - 282: {"description": "The mounted file system does not support extended attributes", - "name": "ERROR_EAS_NOT_SUPPORTED"}, - 288: {"description": "Attempt to release mutex not owned by caller", - "name": "ERROR_NOT_OWNER"}, - 298: {"description": "Too many posts were made to a semaphore", - "name": "ERROR_TOO_MANY_POSTS"}, - 299: {"description": "Only part of a ReadProcessMemory or WriteProcessMemory request was completed", - "name": "ERROR_PARTIAL_COPY"}, - 300: {"description": "The oplock request is denied", - "name": "ERROR_OPLOCK_NOT_GRANTED"}, - 301: {"description": "An invalid oplock acknowledgment was received by the system", - "name": "ERROR_INVALID_OPLOCK_PROTOCOL"}, - 302: {"description": "The volume is too fragmented to complete this operation", - "name": "ERROR_DISK_TOO_FRAGMENTED"}, - 303: {"description": "The file cannot be opened because it is in the process of being deleted", - "name": "ERROR_DELETE_PENDING"}, - 304: {"description": "Short name settings may not be changed on this volume due to the global registry setting", - "name": "ERROR_INCOMPATIBLE_WITH_GLOBAL_SHORT_NAME_REGISTRY_SETTING"}, - 305: {"description": "Short names are not enabled on this volume", - "name": "ERROR_SHORT_NAMES_NOT_ENABLED_ON_VOLUME"}, - 306: {"description": "The security stream for the given volume is in an inconsistent state. Please run CHKDSK on the volume", - "name": "ERROR_SECURITY_STREAM_IS_INCONSISTENT"}, - 307: {"description": "A requested file lock operation cannot be processed due to an invalid byte range", - "name": "ERROR_INVALID_LOCK_RANGE"}, - 308: {"description": "The subsystem needed to support the image type is not present", - "name": "ERROR_IMAGE_SUBSYSTEM_NOT_PRESENT"}, - 309: {"description": "The specified file already has a notification GUID associated with it", - "name": "ERROR_NOTIFICATION_GUID_ALREADY_DEFINED"}, - 310: {"description": "An invalid exception handler routine has been detected", - "name": "ERROR_INVALID_EXCEPTION_HANDLER"}, - 311: {"description": "Duplicate privileges were specified for the token", - "name": "ERROR_DUPLICATE_PRIVILEGES"}, - 312: {"description": "No ranges for the specified operation were able to be processed", - "name": "ERROR_NO_RANGES_PROCESSED"}, - 313: {"description": "Operation is not allowed on a file system internal file", - "name": "ERROR_NOT_ALLOWED_ON_SYSTEM_FILE"}, - 314: {"description": "The physical resources of this disk have been exhausted", - "name": "ERROR_DISK_RESOURCES_EXHAUSTED"}, - 315: {"description": "The token representing the data is invalid", - "name": "ERROR_INVALID_TOKEN"}, - 316: {"description": "The device does not support the command feature", - "name": "ERROR_DEVICE_FEATURE_NOT_SUPPORTED"}, - 317: {"description": "The system cannot find message text for message number 0x%1 in the message file for %2", - "name": "ERROR_MR_MID_NOT_FOUND"}, - 318: {"description": "The scope specified was not found", - "name": "ERROR_SCOPE_NOT_FOUND"}, - 319: {"description": "The Central Access Policy specified is not defined on the target machine", - "name": "ERROR_UNDEFINED_SCOPE"}, - 320: {"description": "The Central Access Policy obtained from Active Directory is invalid", - "name": "ERROR_INVALID_CAP"}, - 321: {"description": "The device is unreachable", - "name": "ERROR_DEVICE_UNREACHABLE"}, - 322: {"description": "The target device has insufficient resources to complete the operation", - "name": "ERROR_DEVICE_NO_RESOURCES"}, - 323: {"description": "A data integrity checksum error occurred. Data in the file stream is corrupt", - "name": "ERROR_DATA_CHECKSUM_ERROR"}, - 324: {"description": "An attempt was made to modify both a KERNEL and normal Extended Attribute (EA) in the same operation", - "name": "ERROR_INTERMIXED_KERNEL_EA_OPERATION"}, - 326: {"description": "Device does not support file-level TRIM", - "name": "ERROR_FILE_LEVEL_TRIM_NOT_SUPPORTED"}, - 327: {"description": "The command specified a data offset that does not align to the device's granularity/alignment", - "name": "ERROR_OFFSET_ALIGNMENT_VIOLATION"}, - 328: {"description": "The command specified an invalid field in its parameter list", - "name": "ERROR_INVALID_FIELD_IN_PARAMETER_LIST"}, - 329: {"description": "An operation is currently in progress with the device", - "name": "ERROR_OPERATION_IN_PROGRESS"}, - 330: {"description": "An attempt was made to send down the command via an invalid path to the target device", - "name": "ERROR_BAD_DEVICE_PATH"}, - 331: {"description": "The command specified a number of descriptors that exceeded the maximum supported by the device", - "name": "ERROR_TOO_MANY_DESCRIPTORS"}, - 332: {"description": "Scrub is disabled on the specified file", - "name": "ERROR_SCRUB_DATA_DISABLED"}, - 333: {"description": "The storage device does not provide redundancy", - "name": "ERROR_NOT_REDUNDANT_STORAGE"}, - 334: {"description": "An operation is not supported on a resident file", - "name": "ERROR_RESIDENT_FILE_NOT_SUPPORTED"}, - 335: {"description": "An operation is not supported on a compressed file", - "name": "ERROR_COMPRESSED_FILE_NOT_SUPPORTED"}, - 336: {"description": "An operation is not supported on a directory", - "name": "ERROR_DIRECTORY_NOT_SUPPORTED"}, - 350: {"description": "No action was taken as a system reboot is required", - "name": "ERROR_FAIL_NOACTION_REBOOT"}, - 351: {"description": "The shutdown operation failed", - "name": "ERROR_FAIL_SHUTDOWN"}, - 352: {"description": "The restart operation failed", - "name": "ERROR_FAIL_RESTART"}, - 353: {"description": "The maximum number of sessions has been reached", - "name": "ERROR_MAX_SESSIONS_REACHED"}, - 400: {"description": "The thread is already in background processing mode", - "name": "ERROR_THREAD_MODE_ALREADY_BACKGROUND"}, - 401: {"description": "The thread is not in background processing mode", - "name": "ERROR_THREAD_MODE_NOT_BACKGROUND"}, - 402: {"description": "The process is already in background processing mode", - "name": "ERROR_PROCESS_MODE_ALREADY_BACKGROUND"}, - 403: {"description": "The process is not in background processing mode", - "name": "ERROR_PROCESS_MODE_NOT_BACKGROUND"}, - 487: {"description": "Attempt to access invalid address", - "name": "ERROR_INVALID_ADDRESS"} -} - -def get_error_string(error_code): - """Error code lookup. - @param error_code: error code. - @return: error description if found. - """ - if error_code in ERRORS: - return "%s (%s)" % (ERRORS[error_code]["description"], ERRORS[error_code]["name"]) - else: - return str(error_code) diff --git a/analyzer/windows/lib/common/exceptions.py b/analyzer/windows/lib/common/exceptions.py deleted file mode 100644 index d37f525b40..0000000000 --- a/analyzer/windows/lib/common/exceptions.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -class CuckooError(Exception): - pass - -class CuckooPackageError(Exception): - pass \ No newline at end of file diff --git a/analyzer/windows/lib/common/hashing.py b/analyzer/windows/lib/common/hashing.py deleted file mode 100644 index 04b32e7e0e..0000000000 --- a/analyzer/windows/lib/common/hashing.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -BUFSIZE = 1024*1024 - - -def hash_file(method, path): - """Calculates an hash on a file by path. - @param method: callable hashing method - @param path: file path - @return: computed hash string - """ - f = open(path, "rb") - h = method() - while True: - buf = f.read(BUFSIZE) - if not buf: - break - h.update(buf) - return h.hexdigest() diff --git a/analyzer/windows/lib/common/registry.py b/analyzer/windows/lib/common/registry.py deleted file mode 100644 index 5a28e147a2..0000000000 --- a/analyzer/windows/lib/common/registry.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import logging - -from ctypes import windll, POINTER, byref, Structure, pointer -from ctypes import c_ushort, c_wchar_p, c_void_p, create_string_buffer -from ctypes.wintypes import HANDLE, DWORD, LPCWSTR, ULONG, LONG -from _winreg import KEY_ALL_ACCESS, KEY_QUERY_VALUE, KEY_SET_VALUE -from _winreg import REG_SZ, REG_MULTI_SZ - -log = logging.getLogger(__name__) - -class UNICODE_STRING(Structure): - _fields_ = [ - ("Length", c_ushort), - ("MaximumLength", c_ushort), - ("Buffer", c_wchar_p), - ] - -RegOpenKeyExW = windll.advapi32.RegOpenKeyExW -RegOpenKeyExW.argtypes = HANDLE, LPCWSTR, DWORD, ULONG, POINTER(HANDLE) -RegOpenKeyExW.restype = LONG - -RegQueryValueExW = windll.advapi32.RegQueryValueExW -RegQueryValueExW.argtypes = \ - HANDLE, LPCWSTR, POINTER(DWORD), POINTER(DWORD), c_void_p, POINTER(DWORD) -RegQueryValueExW.restype = LONG - -RegSetValueExW = windll.advapi32.RegSetValueExW -RegSetValueExW.argtypes = HANDLE, LPCWSTR, DWORD, DWORD, c_void_p, DWORD -RegSetValueExW.restype = LONG - -NtRenameKey = windll.ntdll.NtRenameKey -NtRenameKey.argtypes = HANDLE, POINTER(UNICODE_STRING) - -RegCloseKey = windll.advapi32.RegCloseKey -RegCloseKey.argtypes = HANDLE, - -def rename_regkey(skey, ssubkey, dsubkey): - """Rename an entire tree of values in the registry. - Function by Thorsten Sick.""" - res_handle = HANDLE() - options = DWORD(0) - res = RegOpenKeyExW(skey, ssubkey, options, - KEY_ALL_ACCESS, byref(res_handle)) - if not res: - bsize = c_ushort(len(dsubkey) * 2) - us = UNICODE_STRING() - us.Buffer = c_wchar_p(dsubkey) - us.Length = bsize - us.MaximumLength = bsize - - res = NtRenameKey(res_handle, pointer(us)) - if res: - log.warning("Error renaming %s\\%s to %s (0x%x)", - skey, ssubkey, dsubkey, res % 2**32) - - if res_handle: - RegCloseKey(res_handle) - -def regkey_exists(rootkey, subkey): - res_handle = HANDLE() - res = RegOpenKeyExW(rootkey, subkey, 0, KEY_QUERY_VALUE, byref(res_handle)) - RegCloseKey(res_handle) - return not res - -def set_regkey(rootkey, subkey, name, type_, value): - if type_ == REG_SZ: - value = unicode(value) - if type_ == REG_MULTI_SZ: - value = u"\u0000".join(value) + u"\u0000\u0000" - - res_handle = HANDLE() - res = RegOpenKeyExW(rootkey, subkey, 0, KEY_SET_VALUE, byref(res_handle)) - if not res: - RegSetValueExW(res_handle, name, 0, type_, value, len(value)) - RegCloseKey(res_handle) - -def query_value(rootkey, subkey, name): - res_handle = HANDLE() - type_ = DWORD() - value = create_string_buffer(1024 * 1024) - length = DWORD(1024 * 1024) - - res = RegOpenKeyExW(rootkey, subkey, 0, KEY_QUERY_VALUE, byref(res_handle)) - if not res: - res = RegQueryValueExW(res_handle, name, None, byref(type_), value, byref(length)) - RegCloseKey(res_handle) - - if not res: - if type_.value == REG_SZ: - return value.raw[:length.value].decode("utf16").rstrip("\x00") - if type_.value == REG_MULTI_SZ: - value = value.raw[:length.value].decode("utf16") - return value.rstrip(u"\u0000").split(u"\u0000") - return value.raw[:length.value] diff --git a/analyzer/windows/lib/common/results.py b/analyzer/windows/lib/common/results.py deleted file mode 100644 index 9ba2a4c587..0000000000 --- a/analyzer/windows/lib/common/results.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import logging -import socket -import time - -from lib.core.config import Config - -log = logging.getLogger(__name__) - -BUFSIZE = 1024*1024 - -def upload_to_host(file_path, dump_path): - nc = infd = None - try: - nc = NetlogFile(dump_path) - - infd = open(file_path, "rb") - buf = infd.read(BUFSIZE) - while buf: - nc.send(buf, retry=False) - buf = infd.read(BUFSIZE) - except Exception as e: - log.error("Exception uploading file %s to host: %s", file_path, e) - finally: - if infd: - infd.close() - if nc: - nc.close() - -class NetlogConnection(object): - def __init__(self, proto=""): - config = Config(cfg="analysis.conf") - self.hostip, self.hostport = config.ip, config.port - self.sock = None - self.proto = proto - - def connect(self): - # Try to connect as quickly as possible. Just sort of force it to - # connect with a short timeout. - while not self.sock: - try: - s = socket.create_connection((self.hostip, self.hostport), 0.1) - s.sendall(self.proto) - except socket.error: - time.sleep(0.1) - continue - - self.sock = s - - def send(self, data, retry=True): - if not self.sock: - self.connect() - - try: - self.sock.sendall(data) - except socket.error as e: - if retry: - self.connect() - self.send(data, retry=False) - else: - raise - except Exception as e: - log.error("Unhandled exception in NetlogConnection: %s", str(e)) - # We really have nowhere to log this, if the netlog connection - # does not work, we can assume that any logging won't work either. - # So we just fail silently. - self.close() - - def close(self): - try: - self.sock.close() - except Exception: - pass - -class NetlogFile(NetlogConnection): - def __init__(self, filepath): - self.filepath = filepath - NetlogConnection.__init__(self, proto="FILE\n{0}\n".format(self.filepath)) - self.connect() - -class NetlogHandler(logging.Handler, NetlogConnection): - def __init__(self): - logging.Handler.__init__(self) - NetlogConnection.__init__(self, proto="LOG\n") - self.connect() - - def emit(self, record): - msg = self.format(record) - self.send("{0}\n".format(msg)) diff --git a/analyzer/windows/lib/core/__init__.py b/analyzer/windows/lib/core/__init__.py deleted file mode 100644 index 13fb1d219c..0000000000 --- a/analyzer/windows/lib/core/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. diff --git a/analyzer/windows/lib/core/config.py b/analyzer/windows/lib/core/config.py deleted file mode 100644 index df0b9c6d03..0000000000 --- a/analyzer/windows/lib/core/config.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import ConfigParser - -class Config: - def __init__(self, cfg): - """@param cfg: configuration file.""" - config = ConfigParser.ConfigParser(allow_no_value=True) - config.read(cfg) - - for section in config.sections(): - for name, raw_value in config.items(section): - if name == "file_name": - value = config.get(section, name).decode("utf8") - elif name == "options": - value = self.parse_options(config.get(section, name)) - else: - try: - value = config.getboolean(section, name) - except ValueError: - try: - value = config.getint(section, name) - except ValueError: - value = config.get(section, name) - setattr(self, name, value) - - # Just make sure the options field is available. - if not hasattr(self, "options"): - self.options = {} - - def parse_options(self, options): - """Get analysis options. - @return: options dict. - """ - # The analysis package can be provided with some options in the - # following format: - # option1=value1,option2=value2,option3=value3 - ret = {} - for field in options.split(","): - if "=" not in field: - continue - - key, value = field.split("=", 1) - ret[key.strip()] = value.strip() - return ret diff --git a/analyzer/windows/lib/core/packages.py b/analyzer/windows/lib/core/packages.py deleted file mode 100644 index 409ffa8cc8..0000000000 --- a/analyzer/windows/lib/core/packages.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -def has_com_exports(exports): - com_exports = [ - "DllInstall", - "DllCanUnloadNow", - "DllGetClassObject", - "DllRegisterServer", - "DllUnregisterServer", - ] - - for name in com_exports: - if name not in exports: - return False - return True - -def choose_package(file_type, file_name, exports): - """Choose analysis package due to file type and file extension. - @param file_type: file type. - @param file_name: file name. - @return: package name or None. - """ - if not file_type: - return None - - file_name = file_name.lower() - - if "DLL" in file_type: - if file_name.endswith(".cpl"): - return "cpl" - elif has_com_exports(exports): - return "com" - else: - return "dll" - elif "PE32" in file_type or "MS-DOS" in file_type: - return "exe" - elif "PDF" in file_type or file_name.endswith(".pdf"): - return "pdf" - elif "Rich Text Format" in file_type or \ - "Microsoft Word" in file_type or \ - "Microsoft Office Word" in file_type or \ - file_name.endswith((".doc", ".docx", ".rtf")): - return "doc" - elif "Microsoft Office Excel" in file_type or \ - "Microsoft Excel" in file_type or \ - file_name.endswith((".xls", ".xlsx")): - return "xls" - elif "Microsoft PowerPoint" in file_type or \ - file_name.endswith((".ppt", ".pptx", ".pps", ".ppsx", ".pptm", ".potm", ".potx", ".ppsm")): - return "ppt" - elif "HTML" in file_type or file_name.endswith((".htm", ".html")): - return "ie" - elif file_name.endswith(".jar"): - return "jar" - elif "Zip" in file_type: - return "zip" - elif file_name.endswith((".py", ".pyc")) or "Python script" in file_type: - return "python" - elif file_name.endswith(".vbs"): - return "vbs" - elif file_name.endswith(".js"): - return "js" - elif file_name.endswith(".msi"): - return "msi" - elif file_name.endswith(".ps1"): - return "ps1" - else: - return "generic" diff --git a/analyzer/windows/lib/core/pipe.py b/analyzer/windows/lib/core/pipe.py deleted file mode 100644 index fec47fe33e..0000000000 --- a/analyzer/windows/lib/core/pipe.py +++ /dev/null @@ -1,168 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import logging -import socket -import threading - -from ctypes import create_string_buffer, c_uint, byref, sizeof - -from lib.common.defines import KERNEL32, PIPE_ACCESS_INBOUND, ERROR_MORE_DATA -from lib.common.defines import PIPE_TYPE_BYTE, PIPE_WAIT, ERROR_PIPE_CONNECTED -from lib.common.defines import PIPE_UNLIMITED_INSTANCES, INVALID_HANDLE_VALUE -from lib.common.defines import FILE_FLAG_WRITE_THROUGH, PIPE_READMODE_BYTE -from lib.common.defines import ERROR_BROKEN_PIPE, PIPE_TYPE_MESSAGE -from lib.common.defines import PIPE_ACCESS_DUPLEX, PIPE_READMODE_MESSAGE - -log = logging.getLogger(__name__) - -BUFSIZE = 0x10000 - -class PipeForwarder(threading.Thread): - """The Pipe Forwarder forwards all data received from a local pipe to - the Cuckoo server through a socket.""" - sockets = {} - active = {} - - def __init__(self, pipe_handle, destination): - threading.Thread.__init__(self) - self.pipe_handle = pipe_handle - self.destination = destination - - def run(self): - buf = create_string_buffer(BUFSIZE) - bytes_read = c_uint() - pid = c_uint() - - # The first four bytes indicate the process identifier. In case the - # pipe handle is closed in an unknown way, reopening one and - # specifying the same process identifier will reuse the same socket, - # thus making it look like as if it was never closed in the first - # place. - success = KERNEL32.ReadFile(self.pipe_handle, - byref(pid), sizeof(pid), - byref(bytes_read), None) - - if not success or bytes_read.value != sizeof(pid): - log.warning("Unable to read the process identifier of this " - "log pipe instance.") - KERNEL32.CloseHandle(self.pipe_handle) - return - - if self.active.get(pid.value): - log.warning("A second log pipe handler for an active process is " - "being requested, denying request.") - KERNEL32.CloseHandle(self.pipe_handle) - return - - if pid.value not in self.sockets: - self.sockets[pid.value] = \ - socket.create_connection(self.destination) - - sock = self.sockets[pid.value] - self.active[pid.value] = True - - while True: - success = KERNEL32.ReadFile(self.pipe_handle, - byref(buf), sizeof(buf), - byref(bytes_read), None) - - if success or KERNEL32.GetLastError() == ERROR_MORE_DATA: - sock.sendall(buf.raw[:bytes_read.value]) - # If we get the broken pipe error then this pipe connection has - # been terminated for one reason or another. So break from the - # loop and make the socket "inactive", that is, another pipe - # connection can in theory pick it up. (This will only happen in - # cases where malware for some reason broke our pipe connection). - elif KERNEL32.GetLastError() == ERROR_BROKEN_PIPE: - break - else: - log.warning("The log pipe handler has failed, last error %d.", - KERNEL32.GetLastError()) - break - - self.active[pid.value] = False - -class PipeDispatcher(threading.Thread): - """Receives commands through a local pipe, forwards them to the - dispatcher, and returns the response.""" - def __init__(self, pipe_handle, dispatcher): - threading.Thread.__init__(self) - self.pipe_handle = pipe_handle - self.dispatcher = dispatcher - self.do_run = True - - def _read_message(self, buf): - """Reads a message.""" - bytes_read = c_uint() - ret = "" - - while True: - success = KERNEL32.ReadFile(self.pipe_handle, - byref(buf), sizeof(buf), - byref(bytes_read), None) - - if KERNEL32.GetLastError() == ERROR_MORE_DATA: - ret += buf.raw[:bytes_read.value] - elif success: - return ret + buf.raw[:bytes_read.value] - else: - return - - def run(self): - """Run the pipe dispatcher.""" - buf = create_string_buffer(BUFSIZE) - bytes_written = c_uint() - - while self.do_run: - message = self._read_message(buf) - if not message: - break - - response = self.dispatcher.dispatch(message) or "OK" - - KERNEL32.WriteFile(self.pipe_handle, response, len(response), - byref(bytes_written), None) - - KERNEL32.CloseHandle(self.pipe_handle) - -class PipeServer(threading.Thread): - """The Pipe Server accepts incoming pipe handlers and initializes - them in a new thread.""" - def __init__(self, pipe_handler, pipe_name, message=False, **kwargs): - threading.Thread.__init__(self) - self.pipe_handler = pipe_handler - self.pipe_name = pipe_name - self.message = message - self.kwargs = kwargs - self.do_run = True - - def run(self): - while self.do_run: - flags = FILE_FLAG_WRITE_THROUGH - if self.message: - pipe_handle = KERNEL32.CreateNamedPipeA( - self.pipe_name, PIPE_ACCESS_DUPLEX | flags, - PIPE_TYPE_MESSAGE | PIPE_READMODE_MESSAGE | PIPE_WAIT, - PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE, 0, None) - else: - pipe_handle = KERNEL32.CreateNamedPipeA( - self.pipe_name, PIPE_ACCESS_INBOUND | flags, - PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT, - PIPE_UNLIMITED_INSTANCES, 0, BUFSIZE, 0, None) - - if pipe_handle == INVALID_HANDLE_VALUE: - log.warning("Error opening logging pipe server.") - continue - - if KERNEL32.ConnectNamedPipe(pipe_handle, None) or \ - KERNEL32.GetLastError() == ERROR_PIPE_CONNECTED: - handler = self.pipe_handler(pipe_handle, **self.kwargs) - handler.daemon = True - handler.start() - else: - KERNEL32.CloseHandle(pipe_handle) - - def stop(self): - self.do_run = False diff --git a/analyzer/windows/lib/core/startup.py b/analyzer/windows/lib/core/startup.py deleted file mode 100644 index 7373bd886e..0000000000 --- a/analyzer/windows/lib/core/startup.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import os -import logging - -from lib.common.constants import PATHS -from lib.common.results import NetlogHandler - -log = logging.getLogger() - -def create_folders(): - """Create folders in PATHS.""" - for name, folder in PATHS.items(): - if os.path.exists(folder): - continue - - try: - os.makedirs(folder) - except OSError: - pass - -def init_logging(): - """Initialize logger.""" - formatter = logging.Formatter("%(asctime)s [%(name)s] %(levelname)s: %(message)s") - sh = logging.StreamHandler() - sh.setFormatter(formatter) - log.addHandler(sh) - - nh = NetlogHandler() - nh.setFormatter(formatter) - log.addHandler(nh) - - log.setLevel(logging.DEBUG) diff --git a/analyzer/windows/modules/__init__.py b/analyzer/windows/modules/__init__.py deleted file mode 100644 index 13fb1d219c..0000000000 --- a/analyzer/windows/modules/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. diff --git a/analyzer/windows/modules/auxiliary/__init__.py b/analyzer/windows/modules/auxiliary/__init__.py deleted file mode 100644 index 13fb1d219c..0000000000 --- a/analyzer/windows/modules/auxiliary/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. diff --git a/analyzer/windows/modules/auxiliary/dumptls.py b/analyzer/windows/modules/auxiliary/dumptls.py deleted file mode 100644 index f207f683ae..0000000000 --- a/analyzer/windows/modules/auxiliary/dumptls.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -from lib.api.process import Process -from lib.common.abstracts import Auxiliary - -class DumpTLSMasterSecrets(Auxiliary): - """Dump TLS master secrets as used by various Windows libraries.""" - def start(self): - Process(process_name="lsass.exe").inject(track=False, mode="dumptls") diff --git a/analyzer/windows/modules/auxiliary/human.py b/analyzer/windows/modules/auxiliary/human.py deleted file mode 100644 index 41bbb18079..0000000000 --- a/analyzer/windows/modules/auxiliary/human.py +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import random -import logging -from threading import Thread -from ctypes import WINFUNCTYPE, POINTER -from ctypes import c_bool, c_int, create_unicode_buffer - -from lib.common.abstracts import Auxiliary -from lib.common.defines import KERNEL32, USER32 -from lib.common.defines import WM_GETTEXT, WM_GETTEXTLENGTH, BM_CLICK - -log = logging.getLogger(__name__) - -EnumWindowsProc = WINFUNCTYPE(c_bool, POINTER(c_int), POINTER(c_int)) -EnumChildProc = WINFUNCTYPE(c_bool, POINTER(c_int), POINTER(c_int)) - -RESOLUTION = { - "x": USER32.GetSystemMetrics(0), - "y": USER32.GetSystemMetrics(1) -} - -def foreach_child(hwnd, lparam): - # List of buttons labels to click. - buttons = [ - "yes", - "ok", - "accept", - "next", - "install", - "run", - "agree", - "enable", - "don't send", - "continue", - "unzip", - "open", - ] - - # List of buttons labels to not click. - dontclick = [ - "don't run", - ] - - classname = create_unicode_buffer(50) - USER32.GetClassNameW(hwnd, classname, 50) - - # Check if the class of the child is button. - if classname.value == "Button": - # Get the text of the button. - length = USER32.SendMessageW(hwnd, WM_GETTEXTLENGTH, 0, 0) - text = create_unicode_buffer(length + 1) - USER32.SendMessageW(hwnd, WM_GETTEXT, length + 1, text) - - # Check if the button is set as "clickable" and click it. - textval = text.value.replace("&", "").lower() - for button in buttons: - if button in textval: - for btn in dontclick: - if btn in textval: - break - else: - log.info("Found button \"%s\", clicking it" % text.value) - USER32.SetForegroundWindow(hwnd) - KERNEL32.Sleep(1000) - USER32.SendMessageW(hwnd, BM_CLICK, 0, 0) - - # Recursively search for childs (USER32.EnumChildWindows). - return True - -# Callback procedure invoked for every enumerated window. -def foreach_window(hwnd, lparam): - # If the window is visible, enumerate its child objects, looking - # for buttons. - if USER32.IsWindowVisible(hwnd): - USER32.EnumChildWindows(hwnd, EnumChildProc(foreach_child), 0) - return True - -def move_mouse(): - x = random.randint(0, RESOLUTION["x"]) - y = random.randint(0, RESOLUTION["y"]) - - # Originally was: - # USER32.mouse_event(0x8000, x, y, 0, None) - # Changed to SetCurorPos, since using GetCursorPos would not detect - # the mouse events. This actually moves the cursor around which might - # cause some unintended activity on the desktop. We might want to make - # this featur optional. - USER32.SetCursorPos(x, y) - -def click_mouse(): - # Move mouse to top-middle position. - USER32.SetCursorPos(RESOLUTION["x"] / 2, 0) - # Mouse down. - USER32.mouse_event(2, 0, 0, 0, None) - KERNEL32.Sleep(50) - # Mouse up. - USER32.mouse_event(4, 0, 0, 0, None) - -class Human(Auxiliary, Thread): - """Human after all""" - - def __init__(self, options={}, analyzer=None): - Thread.__init__(self) - Auxiliary.__init__(self, options, analyzer) - self.do_run = True - - def stop(self): - self.do_run = False - - def run(self): - # Global disable flag. - if "human" in self.options: - self.do_move_mouse = int(self.options["human"]) - self.do_click_mouse = int(self.options["human"]) - self.do_click_buttons = int(self.options["human"]) - else: - self.do_move_mouse = True - self.do_click_mouse = True - self.do_click_buttons = True - - # Per-feature enable or disable flag. - if "human.move_mouse" in self.options: - self.do_move_mouse = int(self.options["human.move_mouse"]) - - if "human.click_mouse" in self.options: - self.do_click_mouse = int(self.options["human.click_mouse"]) - - if "human.click_buttons" in self.options: - self.do_click_buttons = int(self.options["human.click_buttons"]) - - while self.do_run: - if self.do_click_mouse: - click_mouse() - - if self.do_move_mouse: - move_mouse() - - if self.do_click_buttons: - USER32.EnumWindows(EnumWindowsProc(foreach_window), 0) - - KERNEL32.Sleep(1000) diff --git a/analyzer/windows/modules/auxiliary/screenshots.py b/analyzer/windows/modules/auxiliary/screenshots.py deleted file mode 100644 index 5c91436f13..0000000000 --- a/analyzer/windows/modules/auxiliary/screenshots.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import time -import logging -import StringIO -from threading import Thread - -from lib.common.abstracts import Auxiliary -from lib.common.results import NetlogFile -from lib.api.screenshot import Screenshot - -log = logging.getLogger(__name__) -SHOT_DELAY = 1 -# Skip the following area when comparing screen shots. -# Example for 800x600 screen resolution. -# SKIP_AREA = ((735, 575), (790, 595)) -SKIP_AREA = None - -class Screenshots(Auxiliary, Thread): - """Take screenshots.""" - - def __init__(self, options={}, analyzer=None): - Thread.__init__(self) - Auxiliary.__init__(self, options, analyzer) - self.do_run = True - - def stop(self): - """Stop screenshotting.""" - self.do_run = False - - def run(self): - """Run screenshotting. - @return: operation status. - """ - if "screenshots" in self.options: - self.do_run = int(self.options["screenshots"]) - - if not Screenshot().have_pil(): - log.warning("Python Image Library is not installed, " - "screenshots are disabled") - return False - - img_counter = 0 - img_last = None - - while self.do_run: - time.sleep(SHOT_DELAY) - - try: - img_current = Screenshot().take() - except IOError as e: - log.error("Cannot take screenshot: %s", e) - continue - - if img_last: - if Screenshot().equal(img_last, img_current, SKIP_AREA): - continue - - img_counter += 1 - - # workaround as PIL can't write to the socket file object :( - tmpio = StringIO.StringIO() - img_current.save(tmpio, format="JPEG") - tmpio.seek(0) - - # now upload to host from the StringIO - nf = NetlogFile("shots/%s.jpg" % str(img_counter).rjust(4, "0")) - - for chunk in tmpio: - nf.sock.sendall(chunk) - - nf.close() - - img_last = img_current - - return True diff --git a/analyzer/windows/modules/packages/__init__.py b/analyzer/windows/modules/packages/__init__.py deleted file mode 100644 index 13fb1d219c..0000000000 --- a/analyzer/windows/modules/packages/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. diff --git a/analyzer/windows/modules/packages/doc.py b/analyzer/windows/modules/packages/doc.py deleted file mode 100644 index a62c12a332..0000000000 --- a/analyzer/windows/modules/packages/doc.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -from _winreg import HKEY_CURRENT_USER - -from lib.common.abstracts import Package - -class DOC(Package): - """Word analysis package.""" - PATHS = [ - ("ProgramFiles", "Microsoft Office", "WINWORD.EXE"), - ("ProgramFiles", "Microsoft Office", "Office11", "WINWORD.EXE"), - ("ProgramFiles", "Microsoft Office", "Office12", "WINWORD.EXE"), - ("ProgramFiles", "Microsoft Office", "Office14", "WINWORD.EXE"), - ("ProgramFiles", "Microsoft Office", "Office15", "WINWORD.EXE"), - ("ProgramFiles", "Microsoft Office 15", "root", "office15", "WINWORD.EXE"), - ("ProgramFiles", "Microsoft Office", "WORDVIEW.EXE"), - ] - - REGKEYS = [ - [ - HKEY_CURRENT_USER, - "Software\\Microsoft\\Office\\12.0\\Common\\General", - { - # "Welcome to the 2007 Microsoft Office system" - "ShownOptIn": 1, - }, - ], - [ - HKEY_CURRENT_USER, - "Software\\Microsoft\\Office\\12.0\\Word\\Security", - { - # Enable VBA macros in Office 2007. - "VBAWarnings": 1, - "AccessVBOM": 1, - - # "The file you are trying to open .xyz is in a different - # format than specified by the file extension. Verify the file - # is not corrupted and is from trusted source before opening - # the file. Do you want to open the file now?" - "ExtensionHardening": 0, - }, - ], - ] - - def start(self, path): - word = self.get_path("Microsoft Office Word") - return self.execute(word, args=[path]) diff --git a/analyzer/windows/modules/packages/generic.py b/analyzer/windows/modules/packages/generic.py deleted file mode 100644 index 9bc7ced071..0000000000 --- a/analyzer/windows/modules/packages/generic.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -from lib.common.abstracts import Package -from lib.common.rand import random_string - -class Generic(Package): - """Generic analysis package. - The sample is started using START command in a cmd.exe prompt. - """ - PATHS = [ - ("System32", "cmd.exe"), - ] - - def start(self, path): - cmd_path = self.get_path("cmd.exe") - - # Create random cmd.exe window title. - rand_title = random_string(4, 16) - - # START syntax. - # See: https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/start.mspx?mfr=true - # start ["title"] [/dPath] [/i] [/min] [/max] [{/separate | /shared}] - # [{/low | /normal | /high | /realtime | /abovenormal | belownormal}] - # [/wait] [/b] [FileName] [parameters] - args = ["/c", "start", "/wait", rand_title, path] - return self.execute(cmd_path, args=args) diff --git a/analyzer/windows/modules/packages/js.py b/analyzer/windows/modules/packages/js.py deleted file mode 100644 index aae418f637..0000000000 --- a/analyzer/windows/modules/packages/js.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -from lib.common.abstracts import Package - -class Javascript(Package): - """Javascript analysis package.""" - PATHS = [ - ("System32", "wscript.exe"), - ] - - def start(self, path): - wscript = self.get_path("WScript") - return self.execute(wscript, args=[path]) diff --git a/analyzer/windows/modules/packages/pdf.py b/analyzer/windows/modules/packages/pdf.py deleted file mode 100644 index 131c1e1256..0000000000 --- a/analyzer/windows/modules/packages/pdf.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -from _winreg import HKEY_LOCAL_MACHINE, HKEY_CURRENT_USER - -from lib.common.abstracts import Package - -class PDF(Package): - """PDF analysis package.""" - PATHS = [ - ("ProgramFiles", "Adobe", "Reader 8.0", "Reader", "AcroRd32.exe"), - ("ProgramFiles", "Adobe", "Reader 9.0", "Reader", "AcroRd32.exe"), - ("ProgramFiles", "Adobe", "Reader 10.0", "Reader", "AcroRd32.exe"), - ("ProgramFiles", "Adobe", "Reader 11.0", "Reader", "AcroRd32.exe"), - ("ProgramFiles", "Adobe", "Acrobat Reader DC", "Reader", "AcroRd32.exe"), - ] - - REGKEYS = [ - [ - HKEY_LOCAL_MACHINE, - "SOFTWARE\\Adobe\\Acrobat Reader\\9.0\\AdobeViewer", - { - # Accept EULA for Adobe Reader 9.0. - "EULA": 1, - }, - ], - [ - HKEY_CURRENT_USER, - "SOFTWARE\\Adobe\\Acrobat Reader\\9.0\\AdobeViewer", - { - # Accept EULA for Adobe Reader 9.0. - "EULA": 1, - }, - ], - ] - - def start(self, path): - reader = self.get_path("Adobe Reader") - return self.execute(reader, args=[path], maximize=True) diff --git a/analyzer/windows/modules/packages/ppt.py b/analyzer/windows/modules/packages/ppt.py deleted file mode 100644 index 832b96950b..0000000000 --- a/analyzer/windows/modules/packages/ppt.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -from _winreg import HKEY_CURRENT_USER - -from lib.common.abstracts import Package - -class PPT(Package): - """PowerPoint analysis package.""" - PATHS = [ - ("ProgramFiles", "Microsoft Office", "POWERPNT.EXE"), - ("ProgramFiles", "Microsoft Office", "Office11", "POWERPNT.EXE"), - ("ProgramFiles", "Microsoft Office", "Office12", "POWERPNT.EXE"), - ("ProgramFiles", "Microsoft Office", "Office14", "POWERPNT.EXE"), - ("ProgramFiles", "Microsoft Office", "Office15", "POWERPNT.EXE"), - ("ProgramFiles", "Microsoft Office 15", "root", "office15", "POWERPNT.EXE"), - ] - - REGKEYS = [ - [ - HKEY_CURRENT_USER, - "Software\\Microsoft\\Office\\12.0\\Common\\General", - { - # "Welcome to the 2007 Microsoft Office system" - "ShownOptIn": 1, - }, - ], - [ - HKEY_CURRENT_USER, - "Software\\Microsoft\\Office\\12.0\\Powerpoint\\Security", - { - # Enable VBA macros in Office 2007. - "VBAWarnings": 1, - "AccessVBOM": 1, - - # "The file you are trying to open .xyz is in a different - # format than specified by the file extension. Verify the file - # is not corrupted and is from trusted source before opening - # the file. Do you want to open the file now?" - "ExtensionHardening": 0, - }, - ], - ] - - def start(self, path): - powerpoint = self.get_path("Microsoft Office PowerPoint") - return self.execute(powerpoint, args=[path]) diff --git a/analyzer/windows/modules/packages/ps1.py b/analyzer/windows/modules/packages/ps1.py deleted file mode 100644 index 2df98181f5..0000000000 --- a/analyzer/windows/modules/packages/ps1.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -from lib.common.abstracts import Package - -# Originally proposed by David Maciejak. - -class PS1(Package): - """PowerShell analysis package.""" - PATHS = [ - ("System32", "WindowsPowerShell", "v1.0", "powershell.exe"), - ("System32", "WindowsPowerShell", "v2.0", "powershell.exe"), - ("System32", "WindowsPowerShell", "v3.0", "powershell.exe"), - ] - - def start(self, path): - powershell = self.get_path("PowerShell") - args = ["-NoProfile", "-ExecutionPolicy", "unrestricted", - "-File", path] - return self.execute(powershell, args=args) diff --git a/analyzer/windows/modules/packages/vbs.py b/analyzer/windows/modules/packages/vbs.py deleted file mode 100644 index f15a9ba5f0..0000000000 --- a/analyzer/windows/modules/packages/vbs.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -from lib.common.abstracts import Package - -# Originally proposed by kidrek: -# https://github.com/cuckoobox/cuckoo/pull/136 - -class VBS(Package): - """VBS analysis package.""" - PATHS = [ - ("System32", "wscript.exe"), - ] - - def start(self, path): - wscript = self.get_path("WScript") - return self.execute(wscript, args=[path]) diff --git a/analyzer/windows/modules/packages/xls.py b/analyzer/windows/modules/packages/xls.py deleted file mode 100644 index 689d7e1e90..0000000000 --- a/analyzer/windows/modules/packages/xls.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -from _winreg import HKEY_CURRENT_USER - -from lib.common.abstracts import Package - -class XLS(Package): - """Excel analysis package.""" - PATHS = [ - ("ProgramFiles", "Microsoft Office", "EXCEL.EXE"), - ("ProgramFiles", "Microsoft Office", "Office11", "EXCEL.EXE"), - ("ProgramFiles", "Microsoft Office", "Office12", "EXCEL.EXE"), - ("ProgramFiles", "Microsoft Office", "Office14", "EXCEL.EXE"), - ("ProgramFiles", "Microsoft Office", "Office15", "EXCEL.EXE"), - ("ProgramFiles", "Microsoft Office 15", "root", "office15", "EXCEL.EXE"), - ] - - REGKEYS = [ - [ - HKEY_CURRENT_USER, - "Software\\Microsoft\\Office\\12.0\\Common\\General", - { - # "Welcome to the 2007 Microsoft Office system" - "ShownOptIn": 1, - }, - ], - [ - HKEY_CURRENT_USER, - "Software\\Microsoft\\Office\\12.0\\Excel\\Security", - { - # Enable VBA macros in Office 2007. - "VBAWarnings": 1, - "AccessVBOM": 1, - - # "The file you are trying to open .xyz is in a different - # format than specified by the file extension. Verify the file - # is not corrupted and is from trusted source before opening - # the file. Do you want to open the file now?" - "ExtensionHardening": 0, - }, - ], - ] - - def start(self, path): - excel = self.get_path("Microsoft Office Excel") - return self.execute(excel, args=[path]) diff --git a/analyzer/windows/modules/packages/zip.py b/analyzer/windows/modules/packages/zip.py deleted file mode 100644 index d064a43108..0000000000 --- a/analyzer/windows/modules/packages/zip.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import os -import shutil -import logging - -from zipfile import ZipFile, BadZipfile - -from lib.common.abstracts import Package -from lib.common.exceptions import CuckooPackageError - -log = logging.getLogger(__name__) - -class Zip(Package): - """Zip analysis package.""" - - def extract_zip(self, zip_path, extract_path, password): - """Extracts a nested ZIP file. - @param zip_path: ZIP path - @param extract_path: where to extract - @param password: ZIP password - """ - # Test if zip file contains a file named as itself. - if self.is_overwritten(zip_path): - log.debug("ZIP file contains a file with the same name, original is going to be overwrite") - # TODO: add random string. - new_zip_path = zip_path + ".old" - shutil.move(zip_path, new_zip_path) - zip_path = new_zip_path - - # Extraction. - with ZipFile(zip_path, "r") as archive: - try: - archive.extractall(path=extract_path, pwd=password) - except BadZipfile: - raise CuckooPackageError("Invalid Zip file") - except RuntimeError: - try: - archive.extractall(path=extract_path, pwd="infected") - except RuntimeError as e: - raise CuckooPackageError("Unable to extract Zip file: " - "{0}".format(e)) - finally: - # Extract nested archives. - for name in archive.namelist(): - if name.endswith(".zip"): - # Recurse. - self.extract_zip(os.path.join(extract_path, name), extract_path, password) - - def is_overwritten(self, zip_path): - """Checks if the ZIP file contains another file with the same name, so it is going to be overwritten. - @param zip_path: zip file path - @return: comparison boolean - """ - with ZipFile(zip_path, "r") as archive: - try: - # Test if zip file contains a file named as itself. - for name in archive.namelist(): - if name == os.path.basename(zip_path): - return True - return False - except BadZipfile: - raise CuckooPackageError("Invalid Zip file") - - def get_infos(self, zip_path): - """Get information from ZIP file. - @param zip_path: zip file path - @return: ZipInfo class - """ - try: - with ZipFile(zip_path, "r") as archive: - return archive.infolist() - except BadZipfile: - raise CuckooPackageError("Invalid Zip file") - - def start(self, path): - password = self.options.get("password") - - zipinfos = self.get_infos(path) - self.extract_zip(path, self.curdir, password) - - file_name = self.options.get("file") - # If no file name is provided via option, take the first file. - if not file_name: - # No name provided try to find a better name. - if len(zipinfos): - # Take the first one. - file_name = zipinfos[0].filename - log.debug("Missing file option, auto executing: {0}".format(file_name)) - else: - raise CuckooPackageError("Empty ZIP archive") - - file_path = os.path.join(self.curdir, file_name) - return self.execute(file_path, self.options.get("arguments")) diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000000..dfa37be4f3 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,56 @@ +environment: + matrix: + - PYTHON: "C:/Python27" + - PYTHON: "C:/Python27-x64" + global: + MYSQL_PWD: Password12! + POSTGRES_PORT: tcp://localhost:5432 + PGUSER: postgres + PGPASSWORD: Password12! + +services: + - mongodb + - mysql + - postgresql + +install: + - "set PATH=%PYTHON%;%PYTHON%/Scripts;%PATH%" + + # Fetch monitor & stuff which is required for setup & CWD migration tests. + - "python.exe stuff/monitor.py" + + # Bump the version of Cuckoo during tests (e.g., for new Signatures). + - "python.exe stuff/patch_version.py setup.py 3.0" + + - "python.exe setup.py sdist" + - "pip.exe install -e ." + - "pip.exe install psycopg2 mysqlclient==1.3.9 codecov flask-testing" + - "pip.exe install pytest==4.1.1 pytest-cov pytest-django pytest-pythonpath" + - "pip.exe install flask-sqlalchemy==2.4.0 mock==2.0.0 responses==0.5.1" + +build: false + +before_test: + - "set PATH=C:/Program Files/MySql/MySQL Server 5.7/bin;%PATH%" + - "set PATH=C:/Program Files/PostgreSQL/9.5/bin;%PATH%" + + - "mysql -u root -e \"CREATE USER cuckoo@localhost IDENTIFIED BY 'cuckoo'\"" + - "mysql -u root -e \"GRANT ALL PRIVILEGES ON * . * TO cuckoo@localhost\"" + - "mysql -u root -e \"FLUSH PRIVILEGES\"" + - "mysql -u root -e \"CREATE DATABASE cuckootest\"" + - "psql -c \"CREATE USER cuckoo WITH PASSWORD 'cuckoo'\"" + - "psql -c \"CREATE DATABASE cuckootest\"" + + - "mysql -u root -e \"CREATE DATABASE cuckootestimport\"" + - "mysql -u root cuckootestimport < tests/files/sql/11my.sql" + - "psql -c \"CREATE DATABASE cuckootestimport\"" + - "psql cuckootestimport NUL" + +test_script: + - "cuckoo.exe -d --ignore-vuln" + - "cuckoo.exe community" + - "cuckoo.exe -d --ignore-vuln || dir >NUL" + - "pytest.exe --cov=cuckoo" + +after_test: + - "codecov.exe" diff --git a/conf/auxiliary.conf b/conf/auxiliary.conf deleted file mode 100644 index ae48519104..0000000000 --- a/conf/auxiliary.conf +++ /dev/null @@ -1,41 +0,0 @@ -[sniffer] -# Enable or disable the use of an external sniffer (tcpdump) [yes/no]. -enabled = yes - -# Specify the path to your local installation of tcpdump. Make sure this -# path is correct. -tcpdump = /usr/sbin/tcpdump - -# Specify the network interface name on which tcpdump should monitor the -# traffic. Make sure the interface is active. -interface = vboxnet0 - -# Specify a Berkeley packet filter to pass to tcpdump. -# bpf = not arp - -[mitm] -# Enable man in the middle proxying (mitmdump) [yes/no]. -enabled = no - -# Specify the path to your local installation of mitmdump. Make sure this -# path is correct. -mitmdump = /usr/local/bin/mitmdump - -# Listen port base. Each virtual machine will use its own port to be -# able to make a good distinction between the various running analyses. -# Generally port 50000 should be fine, in this case port 50001, 50002, etc -# will also be used - again, one port per analyses. -port_base = 50000 - -# Script file to interact with the network traffic. Please refer to the -# documentation of mitmproxy/mitmdump to get an understand of their internal -# workings. (https://mitmproxy.org/doc/scripting/inlinescripts.html) -script = data/mitm.py - -# Path to the certificate to be used by mitmdump. This file will be -# automatically generated for you if you run mitmdump once. It's just that -# you have to copy it from ~/.mitmproxy/mitmproxy-ca-cert.p12 to somewhere -# in the analyzer/windows/ directory. Recommended is to write the certificate -# to analyzer/windows/bin/cert.p12, in that case the following option should -# be set to bin/cert.p12. -certificate = bin/cert.p12 diff --git a/conf/cuckoo.conf b/conf/cuckoo.conf deleted file mode 100644 index 5bca62abc8..0000000000 --- a/conf/cuckoo.conf +++ /dev/null @@ -1,127 +0,0 @@ -[cuckoo] -# Enable or disable startup version check. When enabled, Cuckoo will connect -# to a remote location to verify whether the running version is the latest -# one available. -version_check = on - -# If turned on, Cuckoo will delete the original file after its analysis -# has been completed. -delete_original = off - -# If turned on, Cuckoo will delete the copy of the original file in the -# local binaries repository after the analysis has finished. (On *nix this -# will also invalidate the file called "binary" in each analysis directory, -# as this is a symlink.) -delete_bin_copy = off - -# Specify the name of the machinery module to use, this module will -# define the interaction between Cuckoo and your virtualization software -# of choice. -machinery = virtualbox - -# Enable creation of memory dump of the analysis machine before shutting -# down. Even if turned off, this functionality can also be enabled at -# submission. Currently available for: VirtualBox and libvirt modules (KVM). -memory_dump = off - -# When the timeout of an analysis is hit, the VM is just killed by default. -# For some long-running setups it might be interesting to terminate the -# monitored processes before killing the VM so that connections are closed. -terminate_processes = off - -# Enable automatically re-schedule of "broken" tasks each startup. -# Each task found in status "processing" is re-queued for analysis. -reschedule = off - -# Enable processing of results within the main cuckoo process. -# This is the default behavior but can be switched off for setups that -# require high stability and process the results in a separate task. -process_results = on - -# Limit the amount of analysis jobs a Cuckoo process goes through. -# This can be used together with a watchdog to mitigate risk of memory leaks. -max_analysis_count = 0 - -# Limit the number of concurrently executing analysis machines. -# This may be useful on systems with limited resources. -# Set to 0 to disable any limits. -max_machines_count = 0 - -# Limit the amount of VMs that are allowed to start in parallel. Generally -# speaking starting the VMs is one of the more CPU intensive parts of the -# actual analysis. This option tries to avoid maxing out the CPU completely. -max_vmstartup_count = 10 - -# Minimum amount of free space (in MB) available before starting a new task. -# This tries to avoid failing an analysis because the reports can't be written -# due out-of-diskspace errors. Setting this value to 0 disables the check. -# (Note: this feature is currently not supported under Windows.) -freespace = 64 - -# Temporary directory containing the files uploaded through Cuckoo interfaces -# (api.py and Django web interface). -tmppath = /tmp - -[resultserver] -# The Result Server is used to receive in real time the behavioral logs -# produced by the analyzer. -# Specify the IP address of the host. The analysis machines should be able -# to contact the host through such address, so make sure it's valid. -# NOTE: if you set resultserver IP to 0.0.0.0 you have to set the option -# `resultserver_ip` for all your virtual machines in machinery configuration. -ip = 192.168.56.1 - -# Specify a port number to bind the result server on. -port = 2042 - -# Should the server write the legacy CSV format? -# (if you have any custom processing on those, switch this on) -store_csvs = off - -# Maximum size of uploaded files from VM (screenshots, dropped files, log) -# The value is expressed in bytes, by default 10Mb. -upload_max_size = 10485760 - -[processing] -# Set the maximum size of analyses generated files to process. This is used -# to avoid the processing of big files which may take a lot of processing -# time. The value is expressed in bytes, by default 100Mb. -analysis_size_limit = 104857600 - -# Enable or disable DNS lookups. -resolve_dns = on - -# Enable PCAP sorting, needed for the connection content view in the web interface. -sort_pcap = on - -[database] -# Specify the database connection string. -# NOTE: If you are using a custom database (different from sqlite), you have to -# use utf-8 encoding when issuing the SQL database creation statement. -# Examples, see documentation for more: -# sqlite:///foo.db -# postgresql://foo:bar@localhost:5432/mydatabase -# mysql://foo:bar@localhost/mydatabase -# If empty, default is a SQLite in db/cuckoo.db. -connection = - -# Database connection timeout in seconds. -# If empty, default is set to 60 seconds. -timeout = - -[timeouts] -# Set the default analysis timeout expressed in seconds. This value will be -# used to define after how many seconds the analysis will terminate unless -# otherwise specified at submission. -default = 120 - -# Set the critical timeout expressed in seconds. After this timeout is hit -# Cuckoo will consider the analysis failed and it will shutdown the machine -# no matter what. When this happens the analysis results will most likely -# be lost. Make sure to have a critical timeout greater than the -# default timeout. -critical = 600 - -# Maximum time to wait for virtual machine status change. For example when -# shutting down a vm. Default is 60 seconds. -vm_state = 60 diff --git a/conf/esx.conf b/conf/esx.conf deleted file mode 100644 index 952dd2c891..0000000000 --- a/conf/esx.conf +++ /dev/null @@ -1,58 +0,0 @@ -[esx] -# ?no_verify disables the SSL signature check. By default it is self signed -dsn = esx://127.0.0.1/?no_verify=1 -username = username_goes_here -password = password_goes_here - -machines = analysis1 - -[analysis1] -# Specify the label name of the current machine as specified in your -# libvirt configuration. -label = cuckoo1 - -# Specify the operating system platform used by current machine -# [windows/darwin/linux]. -platform = windows - -# Please specify the name of the base snapshot. This snapshot should be taken -# with agent in startup and the machine shut down. -snapshot = clean_snapshot - -# Specify the IP address of the current virtual machine. Make sure that the -# IP address is valid and that the host machine is able to reach it. If not, -# the analysis will fail. You may want to configure your network settings in -# /etc/libvirt//networks/ -ip = 192.168.122.105 - -# (Optional) Specify the snapshot name to use. If you do not specify a snapshot -# name, the KVM MachineManager will use the current snapshot. -# Example (Snapshot1 is the snapshot name): -# snapshot = Snapshot1 - -# (Optional) Specify the name of the network interface that should be used -# when dumping network traffic from this machine with tcpdump. If specified, -# overrides the default interface specified in auxiliary.conf -# Example (eth0 is the interface name): -# interface = eth0 - -# (Optional) Specify the IP of the Result Server, as your virtual machine sees it. -# The Result Server will always bind to the address and port specified in cuckoo.conf, -# however you could set up your virtual network to use NAT/PAT, so you can specify here -# the IP address for the Result Server as your machine sees it. If you don't specify an -# address here, the machine will use the default value from cuckoo.conf. -# NOTE: if you set this option you have to set result server IP to 0.0.0.0 in cuckoo.conf. -# Example: -# resultserver_ip = 192.168.122.101 - -# (Optional) Specify the port for the Result Server, as your virtual machine sees it. -# The Result Server will always bind to the address and port specified in cuckoo.conf, -# however you could set up your virtual network to use NAT/PAT, so you can specify here -# the port for the Result Server as your machine sees it. If you don't specify a port -# here, the machine will use the default value from cuckoo.conf. -# Example: -# resultserver_port = 2042 - -# (Optional) Set your own tags. These are comma separated and help to identify -# specific VMs. You can run samples on VMs with tag you require. -# tags = windows_xp_sp3,32_bit,acrobat_reader_6 \ No newline at end of file diff --git a/conf/kvm.conf b/conf/kvm.conf deleted file mode 100644 index 8fa4f6dd00..0000000000 --- a/conf/kvm.conf +++ /dev/null @@ -1,52 +0,0 @@ -[kvm] -# Specify a comma-separated list of available machines to be used. For each -# specified ID you have to define a dedicated section containing the details -# on the respective machine. (E.g. cuckoo1,cuckoo2,cuckoo3) -machines = cuckoo1 - -[cuckoo1] -# Specify the label name of the current machine as specified in your -# libvirt configuration. -label = cuckoo1 - -# Specify the operating system platform used by current machine -# [windows/darwin/linux]. -platform = windows - -# Specify the IP address of the current virtual machine. Make sure that the -# IP address is valid and that the host machine is able to reach it. If not, -# the analysis will fail. You may want to configure your network settings in -# /etc/libvirt//networks/ -ip = 192.168.122.105 - -# (Optional) Specify the snapshot name to use. If you do not specify a snapshot -# name, the KVM MachineManager will use the current snapshot. -# Example (Snapshot1 is the snapshot name): -# snapshot = Snapshot1 - -# (Optional) Specify the name of the network interface that should be used -# when dumping network traffic from this machine with tcpdump. If specified, -# overrides the default interface specified in auxiliary.conf -# Example (virbr0 is the interface name): -# interface = virbr0 - -# (Optional) Specify the IP of the Result Server, as your virtual machine sees it. -# The Result Server will always bind to the address and port specified in cuckoo.conf, -# however you could set up your virtual network to use NAT/PAT, so you can specify here -# the IP address for the Result Server as your machine sees it. If you don't specify an -# address here, the machine will use the default value from cuckoo.conf. -# NOTE: if you set this option you have to set result server IP to 0.0.0.0 in cuckoo.conf. -# Example: -# resultserver_ip = 192.168.122.101 - -# (Optional) Specify the port for the Result Server, as your virtual machine sees it. -# The Result Server will always bind to the address and port specified in cuckoo.conf, -# however you could set up your virtual network to use NAT/PAT, so you can specify here -# the port for the Result Server as your machine sees it. If you don't specify a port -# here, the machine will use the default value from cuckoo.conf. -# Example: -# resultserver_port = 2042 - -# (Optional) Set your own tags. These are comma separated and help to identify -# specific VMs. You can run samples on VMs with tag you require. -# tags = windows_xp_sp3,32_bit,acrobat_reader_6 diff --git a/conf/processing.conf b/conf/processing.conf deleted file mode 100644 index cd2325fe60..0000000000 --- a/conf/processing.conf +++ /dev/null @@ -1,76 +0,0 @@ -# Enable or disable the available processing modules [on/off]. -# If you add a custom processing module to your Cuckoo setup, you have to add -# a dedicated entry in this file, or it won't be executed. -# You can also add additional options under the section of your module and -# they will be available in your Python class. - -[analysisinfo] -enabled = yes - -[apkinfo] -enabled = no -# Decompiling dex files with androguard in a heavy operation. For large dex -# files it can really take quite a while - it is recommended to limit to a -# certain filesize. -# decompilation_threshold=5000000 - -[behavior] -enabled = yes - -[buffer] -enabled = yes - -[debug] -enabled = yes - -[droidmon] -enabled = no - -[dropped] -enabled = yes - -[dumptls] -enabled = yes - -[googleplay] -enabled = no -android_id = -google_login = -google_password = - -[memory] -enabled = no - -[network] -enabled = yes - -[procmemory] -enabled = yes - -[screenshots] -enabled = no -tesseract = /usr/bin/tesseract - -[static] -enabled = yes - -[strings] -enabled = yes - -[targetinfo] -enabled = yes - -[virustotal] -enabled = yes -# How much time we can wait to establish VirusTotal connection and get the -# report. -timeout = 60 -# Enable this option if you want to submit files to VirusTotal not yet available -# in their database. -# NOTE: if you are dealing with sensitive stuff, enabling this option you could -# leak some files to VirusTotal. -scan = 0 -# Add your VirusTotal API key here. The default API key, kindly provided -# by the VirusTotal team, should enable you with a sufficient throughput -# and while being shared with all our users, it shouldn't affect your use. -key = a0283a2c3d55728300d064874239b5346fb991317e8449fe43c902879d758088 diff --git a/conf/qemu.conf b/conf/qemu.conf deleted file mode 100644 index 6a893da89e..0000000000 --- a/conf/qemu.conf +++ /dev/null @@ -1,63 +0,0 @@ -[qemu] -# Path to one qemu binary (assumes the other ones are there as well) -path = /usr/bin/qemu-system-x86_64 - -# Specify a comma-separated list of available machines to be used. For each -# specified ID you have to define a dedicated section containing the details -# on the respective machine. (E.g. cuckoo1,cuckoo2,cuckoo3) -machines = vm1,vm2 - -[vm1] -label = vm1 - -# image path -image = /home/rep/vms/qvm_wheezy64_1.qcow2 - -# vm arch (mips/mipsel/arm/x64/x86) -# arch = default - -# Specify the operating system platform used by current machine -# [windows/darwin/linux]. -platform = linux - -# Specify the IP address of the current virtual machine. Make sure that the -# IP address is valid and that the host machine is able to reach it. If not, -# the analysis will fail. -ip = 192.168.55.2 - -# (Optional) Specify the name of the network interface that should be used -# when dumping network traffic from this machine with tcpdump. If specified, -# overrides the default interface specified in auxiliary.conf -# Example (vboxnet0 is the interface name): -interface = qemubr - -# (Optional) Specify the IP of the Result Server, as your virtual machine sees it. -# The Result Server will always bind to the address and port specified in cuckoo.conf, -# however you could set up your virtual network to use NAT/PAT, so you can specify here -# the IP address for the Result Server as your machine sees it. If you don't specify an -# address here, the machine will use the default value from cuckoo.conf. -# NOTE: if you set this option you have to set result server IP to 0.0.0.0 in cuckoo.conf. -# Example: -resultserver_ip = 192.168.55.1 - -# (Optional) Specify the port for the Result Server, as your virtual machine sees it. -# The Result Server will always bind to the address and port specified in cuckoo.conf, -# however you could set up your virtual network to use NAT/PAT, so you can specify here -# the port for the Result Server as your machine sees it. If you don't specify a port -# here, the machine will use the default value from cuckoo.conf. -# Example: -# resultserver_port = 2042 - -# (Optional) Set your own tags. These are comma separated and help to identify -# specific VMs. You can run samples on VMs with tag you require. -tags = debian_wheezy,64_bit - -[vm2] -label = vm2 -image = /home/rep/vms/qvm_wheezy64_1.qcow2 -arch = mipsel -kernel_path = {imagepath}/vmlinux-3.16.0-4-4kc-malta-mipsel -platform = linux -ip = 192.168.55.3 -interface = qemubr -tags = debian_wheezy,mipsel diff --git a/conf/reporting.conf b/conf/reporting.conf deleted file mode 100644 index 879a1e212f..0000000000 --- a/conf/reporting.conf +++ /dev/null @@ -1,21 +0,0 @@ -# Enable or disable the available reporting modules [on/off]. -# If you add a custom reporting module to your Cuckoo setup, you have to add -# a dedicated entry in this file, or it won't be executed. -# You can also add additional options under the section of your module and -# they will be available in your Python class. - -[jsondump] -enabled = yes -indent = 4 -encoding = latin-1 - -[reporthtml] -enabled = no - -[mongodb] -enabled = no -host = 127.0.0.1 -port = 27017 -db = cuckoo -store_memdump = yes -paginate = 100 diff --git a/conf/virtualbox.conf b/conf/virtualbox.conf deleted file mode 100644 index fad62bde72..0000000000 --- a/conf/virtualbox.conf +++ /dev/null @@ -1,59 +0,0 @@ -[virtualbox] -# Specify which VirtualBox mode you want to run your machines on. -# Can be "gui", "sdl" or "headless". Refer to VirtualBox's official -# documentation to understand the differences. -mode = headless - -# Path to the local installation of the VBoxManage utility. -path = /usr/bin/VBoxManage - -# Specify a comma-separated list of available machines to be used. For each -# specified ID you have to define a dedicated section containing the details -# on the respective machine. (E.g. cuckoo1,cuckoo2,cuckoo3) -machines = cuckoo1 - -[cuckoo1] -# Specify the label name of the current machine as specified in your -# VirtualBox configuration. -label = cuckoo1 - -# Specify the operating system platform used by current machine -# [windows/darwin/linux]. -platform = windows - -# Specify the IP address of the current virtual machine. Make sure that the -# IP address is valid and that the host machine is able to reach it. If not, -# the analysis will fail. -ip = 192.168.56.101 - -# (Optional) Specify the snapshot name to use. If you do not specify a snapshot -# name, the VirtualBox MachineManager will use the current snapshot. -# Example (Snapshot1 is the snapshot name): -# snapshot = Snapshot1 - -# (Optional) Specify the name of the network interface that should be used -# when dumping network traffic from this machine with tcpdump. If specified, -# overrides the default interface specified in auxiliary.conf -# Example (vboxnet0 is the interface name): -# interface = vboxnet0 - -# (Optional) Specify the IP of the Result Server, as your virtual machine sees it. -# The Result Server will always bind to the address and port specified in cuckoo.conf, -# however you could set up your virtual network to use NAT/PAT, so you can specify here -# the IP address for the Result Server as your machine sees it. If you don't specify an -# address here, the machine will use the default value from cuckoo.conf. -# NOTE: if you set this option you have to set result server IP to 0.0.0.0 in cuckoo.conf. -# Example: -# resultserver_ip = 192.168.56.1 - -# (Optional) Specify the port for the Result Server, as your virtual machine sees it. -# The Result Server will always bind to the address and port specified in cuckoo.conf, -# however you could set up your virtual network to use NAT/PAT, so you can specify here -# the port for the Result Server as your machine sees it. If you don't specify a port -# here, the machine will use the default value from cuckoo.conf. -# Example: -# resultserver_port = 2042 - -# (Optional) Set your own tags. These are comma separated and help to identify -# specific VMs. You can run samples on VMs with tag you require. -# tags = windows_xp_sp3,32_bit,acrobat_reader_6 diff --git a/conf/vmware.conf b/conf/vmware.conf deleted file mode 100644 index 9fbb1e182d..0000000000 --- a/conf/vmware.conf +++ /dev/null @@ -1,56 +0,0 @@ -[vmware] -# Specify which Vmware Workstation mode you want to run your machines on. -# Can be "gui" or "nogui". Refer to VMware's official -# documentation to understand the differences. -mode = gui - -# Path to the local installation of the vmrun utility. -path = /usr/bin/vmrun - -# Specify a comma-separated list of available machines to be used. For each -# specified ID you have to define a dedicated section containing the details -# on the respective machine. (E.g. cuckoo1,cuckoo2,cuckoo3) -machines = cuckoo1 - -[cuckoo1] -# Specify the path to vmx file of this virtual machine. -vmx_path = ../cuckoo1/cuckoo1.vmx - -# Specify the snapshot name to use. -snapshot = Snapshot1 - -# Specify the operating system platform used by current machine -# [windows/darwin/linux]. -platform = windows - -# Specify the IP address of the current virtual machine. Make sure that the -# IP address is valid and that the host machine is able to reach it. If not, -# the analysis will fail. -ip = 192.168.54.111 - -# (Optional) Specify the name of the network interface that should be used -# when dumping network traffic from this machine with tcpdump. If specified, -# overrides the default interface specified in auxiliary.conf -# Example (virbr0 is the interface name): -# interface = virbr0 - -# (Optional) Specify the IP of the Result Server, as your virtual machine sees it. -# The Result Server will always bind to the address and port specified in cuckoo.conf, -# however you could set up your virtual network to use NAT/PAT, so you can specify here -# the IP address for the Result Server as your machine sees it. If you don't specify an -# address here, the machine will use the default value from cuckoo.conf. -# NOTE: if you set this option you have to set result server IP to 0.0.0.0 in cuckoo.conf. -# Example: -# resultserver_ip = 192.168.122.101 - -# (Optional) Specify the port for the Result Server, as your virtual machine sees it. -# The Result Server will always bind to the address and port specified in cuckoo.conf, -# however you could set up your virtual network to use NAT/PAT, so you can specify here -# the port for the Result Server as your machine sees it. If you don't specify a port -# here, the machine will use the default value from cuckoo.conf. -# Example: -# resultserver_port = 2042 - -# (Optional) Set your own tags. These are comma separated and help to identify -# specific VMs. You can run samples on VMs with tag you require. -# tags = windows_xp_sp3,32_bit,acrobat_reader_6 diff --git a/conf/vsphere.conf b/conf/vsphere.conf deleted file mode 100644 index 07650e9a3e..0000000000 --- a/conf/vsphere.conf +++ /dev/null @@ -1,58 +0,0 @@ -[vsphere] - -# ESXi host connection parameters -host = 10.0.0.1 -port = 443 -user = username_goes_here -pwd = password_goes_here - -machines = analysis1 - -[analysis1] -# Specify the label name of the current machine as specified on your -# vSphere host. -label = cuckoo1 - -# Specify the operating system platform used by current machine -# [windows/darwin/linux]. -platform = windows - -# Please specify the name of the snapshot. This snapshot should be taken -# while the machine is running and the agent started. -snapshot = cuckoo_ready_running - -# Specify the IP address of the current virtual machine. Make sure that the -# IP address is valid and that the host machine is able to reach it. If not, -# the analysis will fail. -ip = 192.168.1.1 - -# (Optional) Specify the name of the network interface that should be used -# when dumping network traffic from this machine with tcpdump. If specified, -# overrides the default interface specified in auxiliary.conf -# Example (eth0 is the interface name): -# interface = eth0 - -# (Optional) Specify the IP of the Result Server, as your virtual machine sees it. -# The Result Server will always bind to the address and port specified in cuckoo.conf, -# however you could set up your virtual network to use NAT/PAT, so you can specify here -# the IP address for the Result Server as your machine sees it. If you don't specify an -# address here, the machine will use the default value from cuckoo.conf. -# NOTE: if you set this option you have to set result server IP to 0.0.0.0 in cuckoo.conf. -# Example: -# resultserver_ip = 192.168.122.101 - -# (Optional) Specify the port for the Result Server, as your virtual machine sees it. -# The Result Server will always bind to the address and port specified in cuckoo.conf, -# however you could set up your virtual network to use NAT/PAT, so you can specify here -# the port for the Result Server as your machine sees it. If you don't specify a port -# here, the machine will use the default value from cuckoo.conf. -# Example: -# resultserver_port = 2042 - -# (Optional) Set your own tags. These are comma separated and help to identify -# specific VMs. You can run samples on VMs with tag you require. -# Note that the 64_bit tag is currently special. For submitted 64-bit PE files, -# the 64_bit tag will automatically be added, forcing them to be run on a 64-bit -# VM. For this reason, make sure all 64-bit VMs have the 64_bit tag. -#tags = windows_xp_sp3,32_bit,acrobat_reader_8 - diff --git a/conf/xenserver.conf b/conf/xenserver.conf deleted file mode 100644 index e1e959239c..0000000000 --- a/conf/xenserver.conf +++ /dev/null @@ -1,64 +0,0 @@ -[xenserver] -# Specify the XenServer username for authentication. -user = root - -# Specify the XenServer password for authentication. -password = changeme - -# Specify the XenServer URL. The url is the XMLRPC location of the XenServer, -# which can be either a hostname or IP address. -url = https://xenserver - -# Specify a comma-separated list of available machines to be used. For each -# specified ID you have to define a dedicated section containing the details -# on the respective machine. (E.g. cuckoo1,cuckoo2,cuckoo3) -machines = cuckoo1 - -[cuckoo1] -# Specify the virtual machine uuid. -uuid = 00000000-0000-0000-0000-000000000000 - -# Specify the snapshot uuid to use. Snapshots are not required, but if they are -# not used, the virtual machine's disks must be configured to reset on boot. -# Resetting the disks on boot ensures that samples cannot permanently modify the -# analysis virtual machine past a shutdown. Refer to the "Saving the Virtual -# Machine" section in the Cuckoo documentation for details on how to enable -# disk resetting on boot. -# Example: -# snapshot = 00000000-0000-0000-0000-000000000000 - -# Specify the operating system platform used by current machine -# [windows/darwin/linux]. -platform = windows - -# Specify the IP address of the current virtual machine. Make sure that the -# IP address is valid and that the host machine is able to reach it. If not, -# the analysis will fail. -ip = 192.168.54.111 - -# (Optional) Specify the name of the network interface that should be used -# when dumping network traffic from this machine with tcpdump. If specified, -# overrides the default interface specified in auxiliary.conf -# Example (virbr0 is the interface name): -# interface = virbr0 - -# (Optional) Specify the IP of the Result Server, as your virtual machine sees it. -# The Result Server will always bind to the address and port specified in cuckoo.conf, -# however you could set up your virtual network to use NAT/PAT, so you can specify here -# the IP address for the Result Server as your machine sees it. If you don't specify an -# address here, the machine will use the default value from cuckoo.conf. -# NOTE: if you set this option you have to set result server IP to 0.0.0.0 in cuckoo.conf. -# Example: -# resultserver_ip = 192.168.122.101 - -# (Optional) Specify the port for the Result Server, as your virtual machine sees it. -# The Result Server will always bind to the address and port specified in cuckoo.conf, -# however you could set up your virtual network to use NAT/PAT, so you can specify here -# the port for the Result Server as your machine sees it. If you don't specify a port -# here, the machine will use the default value from cuckoo.conf. -# Example: -# resultserver_port = 2042 - -# (Optional) Set your own tags. These are comma separated and help to identify -# specific VMs. You can run samples on VMs with tag you require. -# tags = windows_xp_sp3,32_bit,acrobat_reader_6 diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000000..38ca57ad17 --- /dev/null +++ b/conftest.py @@ -0,0 +1,41 @@ +# Copyright (C) 2017-2019 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import os +import shutil +import sys +import tempfile + +from cuckoo.misc import is_windows, is_linux, is_macosx, getuser, mkdir + +# Note that collect_ignore is a parameter for pytest so that it knows which +# unit tests to skip etc. In other words, perform platform-specific unit tests +# (in terms of the Cuckoo Analyzer) depending on the current host machine. +collect_ignore = [] + +if is_windows(): + sys.path.insert(0, "cuckoo/data/analyzer/windows") + collect_ignore.append("tests/linux") + collect_ignore.append("tests/darwin") + + # Copy over the monitoring binaries as if we were in a real analysis. + monitor = open("cuckoo/data/monitor/latest", "rb").read().strip() + for filename in os.listdir("cuckoo/data/monitor/%s" % monitor): + shutil.copy( + "cuckoo/data/monitor/%s/%s" % (monitor, filename), + "cuckoo/data/analyzer/windows/bin/%s" % filename + ) + +if is_linux(): + sys.path.insert(0, "cuckoo/data/analyzer/linux") + collect_ignore.append("tests/windows") + collect_ignore.append("tests/darwin") + +if is_macosx(): + sys.path.insert(0, "cuckoo/data/analyzer/darwin") + collect_ignore.append("tests/windows") + collect_ignore.append("tests/linux") + +# Ensure the Cuckoo TMP dir exists, as some tests rely on it. +mkdir(os.path.join(tempfile.gettempdir(), "cuckoo-tmp-%s" % getuser())) diff --git a/cuckoo.py b/cuckoo.py deleted file mode 100755 index f1ab242068..0000000000 --- a/cuckoo.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2010-2015 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import argparse -import logging -import os -import sys - -try: - from lib.cuckoo.common.constants import CUCKOO_VERSION, CUCKOO_ROOT - from lib.cuckoo.common.exceptions import CuckooCriticalError - from lib.cuckoo.common.exceptions import CuckooDependencyError - from lib.cuckoo.common.logo import logo - from lib.cuckoo.core.resultserver import ResultServer - from lib.cuckoo.core.scheduler import Scheduler - from lib.cuckoo.core.startup import check_working_directory, check_configs - from lib.cuckoo.core.startup import check_version, create_structure - from lib.cuckoo.core.startup import cuckoo_clean, drop_privileges - from lib.cuckoo.core.startup import init_logging, init_modules - from lib.cuckoo.core.startup import init_tasks, init_yara, init_binaries - - import bson - - bson # Pretend like it's actually being used (for static checkers.) -except (CuckooDependencyError, ImportError) as e: - sys.exit("ERROR: Missing dependency: {0}".format(e)) - -log = logging.getLogger() - -def cuckoo_init(quiet=False, debug=False, artwork=False, test=False): - """Cuckoo initialization workflow. - @param quiet: if set enable silent mode, it doesn't print anything except warnings - @param debug: if set enable debug mode, it print all debug messages - @param artwork: if set it will print only artworks, forever - @param test: enable integration test mode, used only for testing - """ - cur_path = os.getcwd() - os.chdir(CUCKOO_ROOT) - - logo() - check_working_directory() - check_configs() - check_version() - create_structure() - - if artwork: - import time - try: - while True: - time.sleep(1) - logo() - except KeyboardInterrupt: - return - - init_logging() - - if quiet: - log.setLevel(logging.WARN) - elif debug: - log.setLevel(logging.DEBUG) - - init_modules() - init_tasks() - init_yara() - init_binaries() - - # TODO: This is just a temporary hack, we need an actual test suite to - # integrate with Travis-CI. - if test: - return - - ResultServer() - - os.chdir(cur_path) - -def cuckoo_main(max_analysis_count=0): - """Cuckoo main loop. - @param max_analysis_count: kill cuckoo after this number of analyses - """ - cur_path = os.getcwd() - os.chdir(CUCKOO_ROOT) - - try: - sched = Scheduler(max_analysis_count) - sched.start() - except KeyboardInterrupt: - sched.stop() - - os.chdir(cur_path) - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("-q", "--quiet", help="Display only error messages", action="store_true", required=False) - parser.add_argument("-d", "--debug", help="Display debug messages", action="store_true", required=False) - parser.add_argument("-v", "--version", action="version", version="You are running Cuckoo Sandbox {0}".format(CUCKOO_VERSION)) - parser.add_argument("-a", "--artwork", help="Show artwork", action="store_true", required=False) - parser.add_argument("-t", "--test", help="Test startup", action="store_true", required=False) - parser.add_argument("-m", "--max-analysis-count", help="Maximum number of analyses", type=int, required=False) - parser.add_argument("-u", "--user", type=str, help="Drop user privileges to this user") - parser.add_argument("--clean", help="Remove all tasks and samples and their associated data", action='store_true', required=False) - args = parser.parse_args() - - if args.user: - drop_privileges(args.user) - - if args.clean: - cuckoo_clean() - sys.exit(0) - - try: - cuckoo_init(quiet=args.quiet, debug=args.debug, artwork=args.artwork, - test=args.test) - - if not args.artwork and not args.test: - cuckoo_main(max_analysis_count=args.max_analysis_count) - except CuckooCriticalError as e: - message = "{0}: {1}".format(e.__class__.__name__, e) - if len(log.handlers): - log.critical(message) - else: - sys.stderr.write("{0}\n".format(message)) - - sys.exit(1) diff --git a/cuckoo/__init__.py b/cuckoo/__init__.py new file mode 100644 index 0000000000..15feedea5e --- /dev/null +++ b/cuckoo/__init__.py @@ -0,0 +1,22 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +from cuckoo import ( + auxiliary, machinery, processing, reporting +) + +from cuckoo.misc import version as __version__ + +signatures = [] + +# Don't include machinery here as its data structure is different from the +# other plugins - of which multiple are in use at any time. +plugins = { + "auxiliary": auxiliary.plugins, + "machinery": machinery.plugins.values(), + "processing": processing.plugins, + "reporting": reporting.plugins, + "signatures": signatures, +} diff --git a/cuckoo/apps/__init__.py b/cuckoo/apps/__init__.py new file mode 100644 index 0000000000..a35e690552 --- /dev/null +++ b/cuckoo/apps/__init__.py @@ -0,0 +1,15 @@ +# Copyright (C) 2016-2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +from .apps import ( + fetch_community, submit_tasks, process_tasks, process_task, + process_task_range, cuckoo_clean, cuckoo_machine, migrate_database, + migrate_cwd +) + +from .api import cuckoo_api +from .distributed import cuckoo_distributed, cuckoo_distributed_instance +from .dnsserve import cuckoo_dnsserve +from .import_ import import_cuckoo +from .rooter import cuckoo_rooter, cleanup_rooter diff --git a/cuckoo/apps/api.py b/cuckoo/apps/api.py new file mode 100644 index 0000000000..ce383d2d58 --- /dev/null +++ b/cuckoo/apps/api.py @@ -0,0 +1,721 @@ +# Copyright (C) 2012-2013 Claudio Guarnieri. +# Copyright (C) 2014-2019 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import datetime +import hashlib +import io +import logging +import multiprocessing +import os +import socket +import tarfile +import zipfile + +from flask import Flask, request, jsonify, make_response, abort, json + +from cuckoo.common.config import config, parse_options +from cuckoo.common.files import Files, Folders +from cuckoo.common.utils import parse_bool, constant_time_compare +from cuckoo.core.database import Database, Task +from cuckoo.core.database import TASK_REPORTED, TASK_COMPLETED, TASK_RUNNING +from cuckoo.core.rooter import rooter +from cuckoo.core.submit import SubmitManager +from cuckoo.misc import cwd, version, decide_cwd, Pidfile + +log = logging.getLogger(__name__) +db = Database() +sm = SubmitManager() + +# Initialize Flask app. +app = Flask(__name__) + +def json_error(status_code, message): + """Return a JSON object with a HTTP error code.""" + r = jsonify(message=message) + r.status_code = status_code + return r + +def shutdown_server(): + """Shutdown API werkzeug server.""" + shutdown = request.environ.get("werkzeug.server.shutdown") + if shutdown: + shutdown() + return True + else: + return False + +@app.after_request +def custom_headers(response): + """Set some custom headers across all HTTP responses.""" + response.headers["Server"] = "Machete Server" + response.headers["X-Content-Type-Options"] = "nosniff" + response.headers["X-Frame-Options"] = "DENY" + response.headers["X-XSS-Protection"] = "1; mode=block" + response.headers["Pragma"] = "no-cache" + response.headers["Cache-Control"] = "no-cache" + response.headers["Expires"] = "0" + return response + +@app.route("/tasks/create/file", methods=["POST"]) +@app.route("/v1/tasks/create/file", methods=["POST"]) +def tasks_create_file(): + data = request.files["file"] + package = request.form.get("package", "") + timeout = request.form.get("timeout", "") + priority = request.form.get("priority", 1) + options = request.form.get("options", "") + machine = request.form.get("machine", "") + platform = request.form.get("platform", "") + tags = request.form.get("tags", None) + custom = request.form.get("custom", "") + owner = request.form.get("owner", "") + clock = request.form.get("clock", None) + + memory = parse_bool(request.form.get("memory", 0)) + unique = parse_bool(request.form.get("unique", 0)) + enforce_timeout = parse_bool(request.form.get("enforce_timeout", 0)) + + content = data.read() + if unique and db.find_sample(sha256=hashlib.sha256(content).hexdigest()): + return json_error(400, "This file has already been submitted") + + temp_file_path = Files.temp_named_put(content, data.filename) + + task_id = db.add_path( + file_path=temp_file_path, + package=package, + timeout=timeout, + priority=priority, + options=options, + machine=machine, + platform=platform, + tags=tags, + custom=custom, + owner=owner, + memory=memory, + enforce_timeout=enforce_timeout, + clock=clock + ) + + return jsonify(task_id=task_id) + +@app.route("/tasks/create/url", methods=["POST"]) +@app.route("/v1/tasks/create/url", methods=["POST"]) +def tasks_create_url(): + url = request.form.get("url") + package = request.form.get("package", "") + timeout = request.form.get("timeout", "") + priority = request.form.get("priority", 1) + options = request.form.get("options", "") + machine = request.form.get("machine", "") + platform = request.form.get("platform", "") + tags = request.form.get("tags", None) + custom = request.form.get("custom", "") + owner = request.form.get("owner", "") + + memory = request.form.get("memory", False) + if memory: + memory = True + + enforce_timeout = request.form.get("enforce_timeout", False) + if enforce_timeout: + enforce_timeout = True + + clock = request.form.get("clock", None) + + task_id = db.add_url( + url=url, + package=package, + timeout=timeout, + options=options, + priority=priority, + machine=machine, + platform=platform, + tags=tags, + custom=custom, + owner=owner, + memory=memory, + enforce_timeout=enforce_timeout, + clock=clock + ) + + return jsonify(task_id=task_id) + +@app.route("/tasks/create/submit", methods=["POST"]) +@app.route("/v1/tasks/create/submit", methods=["POST"]) +def tasks_create_submit(): + files = [] + for f in request.files.getlist("file") + request.files.getlist("files"): + files.append({ + # The pseudo-file "f" has a read() method so passing it along to + # the Submit Manager as-is should be fine. + "name": f.filename, "data": f, + }) + + if files: + submit_type = "files" + elif request.form.get("strings"): + submit_type = "strings" + strings = request.form["strings"].split("\n") + else: + return json_error(500, "No files or strings have been given!") + + # Default options. + options = { + "procmemdump": "yes", + } + options.update(parse_options(request.form.get("options", ""))) + + submit_id = sm.pre( + submit_type, files or strings, sm.translate_options_to(options) + ) + if not submit_id: + return json_error(500, "Error creating Submit entry") + + files, errors, options = sm.get_files(submit_id, astree=True) + + options["full-memory-dump"] = parse_bool( + request.form.get("memory", config("cuckoo:cuckoo:memory_dump")) + ) + options["enforce-timeout"] = parse_bool( + request.form.get("enforce_timeout", 0) + ) + + def selected(files, arcname=None): + ret = [] + for entry in files: + if entry.get("selected"): + entry["arcname"] = arcname + ret.append(entry) + ret += selected(entry["children"], arcname or entry["filename"]) + return ret + + task_ids = sm.submit(submit_id, { + "global": { + "timeout": request.form.get("timeout", ""), + "priority": request.form.get("priority", 1), + "tags": request.form.get("tags", None), + "custom": request.form.get("custom", ""), + "owner": request.form.get("owner", ""), + "clock": request.form.get("clock", None), + "options": options, + }, + "file_selection": selected(files), + }) + return jsonify(submit_id=submit_id, task_ids=task_ids, errors=errors) + +@app.route("/tasks/list") +@app.route("/v1/tasks/list") +@app.route("/tasks/list/") +@app.route("/v1/tasks/list/") +@app.route("/tasks/list//") +@app.route("/v1/tasks/list//") +@app.route("/tasks/sample/") +@app.route("/v1/tasks/sample/") +def tasks_list(limit=None, offset=None, sample_id=None): + response = {} + + response["tasks"] = [] + + completed_after = request.args.get("completed_after") + if completed_after: + completed_after = datetime.datetime.fromtimestamp( + int(completed_after) + ) + + owner = request.args.get("owner") + status = request.args.get("status") + + tasks = db.list_tasks( + limit=limit, details=True, offset=offset, + completed_after=completed_after, owner=owner, + status=status, sample_id=sample_id, + order_by=Task.completed_on.asc() + ) + + for row in tasks: + task = row.to_dict(dt=True) + + task["guest"] = {} + if row.guest: + task["guest"] = row.guest.to_dict() + + task["errors"] = [] + for error in row.errors: + task["errors"].append(error.message) + + task["sample"] = {} + if row.sample_id: + sample = db.view_sample(row.sample_id) + task["sample"] = sample.to_dict() + + response["tasks"].append(task) + + return jsonify(response) + +@app.route("/tasks/view/") +@app.route("/v1/tasks/view/") +def tasks_view(task_id): + response = {} + + task = db.view_task(task_id, details=True) + if not task: + return json_error(404, "Task not found") + + entry = task.to_dict() + entry["guest"] = {} + if task.guest: + entry["guest"] = task.guest.to_dict() + + entry["errors"] = [] + for error in task.errors: + entry["errors"].append(error.message) + + entry["sample"] = {} + if task.sample_id: + sample = db.view_sample(task.sample_id) + entry["sample"] = sample.to_dict() + + response["task"] = entry + return jsonify(response) + +@app.route("/tasks/reschedule/") +@app.route("/tasks/reschedule//") +@app.route("/v1/tasks/reschedule/") +@app.route("/v1/tasks/reschedule//") +def tasks_reschedule(task_id, priority=None): + response = {} + + if not db.view_task(task_id): + return json_error(404, "There is no analysis with the specified ID") + + new_task_id = db.reschedule(task_id, priority) + if not new_task_id: + return json_error( + 500, "An error occurred while trying to reschedule the task" + ) + + response["status"] = "OK" + response["task_id"] = new_task_id + return jsonify(response) + +@app.route("/tasks/delete/") +@app.route("/v1/tasks/delete/") +def tasks_delete(task_id): + response = {} + + task = db.view_task(task_id) + if not task: + return json_error(404, "Task not found") + + if task.status == TASK_RUNNING: + return json_error( + 500, "The task is currently being processed, cannot delete" + ) + + if not db.delete_task(task_id): + return json_error( + 500, "An error occurred while trying to delete the task" + ) + + Folders.delete(cwd("storage", "analyses", "%d" % task_id)) + response["status"] = "OK" + return jsonify(response) + +@app.route("/tasks/report/") +@app.route("/v1/tasks/report/") +@app.route("/tasks/report//") +@app.route("/v1/tasks/report//") +def tasks_report(task_id, report_format="json"): + formats = { + "json": "report.json", + "html": "report.html", + } + + bz_formats = { + "all": {"type": "-", "files": ["memory.dmp"]}, + "dropped": {"type": "+", "files": ["files"]}, + "package_files": {"type": "+", "files": ["package_files"]}, + } + + tar_formats = { + "bz2": "w:bz2", + "gz": "w:gz", + "tar": "w", + } + + if report_format.lower() in formats: + report_path = cwd( + "storage", "analyses", "%d" % task_id, "reports", + formats[report_format.lower()] + ) + elif report_format.lower() in bz_formats: + bzf = bz_formats[report_format.lower()] + srcdir = cwd("storage", "analyses", "%d" % task_id) + s = io.BytesIO() + + # By default go for bz2 encoded tar files (for legacy reasons). + tarmode = tar_formats.get(request.args.get("tar"), "w:bz2") + + tar = tarfile.open(fileobj=s, mode=tarmode, dereference=True) + for filedir in os.listdir(srcdir): + filepath = os.path.join(srcdir, filedir) + if not os.path.exists(filepath): + continue + if bzf["type"] == "-" and filedir not in bzf["files"]: + tar.add(filepath, arcname=filedir) + if bzf["type"] == "+" and filedir in bzf["files"]: + tar.add(filepath, arcname=filedir) + tar.close() + + response = make_response(s.getvalue()) + response.headers["Content-Type"] = \ + "application/x-tar; charset=UTF-8" + return response + else: + return json_error(400, "Invalid report format") + + if not os.path.exists(report_path): + return json_error(404, "Report not found") + + elements = request.args.get("elements") + if report_format.lower() == "json": + report_content = open(report_path, "rb").read() + if elements is not None: + elements_content = json.loads(report_content).get(elements) + if elements_content is None: + return json_error(404, "'{0}' not found".format(elements)) + else: + response = make_response(json.dumps(elements_content)) + response.headers["Content-Type"] = "application/json" + return response + + response = make_response(report_content) + response.headers["Content-Type"] = "application/json" + return response + else: + if elements is not None: + return json_error(404, "Get specific field is not available in HTML format,"\ + " try again with JSON format") + return open(report_path, "rb").read() + +@app.route("/tasks/summary/") +def tasks_summary(task_id): + report_path = cwd("reports", "report.json", analysis=task_id) + if not os.path.exists(report_path): + return json_error(404, "Report not found") + + with open(report_path, "rb") as report_file: + report = json.load(report_file) + + try: + del report["procmemory"] + del report["behavior"]["generic"] + del report["behavior"]["apistats"] + del report["behavior"]["processes"] + del report["debug"] + del report["screenshots"] + del report["metadata"] + except KeyError: + pass + + return jsonify(report) + +@app.route("/tasks/screenshots/") +@app.route("/v1/tasks/screenshots/") +@app.route("/tasks/screenshots//") +@app.route("/v1/tasks/screenshots//") +def task_screenshots(task_id=0, screenshot=None): + folder_path = cwd("storage", "analyses", "%s" % task_id, "shots") + + if not os.path.exists(folder_path): + return json_error(404, "Task not found") + + if screenshot: + screenshot_name = "%s.jpg" % screenshot + screenshot_path = os.path.join(folder_path, screenshot_name) + if not os.path.exists(screenshot_path): + return json_error(404, "Screenshot not found!") + + # TODO: Add content disposition. + response = make_response(open(screenshot_path, "rb").read()) + response.headers["Content-Type"] = "image/jpeg" + return response + else: + zip_data = io.BytesIO() + with zipfile.ZipFile(zip_data, "w", zipfile.ZIP_STORED) as zip_file: + for shot_name in os.listdir(folder_path): + zip_file.write(os.path.join(folder_path, shot_name), shot_name) + + # TODO: Add content disposition. + response = make_response(zip_data.getvalue()) + response.headers["Content-Type"] = "application/zip" + return response + +@app.route("/tasks/rereport/") +def rereport(task_id): + task = db.view_task(task_id) + if not task: + return json_error(404, "Task not found") + + if task.status == TASK_REPORTED: + db.set_status(task_id, TASK_COMPLETED) + return jsonify(success=True) + + return jsonify(success=False) + +@app.route("/tasks/reboot/") +def reboot(task_id): + reboot_id = Database().add_reboot(task_id=task_id) + if not reboot_id: + return json_error(404, "Error creating reboot task") + + return jsonify(task_id=task_id, reboot_id=reboot_id) + +@app.route("/files/view/md5/") +@app.route("/v1/files/view/md5/") +@app.route("/files/view/sha256/") +@app.route("/v1/files/view/sha256/") +@app.route("/files/view/id/") +@app.route("/v1/files/view/id/") +def files_view(md5=None, sha256=None, sample_id=None): + response = {} + + if md5: + sample = db.find_sample(md5=md5) + elif sha256: + sample = db.find_sample(sha256=sha256) + elif sample_id: + sample = db.view_sample(sample_id) + else: + return json_error(400, "Invalid lookup term") + + if not sample: + return json_error(404, "File not found") + + tasks = sorted( + list(map(lambda t: t.id, db.list_tasks(sample_id=sample.id))) + ) + response["sample"] = sample.to_dict() + response["sample"]["tasks"] = tasks + return jsonify(response) + +@app.route("/files/get/") +@app.route("/v1/files/get/") +def files_get(sha256): + file_path = cwd("storage", "binaries", sha256) + if not os.path.exists(file_path): + return json_error(404, "File not found") + + response = make_response(open(file_path, "rb").read()) + response.headers["Content-Type"] = \ + "application/octet-stream; charset=UTF-8" + return response + +@app.route("/pcap/get/") +@app.route("/v1/pcap/get/") +def pcap_get(task_id): + file_path = cwd("storage", "analyses", "%s" % task_id, "dump.pcap") + if not os.path.exists(file_path): + return json_error(404, "File not found") + + try: + # TODO This could be a big file, so eventually we have to switch + # to app.send_static_file() instead. + response = make_response(open(file_path, "rb").read()) + response.headers["Content-Type"] = \ + "application/octet-stream; charset=UTF-8" + return response + except: + return json_error(500, "An error occurred while reading PCAP") + +@app.route("/machines/list") +@app.route("/v1/machines/list") +def machines_list(): + response = {} + + machines = db.list_machines() + + response["machines"] = [] + for row in machines: + response["machines"].append(row.to_dict()) + + return jsonify(response) + +@app.route("/machines/view/") +@app.route("/v1/machines/view/") +def machines_view(name=None): + response = {} + + machine = db.view_machine(name=name) + if not machine: + return json_error(404, "Machine not found") + + response["machine"] = machine.to_dict() + return jsonify(response) + +@app.route("/cuckoo/status") +@app.route("/v1/cuckoo/status") +def cuckoo_status(): + # In order to keep track of the diskspace statistics of the temporary + # directory we create a temporary file so we can statvfs() on that. + temp_file = Files.temp_put("") + + paths = dict( + binaries=cwd("storage", "binaries"), + analyses=cwd("storage", "analyses"), + temporary=temp_file, + ) + + diskspace = {} + for key, path in paths.items(): + if hasattr(os, "statvfs") and os.path.isdir(path): + stats = os.statvfs(path) + diskspace[key] = dict( + free=stats.f_bavail * stats.f_frsize, + total=stats.f_blocks * stats.f_frsize, + used=(stats.f_blocks - stats.f_bavail) * stats.f_frsize, + ) + + # Now we remove the temporary file and its parent directory. + os.unlink(temp_file) + + # Get the CPU load. + if hasattr(os, "getloadavg"): + cpuload = os.getloadavg() + else: + cpuload = [] + + if os.path.isfile("/proc/meminfo"): + values = {} + for line in open("/proc/meminfo"): + key, value = line.split(":", 1) + values[key.strip()] = value.replace("kB", "").strip() + + if "MemAvailable" in values and "MemTotal" in values: + memavail = int(values["MemAvailable"]) + memtotal = int(values["MemTotal"]) + memory = 100 - 100.0 * memavail / memtotal + else: + memory = memavail = memtotal = None + else: + memory = memavail = memtotal = None + + try: + cpu_core_count = multiprocessing.cpu_count() + except NotImplementedError: + cpu_core_count = None + + response = dict( + version=version, + hostname=socket.gethostname(), + machines=dict( + total=len(db.list_machines()), + available=db.count_machines_available() + ), + tasks=dict( + total=db.count_tasks(), + pending=db.count_tasks("pending"), + running=db.count_tasks("running"), + completed=db.count_tasks("completed"), + reported=db.count_tasks("reported") + ), + diskspace=diskspace, + cpuload=cpuload, + cpu_count=cpu_core_count, + memory=memory, + memavail=memavail, + memtotal=memtotal, + processes=Pidfile.get_active_pids() + ) + + return jsonify(response) + +@app.route("/memory/list/") +def memorydumps_list(task_id): + folder_path = cwd("storage", "analyses", "%s" % task_id, "memory") + + if not os.path.exists(folder_path): + return json_error(404, "Memory dump not found") + + memory_files = [] + for subdir, dirs, files in os.walk(folder_path): + for filename in files: + memory_files.append(filename.replace(".dmp", "")) + + if not memory_files: + return json_error(404, "Memory dump not found") + + return jsonify({"dump_files": memory_files}) + +@app.route("/memory/get//") +def memorydumps_get(task_id, pid=None): + folder_path = cwd("storage", "analyses", "%s" % task_id, "memory") + + if not os.path.exists(folder_path) or not pid: + return json_error(404, "Memory dump not found") + + pid_path = os.path.join(folder_path, "%s.dmp" % pid) + if not os.path.exists(pid_path): + return json_error(404, "Memory dump not found") + + response = make_response(open(pid_path, "rb").read()) + response.headers["Content-Type"] = \ + "application/octet-stream; charset=UTF-8" + return response + +@app.route("/vpn/status") +def vpn_status(): + status = rooter("vpn_status") + if status is None: + return json_error(500, "Rooter not available") + + return jsonify({"vpns": status}) + +@app.route("/exit") +def exit_api(): + """Shut down the server if in debug mode and + using the werkzeug server.""" + if not app.debug: + return json_error(403, "This call can only be used in debug mode") + + if not shutdown_server(): + return json_error( + 500, "Shutdown only possible if using werkzeug server" + ) + else: + return jsonify(message="Server stopped") + +@app.errorhandler(401) +def api_auth_required(error): + return json_error( + 401, "Authentication in the form of an " + "'Authorization: Bearer ' header is required" + ) + +@app.before_request +def check_authentication(): + token = config("cuckoo:cuckoo:api_token") + if token: + expect = "Bearer " + token + auth = request.headers.get("Authorization") + if not constant_time_compare(auth, expect): + abort(401) + +def cuckoo_api(hostname, port, debug): + if not config("cuckoo:cuckoo:api_token"): + log.warning( + "It is strongly recommended to enable API authentication to " + "protect against unauthorized access and CSRF attacks." + ) + log.warning("Please check the API documentation for more information.") + app.run(host=hostname, port=port, debug=debug) + +if os.environ.get("CUCKOO_APP") == "api": + from cuckoo.core.startup import ensure_tmpdir, init_console_logging + decide_cwd(exists=True) + Database().connect() + init_console_logging() + ensure_tmpdir() diff --git a/cuckoo/apps/apps.py b/cuckoo/apps/apps.py new file mode 100644 index 0000000000..545cf00260 --- /dev/null +++ b/cuckoo/apps/apps.py @@ -0,0 +1,597 @@ +# Copyright (C) 2016-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import datetime +import fnmatch +import hashlib +import io +import logging +import os +import random +import requests +import shutil +import subprocess +import sys +import tarfile +import time + +from cuckoo.common.colors import bold, red, yellow +from cuckoo.common.config import config, emit_options, Config +from cuckoo.common.elastic import elastic +from cuckoo.common.exceptions import ( + CuckooOperationalError, CuckooDatabaseError, CuckooDependencyError +) +from cuckoo.common.mongo import mongo +from cuckoo.common.objects import Dictionary, File +from cuckoo.common.utils import to_unicode +from cuckoo.core.database import ( + Database, TASK_FAILED_PROCESSING, TASK_REPORTED +) +from cuckoo.core.init import write_cuckoo_conf +from cuckoo.core.log import task_log_start, task_log_stop, logger +from cuckoo.core.plugins import RunProcessing, RunSignatures, RunReporting +from cuckoo.core.startup import init_console_logging +from cuckoo.misc import cwd, mkdir + +log = logging.getLogger(__name__) + +URL = "https://github.com/cuckoosandbox/community/archive/%s.tar.gz" + +def fetch_community(branch="master", force=False, filepath=None): + if filepath: + buf = open(filepath, "rb").read() + else: + log.info("Downloading.. %s", URL % branch) + r = requests.get(URL % branch) + if r.status_code != 200: + raise CuckooOperationalError( + "Error fetching the Cuckoo Community binaries " + "(status_code: %d)!" % r.status_code + ) + + buf = r.content + + t = tarfile.TarFile.open(fileobj=io.BytesIO(buf), mode="r:gz") + + folders = { + "modules/signatures": "signatures", + "data/monitor": "monitor", + "data/yara": "yara", + "agent": "agent", + "analyzer": "analyzer", + } + + members = t.getmembers() + + directory = members[0].name.split("/")[0] + for tarfolder, outfolder in folders.items(): + mkdir(cwd(outfolder)) + + # E.g., "community-master/modules/signatures". + name_start = "%s/%s" % (directory, tarfolder) + for member in members: + if not member.name.startswith(name_start) or \ + name_start == member.name: + continue + + filepath = cwd(outfolder, member.name[len(name_start)+1:]) + if member.isdir(): + mkdir(filepath) + continue + + # TODO Ask for confirmation as we used to do. + if os.path.exists(filepath) and not force: + log.debug( + "Not overwriting file which already exists: %s", + member.name[len(name_start)+1:] + ) + continue + + if member.issym(): + t.makelink(member, filepath) + continue + + if not os.path.exists(os.path.dirname(filepath)): + os.makedirs(os.path.dirname(filepath)) + + log.debug("Extracted %s..", member.name[len(name_start)+1:]) + open(filepath, "wb").write(t.extractfile(member).read()) + +def enumerate_files(path, pattern): + """Yield all filepaths from a directory.""" + if os.path.isfile(path): + yield path + elif os.path.isdir(path): + for dirname, dirnames, filenames in os.walk(path): + for filename in filenames: + filepath = os.path.join(dirname, filename) + + if os.path.isfile(filepath): + if pattern: + if fnmatch.fnmatch(filename, pattern): + yield to_unicode(filepath) + else: + yield to_unicode(filepath) + +def submit_tasks(target, options, package, custom, owner, timeout, priority, + machine, platform, memory, enforce_timeout, clock, tags, + remote, pattern, maxcount, is_unique, is_url, is_baseline, + is_shuffle): + db = Database() + + data = dict( + package=package or "", + timeout=timeout, + options=options, + priority=priority, + machine=machine, + platform=platform, + custom=custom, + owner=owner, + tags=tags, + memory="1" if memory else "0", + enforce_timeout="1" if enforce_timeout else "0", + clock=clock, + unique="1" if is_unique else "0", + ) + + if is_baseline: + if remote: + print "Remote baseline support has not yet been implemented." + return + + task_id = db.add_baseline(timeout, owner, machine, memory) + yield "Baseline", machine, task_id + return + + if is_url and is_unique: + print "URL doesn't have --unique support yet." + return + + if is_url: + for url in target: + if not remote: + data.pop("unique", None) + task_id = db.add_url(to_unicode(url), **data) + yield "URL", url, task_id + continue + + data["url"] = to_unicode(url) + try: + r = requests.post( + "http://%s/tasks/create/url" % remote, data=data + ) + yield "URL", url, r.json()["task_id"] + except Exception as e: + print "%s: unable to submit URL: %s" % ( + bold(red("Error")), e + ) + else: + files = [] + for path in target: + files.extend(enumerate_files(os.path.abspath(path), pattern)) + + if is_shuffle: + random.shuffle(files) + + for filepath in files: + if not os.path.getsize(filepath): + print "%s: sample %s (skipping file)" % ( + bold(yellow("Empty")), filepath + ) + continue + + if maxcount is not None: + if not maxcount: + break + maxcount -= 1 + + if not remote: + if is_unique: + sha256 = File(filepath).get_sha256() + if db.find_sample(sha256=sha256): + yield "File", filepath, None + continue + + data.pop("unique", None) + task_id = db.add_path(file_path=filepath, **data) + yield "File", filepath, task_id + continue + + files = { + "file": (os.path.basename(filepath), open(filepath, "rb")), + } + + try: + r = requests.post( + "http://%s/tasks/create/file" % remote, + data=data, files=files + ) + yield "File", filepath, r.json()["task_id"] + except Exception as e: + print "%s: unable to submit file: %s" % ( + bold(red("Error")), e + ) + continue + +def process(target, copy_path, task): + results = RunProcessing(task=task).run() + RunSignatures(results=results).run() + RunReporting(task=task, results=results).run() + + if config("cuckoo:cuckoo:delete_original"): + try: + if target and os.path.exists(target): + os.remove(target) + except OSError as e: + log.error( + "Unable to delete original file at path \"%s\": %s", + target, e + ) + + if config("cuckoo:cuckoo:delete_bin_copy"): + try: + if copy_path and os.path.exists(copy_path): + os.remove(copy_path) + except OSError as e: + log.error( + "Unable to delete the copy of the original file at " + "path \"%s\": %s", copy_path, e + ) + +def process_task(task): + db = Database() + + try: + task_log_start(task["id"]) + + logger( + "Starting task reporting", + action="task.report", status="pending", + target=task["target"], category=task["category"], + package=task["package"], options=emit_options(task["options"]), + custom=task["custom"] + ) + + if task["category"] == "file" and task.get("sample_id"): + sample = db.view_sample(task["sample_id"]) + copy_path = cwd("storage", "binaries", sample.sha256) + else: + copy_path = None + + try: + process(task["target"], copy_path, task) + db.set_status(task["id"], TASK_REPORTED) + except Exception as e: + log.exception("Task #%d: error reporting: %s", task["id"], e) + db.set_status(task["id"], TASK_FAILED_PROCESSING) + + log.info("Task #%d: reports generation completed", task["id"], extra={ + "action": "task.report", "status": "success", + }) + except Exception as e: + log.exception("Caught unknown exception: %s", e) + finally: + task_log_stop(task["id"]) + +def process_task_range(tasks): + db, task_ids = Database(), [] + for entry in tasks.split(","): + if entry.isdigit(): + task_ids.append(int(entry)) + elif entry.count("-") == 1: + start, end = entry.split("-") + if not start.isdigit() or not end.isdigit(): + log.warning("Invalid range provided: %s", entry) + continue + task_ids.extend(range(int(start), int(end)+1)) + elif entry: + log.warning("Invalid range provided: %s", entry) + + for task_id in sorted(set(task_ids)): + task = db.view_task(task_id) + if not task: + task = { + "id": task_id, + "category": "file", + "target": "", + "options": {}, + "package": None, + "custom": None, + } + else: + task = task.to_dict() + + if os.path.isdir(cwd(analysis=task_id)): + process_task(Dictionary(task)) + +def process_check_stop(count, maxcount, endtime): + """Check if we need to stop processing. + Options passed by maxcount (-m) or calculated endtime (-t) + """ + if maxcount and count >= maxcount: + return False + + if endtime and int(time.time()) > endtime: + return False + + return True + +def process_tasks(instance, maxcount, timeout): + count = 0 + endtime = 0 + db = Database() + + if timeout: + endtime = int(time.time() + timeout) + + try: + while process_check_stop(count, maxcount, endtime): + task_id = db.processing_get_task(instance) + + # Wait a small while before trying to fetch a new task. + if task_id is None: + time.sleep(1) + continue + + task = db.view_task(task_id) + + log.info("Task #%d: reporting task", task.id) + + process_task(task.to_dict()) + count += 1 + except Exception as e: + log.exception("Caught unknown exception: %s", e) + +def cuckoo_clean(): + """Clean up cuckoo setup. + Delete logs, all stored data from file system and configured + databases (SQL and MongoDB). + """ + # Init logging (without writing to file). + init_console_logging() + + try: + # Initialize the database connection. + db = Database() + db.connect(schema_check=False) + + # Drop all tables. + db.drop() + except (CuckooDependencyError, CuckooDatabaseError) as e: + # If something is screwed due to incorrect database migrations or bad + # database SqlAlchemy would be unable to connect and operate. + log.warning("Error connecting to database: it is suggested to check " + "the connectivity, apply all migrations if needed or purge " + "it manually. Error description: %s", e) + + # Check if MongoDB reporting is enabled and drop the database if it is. + if mongo.init(): + try: + mongo.connect() + mongo.drop() + mongo.close() + except Exception as e: + log.warning("Unable to drop MongoDB database: %s", e) + + # Check if ElasticSearch reporting is enabled and drop its data if it is. + if elastic.init(): + elastic.connect() + + # TODO This should be moved to the elastic abstract. + # TODO We should also drop historic data, i.e., from pervious days, + # months, and years. + date_index = datetime.datetime.utcnow().strftime({ + "yearly": "%Y", + "monthly": "%Y-%m", + "daily": "%Y-%m-%d", + }[elastic.index_time_pattern]) + dated_index = "%s-%s" % (elastic.index, date_index) + + elastic.client.indices.delete( + index=dated_index, ignore=[400, 404] + ) + + template_name = "%s_template" % dated_index + if elastic.client.indices.exists_template(template_name): + elastic.client.indices.delete_template(template_name) + + # Paths to clean. + paths = [ + cwd("cuckoo.db"), + cwd("log"), + cwd("storage", "analyses"), + cwd("storage", "baseline"), + cwd("storage", "binaries"), + ] + + # Delete the various files and directories. In case of directories, keep + # the parent directories, so to keep the state of the CWD in tact. + for path in paths: + if os.path.isdir(path): + try: + shutil.rmtree(path) + os.mkdir(path) + except (IOError, OSError) as e: + log.warning("Error removing directory %s: %s", path, e) + elif os.path.isfile(path): + try: + os.unlink(path) + except (IOError, OSError) as e: + log.warning("Error removing file %s: %s", path, e) + +def cuckoo_machine(vmname, action, ip, platform, options, tags, + interface, snapshot, resultserver): + db = Database() + + cfg = Config.from_confdir(cwd("conf")) + machinery = cfg["cuckoo"]["cuckoo"]["machinery"] + machines = cfg[machinery][machinery]["machines"] + + if action == "add": + if not ip: + sys.exit("You have to specify a legitimate IP address for --add.") + + if db.view_machine(vmname): + sys.exit("A Virtual Machine with this name already exists!") + + if vmname in machines: + sys.exit("A Virtual Machine with this name already exists!") + + if resultserver and resultserver.count(":") == 1: + resultserver_ip, resultserver_port = resultserver.split(":") + resultserver_port = int(resultserver_port) + else: + resultserver_ip = cfg["cuckoo"]["resultserver"]["ip"] + resultserver_port = 0 + + machines.append(vmname) + cfg[machinery][vmname] = { + "label": vmname, + "platform": platform, + "ip": ip, + "options": options, + "snapshot": snapshot, + "interface": interface, + "resultserver_ip": resultserver_ip, + "resultserver_port": resultserver_port, + "tags": tags, + } + + db.add_machine( + vmname, vmname, ip, platform, options, tags, interface, snapshot, + resultserver_ip, int(resultserver_port) + ) + db.unlock_machine(vmname) + + if action == "delete": + # TODO Add a db.del_machine() function for runtime modification. + + if vmname not in machines: + sys.exit("A Virtual Machine with this name doesn't exist!") + + machines.remove(vmname) + cfg[machinery].pop(vmname) + + write_cuckoo_conf(cfg=cfg) + +def migrate_database(revision="head"): + args = [ + "alembic", "-x", "cwd=%s" % cwd(), "upgrade", revision, + ] + try: + subprocess.check_call(args, cwd=cwd("db_migration", private=True)) + except subprocess.CalledProcessError: + return False + return True + +def migrate_cwd(): + log.warning( + "This is the first time you're running Cuckoo after updating your " + "local version of Cuckoo. We're going to update files in your CWD " + "that require updating. Note that we'll first ensure that no custom " + "patches have been applied by you before applying any modifications " + "of our own." + ) + + # Remove now-obsolete index_*.yar files. + for filename in os.listdir(cwd("yara")): + if filename.startswith("index_") and filename.endswith(".yar"): + os.remove(cwd("yara", filename)) + + # Create new directories if not present yet. + mkdir(cwd("stuff")) + mkdir(cwd("yara", "office")) + + # Create the new $CWD/safelist/ directory. + if not os.path.exists(cwd("safelist")): + shutil.copytree( + cwd("..", "data", "safelist", private=True), cwd("safelist") + ) + else: + data_wl = cwd("..", "data", "safelist", private=True) + for wl_file in os.listdir(data_wl): + cwd_wl = cwd("safelist", wl_file) + if not os.path.isfile(cwd_wl): + shutil.copy(os.path.join(data_wl, wl_file), cwd_wl) + + # Create the new $CWD/yara/dumpmem/ directory. + if not os.path.exists(cwd("yara", "dumpmem")): + mkdir(cwd("yara", "dumpmem")) + + hashes = {} + for line in open(cwd("cwd", "hashes.txt", private=True), "rb"): + if not line.strip() or line.startswith("#"): + continue + hash_, filename = line.split() + hashes[filename] = hashes.get(filename, []) + [hash_] + + # We remove $CWD/monitor/latest upfront if it's a symbolic link, because + # our migration code doesn't properly handle symbolic links. + if os.path.islink(cwd("monitor", "latest")): + os.remove(cwd("monitor", "latest")) + + modified, outdated, deleted = [], [], [] + for filename, hashes in hashes.items(): + if not os.path.exists(cwd(filename)): + if hashes[-1] != "0"*40: + outdated.append(filename) + continue + hash_ = hashlib.sha1(open(cwd(filename), "rb").read()).hexdigest() + if hash_ not in hashes: + modified.append(filename) + elif hashes[-1] == "0"*40: + deleted.append(filename) + elif hash_ != hashes[-1]: + outdated.append(filename) + + if modified: + log.error( + "One or more files in the CWD have been modified outside of " + "regular Cuckoo usage. Due to these changes Cuckoo isn't able to " + "automatically upgrade your setup." + ) + + for filename in sorted(modified): + log.warning("Modified file: %s (=> %s)", filename, cwd(filename)) + + log.error("Moving forward you have two options:") + log.warning( + "1) You make a backup of the affected files, remove their " + "presence in the CWD (yes, actually 'rm -f' the file), and " + "re-run Cuckoo to automatically restore the new version of the " + "file. Afterwards you'll be able to re-apply any changes as you " + "like." + ) + log.warning( + "2) You revert back to the version of Cuckoo you were on " + "previously and accept that manual changes that have not been " + "merged upstream require additional maintenance that you'll " + "pick up at a later point in time." + ) + + sys.exit(1) + + for filename in sorted(deleted): + log.debug("Deleted %s", filename) + os.unlink(cwd(filename)) + + for filename in sorted(outdated): + filepath = cwd("..", "data", filename, private=True) + if not os.path.exists(filepath): + log.debug( + "Failed to upgrade file not shipped with this release: %s", + filename + ) + continue + + log.debug("Upgraded %s", filename) + if not os.path.exists(os.path.dirname(cwd(filename))): + os.makedirs(os.path.dirname(cwd(filename))) + shutil.copy(filepath, cwd(filename)) + + log.info( + "Automated migration of your CWD was successful! Continuing " + "execution of Cuckoo as expected." + ) diff --git a/cuckoo/apps/distributed.py b/cuckoo/apps/distributed.py new file mode 100644 index 0000000000..16e78b4d94 --- /dev/null +++ b/cuckoo/apps/distributed.py @@ -0,0 +1,32 @@ +# Copyright (C) 2016-2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import os + +from cuckoo.distributed.app import create_app +from cuckoo.distributed.instance import ( + scheduler, status_caching, handle_node +) +from cuckoo.misc import decide_cwd + +app = None + +def cuckoo_distributed(hostname, port, debug): + app = create_app() + app.run(host=hostname, port=port, debug=debug) + +def cuckoo_distributed_instance(name): + app = create_app() + + with app.app_context(): + if name == "dist.scheduler": + scheduler() + elif name == "dist.status": + status_caching() + else: + handle_node(name) + +if os.environ.get("CUCKOO_APP") == "dist": + decide_cwd(exists=True) + app = create_app() diff --git a/cuckoo/apps/dnsserve.py b/cuckoo/apps/dnsserve.py new file mode 100644 index 0000000000..885755fb90 --- /dev/null +++ b/cuckoo/apps/dnsserve.py @@ -0,0 +1,68 @@ +# Copyright (C) 2016-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import logging +import socket +import sys + +logging.getLogger("scapy.loading").setLevel(logging.ERROR) +log = logging.getLogger(__name__) + +def cuckoo_dnsserve(host, port, nxdomain, hardcode): + try: + from scapy.layers.dns import DNS, DNSQR, DNSRR + except ImportError: + sys.exit( + "Currently the DNS serve script is not available due to issues " + "in upstream Scapy for Windows " + "(https://github.com/secdev/scapy/issues/111)." + ) + + udps = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + udps.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + udps.bind((host, port)) + log.info("Listening for DNS queries at %s:%d", host, port) + + while True: + data, addr = udps.recvfrom(1024) + + p = DNS(data) + rp = DNS(id=p.id, qr=1, qdcount=p.qdcount, ancount=1, rcode=0) + rp.qd = p[DNSQR] + + # IN A, actually look the domain up. + if p.opcode == 0 and p[DNSQR].qtype == 1 and p[DNSQR].qclass == 1: + if hardcode: + answer_ip = hardcode + else: + try: + answer_ip = socket.gethostbyname(p.qd[0].qname) + except: + if nxdomain: + answer_ip = nxdomain + else: + rp.ancount = 0 + rp.rcode = 3 + answer_ip = None + + if answer_ip: + rp.an = DNSRR( + rrname=p.qd[0].qname, ttl=60, rdlen=4, rdata=answer_ip + ) + + log.debug("IN A %s -> %s.", p.qd[0].qname, answer_ip) + # IN PTR, we reply with NXDOMAIN. + elif p.opcode == 0 and p[DNSQR].qtype == 12 and p[DNSQR].qclass == 1: + rp.ancount = 0 + rp.rcode = 3 + log.info("IN PTR %s -> NXDOMAIN.", p.qd[0].qname) + else: + rp.ancount = 0 + rp.rcode = 2 + log.warn( + "Unhandled query %s for %s/%s,%s - answering with servfail.", + p.opcode, p.qd[0].qname, p[DNSQR].qtype, p[DNSQR].qclass + ) + + udps.sendto(rp.build(), addr) diff --git a/cuckoo/apps/import_.py b/cuckoo/apps/import_.py new file mode 100644 index 0000000000..b8250d0edf --- /dev/null +++ b/cuckoo/apps/import_.py @@ -0,0 +1,227 @@ +# Copyright (C) 2016-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import click +import os.path +import logging +import shutil +import subprocess +import sqlalchemy + +from cuckoo.common.config import Config +from cuckoo.common.colors import yellow, red +from cuckoo.common.exceptions import CuckooOperationalError +from cuckoo.compat.config import migrate as migrate_conf +from cuckoo.misc import cwd, is_windows + +log = logging.getLogger(__name__) + +def identify(dirpath): + filepath = os.path.join(dirpath, "lib", "cuckoo", "common", "constants.py") + if os.path.exists(filepath): + for line in open(filepath, "rb"): + if line.startswith("CUCKOO_VERSION"): + return line.split('"')[1] + +def _dburi_engine(dburi): + # Defaults to a sqlite3 database. + if not dburi: + dburi = "sqlite:///db/cuckoo.db" + + try: + return sqlalchemy.create_engine(dburi).engine + except sqlalchemy.exc.ArgumentError: + raise CuckooOperationalError( + "Error creating SQL database backup as your SQL database URI " + "wasn't understood by us: %r!" % dburi + ) + +def dumpcmd(dburi, dirpath): + engine = _dburi_engine(dburi) + + if engine.name == "sqlite": + # If the SQLite3 database filepath is relative, then make it relative + # against the old Cuckoo setup. If it's absolute, os.path.join() will + # keep it absolute as-is (see also our version 1.1.1 release :-P). + return [ + "sqlite3", os.path.join(dirpath, engine.url.database), ".dump" + ], {} + + if engine.name == "mysql": + args = ["mysqldump"] + if engine.url.username: + args += ["-u", engine.url.username] + if engine.url.password: + args.append("-p%s" % engine.url.password) + if engine.url.host and engine.url.host != "localhost": + args += ["-h", engine.url.host] + args.append(engine.url.database) + return args, {} + + if engine.name == "postgresql": + args, env = ["pg_dump"], {} + if engine.url.username: + args += ["-U", engine.url.username] + if engine.url.password: + env["PGPASSWORD"] = engine.url.password + if engine.url.host and engine.url.host != "localhost": + args += ["-h", engine.url.host] + args.append(engine.url.database) + return args, env + + raise CuckooOperationalError( + "Error creating SQL database backup as your SQL database URI " + "wasn't understood by us: %r!" % dburi + ) + +def movesql(dburi, mode, dirpath): + engine = _dburi_engine(dburi) + if engine.name != "sqlite": + return + + if mode == "copy": + import_file = shutil.copy + elif mode == "move": + import_file = shutil.move + elif mode == "symlink": + if is_windows(): + raise RuntimeError("Can't use 'symlink' mode under Windows!") + import_file = os.symlink + + # For more information on the os.path.join() usage see also dumpcmd(). + import_file( + os.path.abspath(os.path.join(dirpath, engine.url.database)), + cwd("cuckoo.db") + ) + +def sqldump(dburi, dirpath): + args, env = dumpcmd(dburi, dirpath) + + envargs = " ".join("%s=%s" % (k, v) for k, v in env.items()) + cmdline = " ".join('"%s"' % arg if " " in arg else arg for arg in args) + cmd = "%s %s" % (envargs, cmdline) if envargs else cmdline + + print "We can make a SQL database backup as follows:" + print "input cmd =>", cmd + print "output SQL =>", cwd("backup.sql") + + if not click.confirm("Would you like to make a backup", default=True): + return + + try: + subprocess.check_call( + args, stdout=open(cwd("backup.sql"), "wb"), + env=dict(os.environ.items() + env.items()) + ) + except (subprocess.CalledProcessError, OSError) as e: + raise CuckooOperationalError( + "Error creating SQL database dump as the command returned an " + "error code: %s. Please make sure that the required tooling " + "for making a database backup is installed and review the " + "database URI to make sure it's correct: %s!" % (e, dburi) + ) + +def import_cuckoo(username, mode, dirpath): + version = identify(dirpath) + if not version: + raise CuckooOperationalError( + "The path that you specified is not a proper Cuckoo setup. Please " + "point the path to the root of your legacy Cuckoo setup, i.e., to " + "the directory containing the cuckoo.py script!" + ) + + # TODO Copy over the configuration and ignore the database. + if version in ("0.4", "0.4.1", "0.4.2"): + raise CuckooOperationalError( + "Importing from version 0.4, 0.4.1, or 0.4.2 is not supported as " + "there are no database migrations for that version. Please start " + "from scratch, your configuration would have been obsolete anyway!" + ) + + print "We've identified a Cuckoo Sandbox %s installation!" % version + + if os.path.isdir(cwd()) and os.listdir(cwd()): + raise CuckooOperationalError( + "This Cuckoo Working Directory already exists. Please import to " + "a new/clean Cuckoo Working Directory." + ) + + # Following are various recursive imports. + from cuckoo.apps import migrate_database + from cuckoo.main import cuckoo_create + + print "Reading in the old configuration.." + + # Port the older configuration. + cfg = Config.from_confdir(os.path.join(dirpath, "conf"), loose=True) + cfg = migrate_conf(cfg, version) + + print " configuration has been migrated to the latest version!" + print + + # Create a fresh Cuckoo Working Directory. + cuckoo_create(username, cfg, quiet=True) + + dburi = cfg["cuckoo"]["database"]["connection"] + + # Ask if the user would like to make a backup of the SQL database and in + # the case of sqlite3, copy/move/symlink cuckoo.db to the CWD. + sqldump(dburi, dirpath) + movesql(dburi, mode, dirpath) + + # Run database migrations. + if not migrate_database(): + raise CuckooOperationalError( + "Error migrating your old Cuckoo database!" + ) + + # Link or copy all of the older results to the new CWD. + import_legacy_analyses(mode, dirpath) + + # Urge the user to run the community command. + print + print "You have successfully imported your old version of Cuckoo!" + print "However, in order to get up-to-date, you'll probably want to" + print yellow("run the community command"), + print "by running", red("'cuckoo community'"), "manually." + print "The community command will fetch the latest monitoring updates" + print "and Cuckoo Signatures." + +def import_analysis_copy(src, dst): + def ignore(src, names): + if "binary" not in names: + return [] + if not os.path.exists(os.path.join(src, "binary")): + return ["binary"] + return [] + + shutil.copytree(src, dst, ignore=ignore) + +def import_legacy_analyses(mode, dirpath): + """Imports the raw results of a legacy analysis. Using either the 'copy', + 'move', or 'symlink' mode.""" + if mode == "copy": + import_analysis = import_analysis_copy + elif mode == "move": + import_analysis = shutil.move + elif mode == "symlink": + if is_windows(): + raise RuntimeError("Can't use 'symlink' mode under Windows!") + import_analysis = os.symlink + + analyses = os.path.join(dirpath, "storage", "analyses") + if not os.path.isdir(analyses): + log.warning("Didn't find any analyses, so not much to import!") + return + + tasks = [] + for task_id in os.listdir(analyses): + if task_id == "latest": + continue + + import_analysis( + os.path.join(analyses, task_id), cwd(analysis=task_id) + ) + tasks.append(int(task_id)) + return tasks diff --git a/cuckoo/apps/rooter.py b/cuckoo/apps/rooter.py new file mode 100644 index 0000000000..f4d8908e66 --- /dev/null +++ b/cuckoo/apps/rooter.py @@ -0,0 +1,519 @@ +# Copyright (C) 2015-2019 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import errno +import json +import logging +import os.path +import re +import signal +import socket +import stat +import subprocess +import sys + +from cuckoo.common.colors import red +from cuckoo.misc import version as __version__ + +class s(object): + service = None + iptables = None + iptables_save = None + iptables_restore = None + ip = None + +log = logging.getLogger(__name__) + +def run(*args): + """Wrapper to Popen.""" + log.debug("Running command: %s", " ".join(args)) + p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + return stdout, stderr + +def run_iptables(*args): + iptables_args = [s.iptables] + iptables_args.extend(list(args)) + iptables_args.extend(["-m", "comment", "--comment", "cuckoo-rooter"]) + return run(*iptables_args) + +def cleanup_rooter(): + """Filter out all Cuckoo rooter entries from iptables-save and + restore the resulting ruleset.""" + try: + stdout, stderr = run(s.iptables_save) + except OSError as e: + log.error( + "Failed to clean Cuckoo rooter rules. Is iptables-save " + "available? %s", e + ) + return + + if not stdout: + return + + cleaned = [] + for l in stdout.split("\n"): + if l and "cuckoo-rooter" not in l: + cleaned.append(l) + + p = subprocess.Popen([s.iptables_restore], stdin=subprocess.PIPE) + p.communicate(input="\n".join(cleaned)) + +def version(): + return { + "version": __version__, + "features": [], + } + +def nic_available(interface): + """Check if specified network interface is available.""" + if not re.match("[a-zA-Z0-9-._]+$", interface): + return False + + try: + subprocess.check_call([s.ip, "link", "show", interface], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + return True + except subprocess.CalledProcessError: + return False + +def rt_available(rt_table): + """Check if specified routing table is defined.""" + try: + subprocess.check_call([s.ip, "route", "list", "table", rt_table], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + return True + except subprocess.CalledProcessError: + return False + +def vpn_status(): + """Get current VPN status.""" + ret = {} + for line in run(s.service, "openvpn", "status")[0].split("\n"): + x = re.search("'(?P\\w+)'\\ is\\ (?Pnot)?", line) + if x: + ret[x.group("vpn")] = x.group("running") != "not" + + return ret + +def vpn_enable(name): + """Start a VPN.""" + run(s.service, "openvpn", "start", name) + +def vpn_disable(name): + """Stop a running VPN.""" + run(s.service, "openvpn", "stop", name) + +def forward_drop(): + """Disable any and all forwarding unless explicitly said so.""" + run_iptables("-P", "FORWARD", "DROP") + +def state_enable(): + """Enable stateful connection tracking.""" + run_iptables( + "-A", "INPUT", "-m", "state", "--state", "ESTABLISHED,RELATED", + "-j", "ACCEPT" + ) + +def state_disable(): + """Disable stateful connection tracking.""" + while True: + _, err = run_iptables( + "-D", "INPUT", "-m", "state", + "--state", "ESTABLISHED,RELATED", "-j", "ACCEPT" + ) + if err: + break + +def enable_nat(interface): + """Enable NAT on this interface.""" + run_iptables( + "-t", "nat", "-A", "POSTROUTING", "-o", interface, "-j", "MASQUERADE" + ) + +def disable_nat(interface): + """Disable NAT on this interface.""" + while True: + _, err = run_iptables( + "-t", "nat", "-D", "POSTROUTING", + "-o", interface, "-j", "MASQUERADE" + ) + if err: + break + +def init_rttable(rt_table, interface): + """Initialise routing table for this interface using routes + from main table.""" + if rt_table in ["local", "main", "default"]: + return + + stdout, _ = run(s.ip, "route", "list", "dev", interface) + for line in stdout.split("\n"): + args = ["route", "add"] + [x for x in line.split(" ") if x] + args += ["dev", interface, "table", rt_table] + run(s.ip, *args) + +def flush_rttable(rt_table): + """Flush specified routing table entries.""" + if rt_table in ["local", "main", "default"]: + return + + run(s.ip, "route", "flush", "table", rt_table) + +def dns_forward(action, vm_ip, dns_ip, dns_port="53"): + """Route DNS requests from the VM to a custom DNS on a separate network.""" + run_iptables( + "-t", "nat", action, "PREROUTING", "-p", "tcp", + "--dport", "53", "--source", vm_ip, "-j", "DNAT", + "--to-destination", "%s:%s" % (dns_ip, dns_port) + ) + + run_iptables( + "-t", "nat", action, "PREROUTING", "-p", "udp", + "--dport", "53", "--source", vm_ip, "-j", "DNAT", + "--to-destination", "%s:%s" % (dns_ip, dns_port) + ) + +def forward_enable(src, dst, ipaddr): + """Enable forwarding a specific IP address from one interface into + another.""" + # Delete libvirt's default FORWARD REJECT rules. e.g.: + # -A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable + # -A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable + run_iptables("-D", "FORWARD", "-i", src, "-j", "REJECT") + run_iptables("-D", "FORWARD", "-o", src, "-j", "REJECT") + + run_iptables( + "-A", "FORWARD", "-i", src, "-o", dst, + "--source", ipaddr, "-j", "ACCEPT" + ) + + run_iptables( + "-A", "FORWARD", "-i", dst, "-o", src, + "--destination", ipaddr, "-j", "ACCEPT" + ) + +def forward_disable(src, dst, ipaddr): + """Disable forwarding of a specific IP address from one interface into + another.""" + run_iptables( + "-D", "FORWARD", "-i", src, "-o", dst, + "--source", ipaddr, "-j", "ACCEPT" + ) + + run_iptables( + "-D", "FORWARD", "-i", dst, "-o", src, + "--destination", ipaddr, "-j", "ACCEPT" + ) + +def srcroute_enable(rt_table, ipaddr): + """Enable routing policy for specified source IP address.""" + run(s.ip, "rule", "add", "from", ipaddr, "table", rt_table) + run(s.ip, "route", "flush", "cache") + +def srcroute_disable(rt_table, ipaddr): + """Disable routing policy for specified source IP address.""" + run(s.ip, "rule", "del", "from", ipaddr, "table", rt_table) + run(s.ip, "route", "flush", "cache") + +def inetsim_redirect_port(action, srcip, dstip, ports): + """Note that the parameters (probably) mean the opposite of what they + imply; this method adds or removes an iptables rule for redirect traffic + from (srcip, srcport) to (dstip, dstport). + E.g., if 192.168.56.101:80 -> 192.168.56.1:8080, then it redirects + outgoing traffic from 192.168.56.101 to port 80 to 192.168.56.1:8080. + """ + for entry in ports.split(): + if entry.count(":") != 1: + log.debug("Invalid inetsim ports entry: %s", entry) + continue + srcport, dstport = entry.split(":") + if not srcport.isdigit() or not dstport.isdigit(): + log.debug("Invalid inetsim ports entry: %s", entry) + continue + run_iptables( + "-t", "nat", action, "PREROUTING", "--source", srcip, + "-p", "tcp", "--syn", "--dport", srcport, + "-j", "DNAT", "--to-destination", "%s:%s" % (dstip, dstport) + ) + +def inetsim_enable(ipaddr, inetsim_ip, machinery_iface, resultserver_port, + ports): + """Enable hijacking of all traffic and send it to InetSim.""" + inetsim_redirect_port("-A", ipaddr, inetsim_ip, ports) + + run_iptables( + "-t", "nat", "-A", "PREROUTING", "--source", ipaddr, + "-p", "tcp", "--syn", "!", "--dport", resultserver_port, + "-j", "DNAT", "--to-destination", inetsim_ip + ) + + run_iptables( + "-t", "nat", "-A", "PREROUTING", "--source", ipaddr, + "-p", "udp", "-j", "DNAT", "--to-destination", inetsim_ip + ) + + run_iptables( + "-A", "OUTPUT", "-m", "conntrack", "--ctstate", + "INVALID", "-j", "DROP" + ) + + run_iptables( + "-A", "OUTPUT", "-m", "state", "--state", + "INVALID", "-j", "DROP" + ) + + dns_forward("-A", ipaddr, inetsim_ip) + forward_enable(machinery_iface, machinery_iface, ipaddr) + + run_iptables( + "-t", "nat", "-A", "POSTROUTING", "--source", ipaddr, + "-o", machinery_iface, "--destination", inetsim_ip, "-j", "MASQUERADE") + + run_iptables("-A", "OUTPUT", "-s", ipaddr, "-j", "DROP") + +def inetsim_disable(ipaddr, inetsim_ip, machinery_iface, resultserver_port, + ports): + """Enable hijacking of all traffic and send it to InetSim.""" + inetsim_redirect_port("-D", ipaddr, inetsim_ip, ports) + + run_iptables( + "-D", "PREROUTING", "-t", "nat", "--source", ipaddr, + "-p", "tcp", "--syn", "!", "--dport", resultserver_port, "-j", "DNAT", + "--to-destination", inetsim_ip + ) + run_iptables( + "-t", "nat", "-D", "PREROUTING", "--source", ipaddr, + "-p", "udp", "-j", "DNAT", "--to-destination", inetsim_ip + ) + + run_iptables( + "-D", "OUTPUT", "-m", "conntrack", "--ctstate", + "INVALID", "-j", "DROP" + ) + + run_iptables( + "-D", "OUTPUT", "-m", "state", "--state", + "INVALID", "-j", "DROP" + ) + + dns_forward("-D", ipaddr, inetsim_ip) + forward_disable(machinery_iface, machinery_iface, ipaddr) + + run_iptables( + "-t", "nat", "-D", "POSTROUTING", "--source", ipaddr, + "-o", machinery_iface, "--destination", inetsim_ip, "-j", "MASQUERADE") + + run_iptables("-D", "OUTPUT", "-s", ipaddr, "-j", "DROP") + +def tor_toggle(action, vm_ip, resultserver_ip, dns_port, proxy_port): + """Toggle Tor iptables routing rules.""" + dns_forward(action, vm_ip, resultserver_ip, dns_port) + + run_iptables( + "-t", "nat", action, "PREROUTING", "-p", "tcp", + "--source", vm_ip, "!", "--destination", resultserver_ip, + "-j", "DNAT", "--to-destination", + "%s:%s" % (resultserver_ip, proxy_port) + ) + + run_iptables( + "-t", "nat", action, "PREROUTING", "-p", "udp", + "--source", vm_ip, "!", "--destination", resultserver_ip, + "-j", "DNAT", "--to-destination", + "%s:%s" % (resultserver_ip, proxy_port) + ) + run_iptables(action, "OUTPUT", "-s", vm_ip, "-j", "DROP") + +def tor_enable(vm_ip, resultserver_ip, dns_port, proxy_port): + """Enable hijacking of all traffic and send it to TOR.""" + tor_toggle("-A", vm_ip, resultserver_ip, dns_port, proxy_port) + +def tor_disable(vm_ip, resultserver_ip, dns_port, proxy_port): + """Enable hijacking of all traffic and send it to TOR.""" + tor_toggle("-D", vm_ip, resultserver_ip, dns_port, proxy_port) + +def drop_toggle(action, vm_ip, resultserver_ip, resultserver_port, agent_port): + """Toggle iptables to allow internal Cuckoo traffic.""" + run_iptables( + action, "INPUT", "--source", vm_ip, "-p", "tcp", + "--destination", resultserver_ip, "--dport", "%s" % resultserver_port, + "-j", "ACCEPT" + ) + + run_iptables( + action, "OUTPUT", "--source", resultserver_ip, + "-p", "tcp", "--destination", vm_ip, "--dport", "%s" % agent_port, + "-j", "ACCEPT" + ) + + run_iptables(action, "INPUT", "--source", vm_ip, "-j", "DROP") + run_iptables(action, "OUTPUT", "--source", vm_ip, "-j", "DROP") + +def drop_enable(vm_ip, resultserver_ip, resultserver_port, agent_port=8000): + """Enable complete dropping of all non-Cuckoo traffic by default.""" + return drop_toggle( + "-A", vm_ip, resultserver_ip, resultserver_port, agent_port + ) + +def drop_disable(vm_ip, resultserver_ip, resultserver_port, agent_port=8000): + """Disable complete dropping of all non-Cuckoo traffic by default.""" + return drop_toggle( + "-D", vm_ip, resultserver_ip, resultserver_port, agent_port + ) + +handlers = { + "version": version, + "nic_available": nic_available, + "rt_available": rt_available, + "vpn_status": vpn_status, + "vpn_enable": vpn_enable, + "vpn_disable": vpn_disable, + "forward_drop": forward_drop, + "state_enable": state_enable, + "state_disable": state_disable, + "enable_nat": enable_nat, + "disable_nat": disable_nat, + "init_rttable": init_rttable, + "flush_rttable": flush_rttable, + "forward_enable": forward_enable, + "forward_disable": forward_disable, + "srcroute_enable": srcroute_enable, + "srcroute_disable": srcroute_disable, + "inetsim_enable": inetsim_enable, + "inetsim_disable": inetsim_disable, + "tor_enable": tor_enable, + "tor_disable": tor_disable, + "drop_enable": drop_enable, + "drop_disable": drop_disable, +} + +def cuckoo_rooter(socket_path, group, service, iptables, ip): + try: + import grp + except ImportError: + sys.exit(red( + "Could not find the `grp` module, the Cuckoo Rooter is only " + "supported under Linux operating systems." + )) + + if not service or not os.path.exists(service): + sys.exit(red( + "The service binary is not available, please configure it!\n" + "Note that on CentOS you should provide --service /sbin/service, " + "rather than using the Ubuntu/Debian default /usr/sbin/service." + )) + + if not iptables or not os.path.exists(iptables): + sys.exit(red("The `iptables` binary is not available, eh?!")) + + if not ip or not os.path.exists(ip): + sys.exit(red("The `ip` binary is not available, eh?!")) + + if os.getuid(): + sys.exit(red( + "This utility is supposed to be ran as root user. Please invoke " + "it with the --sudo flag (e.g., 'cuckoo rooter --sudo') so it " + "will automatically prompt for your password (this naturally only " + "works for users with sudo capabilities)." + )) + + if os.path.exists(socket_path): + os.remove(socket_path) + + server = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + server.bind(socket_path) + + # Provide the correct file ownership and permission so Cuckoo can use it + # from an unprivileged process, based on Sean Whalen's routetor. + try: + gr = grp.getgrnam(group) + except KeyError: + sys.exit(red( + "The group ('%s') does not exist. Please define the group / user " + "through which Cuckoo will connect to the rooter, e.g., " + "'cuckoo rooter -g myuser'." % group + )) + + os.chown(socket_path, 0, gr.gr_gid) + os.chmod(socket_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IWGRP) + + # Initialize global variables. + s.service = service + s.iptables = iptables + s.iptables_save = "/sbin/iptables-save" + s.iptables_restore = "/sbin/iptables-restore" + s.ip = ip + + # Simple object to allow a signal handler to stop the rooter loop + class Run(object): + def __init__(self): + self.run = True + do = Run() + + def handle_sigterm(sig, f): + do.run = False + server.shutdown(socket.SHUT_RDWR) + server.close() + cleanup_rooter() + + signal.signal(signal.SIGTERM, handle_sigterm) + + while do.run: + try: + command, addr = server.recvfrom(4096) + except socket.error as e: + if e.errno == errno.EINTR: + continue + elif e.errno == errno.EBADF and not do.run: + continue + + raise e + + try: + obj = json.loads(command) + except: + log.info("Received invalid request: %r", command) + continue + + command = obj.get("command") + args = obj.get("args", []) + kwargs = obj.get("kwargs", {}) + + if not isinstance(command, basestring) or command not in handlers: + log.info("Received incorrect command: %r", command) + continue + + if not isinstance(args, (tuple, list)): + log.info("Invalid arguments type: %r", args) + continue + + if not isinstance(kwargs, dict): + log.info("Invalid keyword arguments: %r", kwargs) + continue + + for arg in args + kwargs.keys() + kwargs.values(): + if not isinstance(arg, basestring): + log.info("Invalid argument detected: %r", arg) + break + else: + log.info( + "Processing command: %s %s %s", command, + " ".join(args), + " ".join("%s=%s" % (k, v) for k, v in kwargs.items()) + ) + + output = e = None + try: + output = handlers[command](*args, **kwargs) + except Exception as e: + log.exception("Error executing command: %s", e) + + server.sendto(json.dumps({ + "output": output, + "exception": str(e) if e else None, + }), addr) diff --git a/cuckoo/auxiliary/__init__.py b/cuckoo/auxiliary/__init__.py new file mode 100644 index 0000000000..ef6e8536fb --- /dev/null +++ b/cuckoo/auxiliary/__init__.py @@ -0,0 +1,11 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +from cuckoo.core.plugins import enumerate_plugins +from cuckoo.common.abstracts import Auxiliary + +plugins = enumerate_plugins( + __file__, "cuckoo.auxiliary", globals(), Auxiliary +) diff --git a/cuckoo/auxiliary/mitm.py b/cuckoo/auxiliary/mitm.py new file mode 100644 index 0000000000..ce47de115f --- /dev/null +++ b/cuckoo/auxiliary/mitm.py @@ -0,0 +1,105 @@ +# Copyright (C) 2015-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import logging +import os.path +import threading + +from cuckoo.common.abstracts import Auxiliary +from cuckoo.misc import cwd, Popen + +log = logging.getLogger(__name__) +PORTS = [] +PORT_LOCK = threading.Lock() + +class MITM(Auxiliary): + def __init__(self): + Auxiliary.__init__(self) + self.proc = None + + def start(self): + port_base = self.options["port_base"] + certificate = self.options["certificate"] + + mitmdump = self.options["mitmdump"] + if not os.path.exists(mitmdump): + log.error("Mitmdump does not exist at path \"%s\", man in the " + "middle interception aborted.", mitmdump) + return + + script = cwd(self.options["script"]) + if not os.path.exists(script): + log.error("Mitmdump script file does not exist at path \"%s\", " + "man in the middle interception aborted.", script) + return + + cert_path = cwd("analyzer", "windows", certificate) + if not os.path.exists(cert_path): + log.error("Mitmdump root certificate not found at path \"%s\" " + "(real path \"%s\"), man in the middle interception " + "aborted.", certificate, cert_path) + return + + PORT_LOCK.acquire() + + for port in xrange(port_base, port_base + 512): + if port not in PORTS: + self.port = port + break + + PORTS.append(self.port) + + PORT_LOCK.release() + + args = [ + mitmdump, "-q", + "-s", '"{}" {}'.format( + script, self.task.options.get("mitm", "") + ).strip(), + "-p", "%d" % self.port, + "-w", cwd("dump.mitm", analysis=self.task.id), + ] + + self.proc = Popen( + args, close_fds=True, + stdout=open(cwd("mitm.log", analysis=self.task.id), "wb"), + stderr=open(cwd("mitm.err", analysis=self.task.id), "wb") + ) + + if "cert" in self.task.options: + log.warning("A root certificate has been provided for this task, " + "however, this is overridden by the mitm auxiliary " + "module.") + + self.task.options["cert"] = certificate + + if "proxy" in self.task.options: + log.warning("A proxy has been provided for this task, however, " + "this is overridden by the mitm auxiliary module.") + + # We are using the resultserver IP address as address for the host + # where our mitmdump instance is running. TODO Is this correct? + self.task.options["proxy"] = ( + "%s:%d" % (self.machine.resultserver_ip, port) + ) + + log.info("Started mitm interception with PID %d (ip=%s, port=%d).", + self.proc.pid, self.machine.resultserver_ip, self.port) + + def stop(self): + if self.proc and not self.proc.poll(): + try: + self.proc.terminate() + PORTS.remove(self.port) + except: + try: + if not self.proc.poll(): + log.debug("Killing mitmdump") + self.proc.kill() + PORTS.remove(self.port) + except OSError as e: + log.debug("Error killing mitmdump: %s. Continue", e) + except Exception as e: + log.exception("Unable to stop mitmdump with pid %d: %s", + self.proc.pid, e) diff --git a/cuckoo/auxiliary/reboot.py b/cuckoo/auxiliary/reboot.py new file mode 100644 index 0000000000..3d7c232361 --- /dev/null +++ b/cuckoo/auxiliary/reboot.py @@ -0,0 +1,69 @@ +# Copyright (C) 2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import json +import logging +import os.path + +from cuckoo.common.abstracts import Auxiliary +from cuckoo.common.exceptions import CuckooDisableModule +from cuckoo.misc import cwd + +log = logging.getLogger(__name__) + +class Reboot(Auxiliary): + def start(self): + if self.task.package != "reboot": + raise CuckooDisableModule + + def cb_legacy_agent(self): + log.error( + "Reboot analysis is not backwards compatible with the Old Agent, " + "please upgrade your target machine (%s) to the New Agent to use " + "the reboot analysis capabilities. Use this new Cuckoo Agent for " + "reboot analysis, https://github.com/jbremer/agent", self.machine + ) + raise CuckooDisableModule + + def _push_dropped_files(self, analysis_path): + files_json = os.path.join(analysis_path, "files.json") + if not os.path.exists(files_json): + return + + # Push dropped files through. + for line in open(files_json, "rb"): + entry = json.loads(line) + + # Screenshots etc. + if not entry["filepath"]: + continue + + filepath = os.path.join(analysis_path, entry["path"]) + + data = { + "filepath": entry["filepath"], + } + files = { + "file": open(filepath, "rb"), + } + self.guest_manager.post("/store", files=files, data=data) + + def cb_prepare_guest(self): + log.info("Preparing task #%d for a reboot analysis..", self.task.id) + + analysis_path = cwd("storage", "analyses", self.task.custom) + + self._push_dropped_files(analysis_path) + + # Push the reboot.json file to the Analyzer. + files = { + "file": open(os.path.join(analysis_path, "reboot.json"), "rb"), + } + reboot_path = os.path.join( + self.guest_manager.analyzer_path, "reboot.json" + ) + data = { + "filepath": reboot_path, + } + self.guest_manager.post("/store", files=files, data=data) diff --git a/cuckoo/auxiliary/replay.py b/cuckoo/auxiliary/replay.py new file mode 100644 index 0000000000..984dd11a5e --- /dev/null +++ b/cuckoo/auxiliary/replay.py @@ -0,0 +1,150 @@ +# Copyright (C) 2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import httpreplay.utils +import logging +import os.path +import tempfile +import threading + +from cuckoo.common.abstracts import Auxiliary +from cuckoo.common.config import config +from cuckoo.core.rooter import rooter +from cuckoo.misc import Popen, cwd + +log = logging.getLogger(__name__) +PORTS = [] +PORT_LOCK = threading.Lock() + +class Replay(Auxiliary): + def __init__(self): + Auxiliary.__init__(self) + self.proc = None + self.port = None + + def pcap2mitm(self, pcappath, tlsmaster): + """Translate a .pcap into a .mitm file.""" + mitmpath = tempfile.mktemp(suffix=".mitm") + with open(mitmpath, "wb") as f: + httpreplay.utils.pcap2mitm(pcappath, f, tlsmaster, True) + return mitmpath + + def start(self): + # Have to explicitly enable Replay. + if not self.task.options.get("replay"): + return + + if self.task.options.get("route"): + log.error( + "A network route must not be specified when performing a " + "Replay analysis." + ) + return + + # TODO We have to do version checking on mitmdump. + mitmdump = self.options["mitmdump"] + port_base = self.options["port_base"] + certificate = self.options["certificate"] + + cert_path = cwd("analyzer", "windows", certificate) + if not os.path.exists(cert_path): + log.error("Mitmdump root certificate not found at path \"%s\" " + "(real path \"%s\"), man in the middle interception " + "aborted.", certificate, cert_path) + return + + mitmpath = self.task.options["replay"] + if not mitmpath.endswith((".pcap", ".mitm")): + log.error( + "Invalid filename (should end with .pcap or .mitm): %s. " + "Can't proceed with replay analysis.", mitmpath + ) + return + + # We support both .mitm and .pcap files. + if mitmpath.endswith(".pcap"): + tlsmaster = self.task.options.get("replay.tls") + mitmpath = self.pcap2mitm(mitmpath, tlsmaster) + + if not os.path.getsize(mitmpath): + log.error( + "Empty .mitm file (potentially after conversion from .pcap), " + "do you have the mitmproxy version 0.18.2 installed (in the " + "same environment as Cuckoo)?" + ) + log.info("Aborting Replay capabilities.") + return + + PORT_LOCK.acquire() + + for port in xrange(port_base, port_base + 512): + if port not in PORTS: + self.port = port + break + + PORTS.append(self.port) + + PORT_LOCK.release() + + # TODO Better access to self.machine and its related fields. + machinery = config("cuckoo:cuckoo:machinery") + rooter( + "inetsim_enable", self.machine.ip, + config("cuckoo:resultserver:ip"), + config("%s:%s:interface" % (machinery, machinery)), + str(config("cuckoo:resultserver:port")), + "80:%d 443:%d" % (self.port, self.port) + ) + + args = [ + mitmdump, "-S", mitmpath, + "--set", "server_replay_ignore_content", + "--set", "server_replay_ignore_host", + # With the port redirection provided by our InetSim support, + # server_replay_ignore_port is strictly speaking irrelevant. + # "--set", "server_replay_ignore_port", + "--server-replay-kill-extra", + "--mode", "transparent", + "-k", "-q", "-p", "%d" % self.port, + ] + + self.proc = Popen(args, close_fds=True) + + if "cert" in self.task.options: + log.warning("A root certificate has been provided for this task, " + "however, this is overridden by the mitm auxiliary " + "module.") + + self.task.options["cert"] = certificate + + log.info( + "Started PCAP replay PID %d (ip=%s, port=%d).", + self.proc.pid, self.machine.resultserver_ip, self.port + ) + + def stop(self): + machinery = config("cuckoo:cuckoo:machinery") + self.port and rooter( + "inetsim_disable", self.machine.ip, + config("cuckoo:resultserver:ip"), + config("%s:%s:interface" % (machinery, machinery)), + str(config("cuckoo:resultserver:port")), + "80:%d 443:%d" % (self.port, self.port) + ) + + if self.proc and not self.proc.poll(): + try: + self.proc.terminate() + PORTS.remove(self.port) + except: + try: + if not self.proc.poll(): + log.debug("Killing mitmdump") + self.proc.kill() + PORTS.remove(self.port) + except OSError as e: + log.debug("Error killing mitmdump: %s. Continue", e) + except Exception as e: + log.exception("Unable to stop mitmdump with pid %d: %s", + self.proc.pid, e) diff --git a/cuckoo/auxiliary/services.py b/cuckoo/auxiliary/services.py new file mode 100644 index 0000000000..55517e0e84 --- /dev/null +++ b/cuckoo/auxiliary/services.py @@ -0,0 +1,75 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import logging +import time + +from cuckoo.common.abstracts import Auxiliary +from cuckoo.common.config import Config +from cuckoo.core.database import Database + +log = logging.getLogger(__name__) +db = Database() + +class Services(Auxiliary): + """Allow one or more additional VMs to be run next to an analysis. Either + as global services (which are generally never rebooted) or on a + per-analysis basis.""" + + def start_service(self, service): + """Start a VM containing one or more services.""" + # We give all services a total of 5 minutes to boot up before + # starting the actual analysis. + timeout = self.task.timeout or Config().timeouts.default + timeout += 300 + tags = "service,%s" % service + + return db.add_service(timeout, self.task.owner, tags) + + def stop_service(self, task_id): + """Stop a VM containing one or more services.""" + db.guest_set_status(task_id, "stop") + + def start(self): + self.tasks = [] + + if self.task.category == "service": + return + + # Have to explicitly enable services. + if not self.task.options.get("services"): + return + + for service in self.options.get("services", "").split(","): + service = service.strip() + if not service: + continue + + task_id = self.start_service(service) + self.tasks.append((task_id, service)) + + log.info("Started service %s #%d for task #%d", + service, task_id, self.task.id) + + # Wait until each service is either starting to run, running, or for + # some reason stopped. + wait_states = "starting", "running", "stopping" + for task_id, service in self.tasks: + while db.guest_get_status(task_id) not in wait_states: + time.sleep(1) + + # Wait an additional timeout before starting the actual analysis. + timeout = self.options.get("timeout") + if isinstance(timeout, int): + time.sleep(timeout) + + def stop(self): + if self.task.category == "service": + return + + for task_id, service in self.tasks: + log.info("Stopping service %s #%d for task #%d", + service, task_id, self.task.id) + self.stop_service(task_id) diff --git a/cuckoo/auxiliary/sniffer.py b/cuckoo/auxiliary/sniffer.py new file mode 100644 index 0000000000..3962269b35 --- /dev/null +++ b/cuckoo/auxiliary/sniffer.py @@ -0,0 +1,174 @@ +# Copyright (C) 2012-2013 Claudio Guarnieri. +# Copyright (C) 2014-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import logging +import os +import subprocess + +from cuckoo.common.abstracts import Auxiliary +from cuckoo.common.constants import CUCKOO_GUEST_PORT, faq +from cuckoo.common.exceptions import CuckooOperationalError +from cuckoo.misc import cwd, getuser, Popen + +log = logging.getLogger(__name__) + +class Sniffer(Auxiliary): + def __init__(self): + Auxiliary.__init__(self) + self.proc = None + + def start(self): + if not self.machine.interface: + log.error("Network interface not defined, network capture aborted") + return False + + # Handle special pcap dumping options. + if "nictrace" in self.machine.options: + return True + + tcpdump = self.options["tcpdump"] + bpf = self.options["bpf"] or "" + file_path = cwd("storage", "analyses", "%s" % self.task.id, "dump.pcap") + + if not os.path.exists(tcpdump): + log.error("Tcpdump does not exist at path \"%s\", network " + "capture aborted", tcpdump) + return False + + # TODO: this isn't working. need to fix. + # mode = os.stat(tcpdump)[stat.ST_MODE] + # if (mode & stat.S_ISUID) == 0: + # log.error("Tcpdump is not accessible from this user, " + # "network capture aborted") + # return + + pargs = [ + tcpdump, "-U", "-q", "-s", "0", "-n", + "-i", self.machine.interface, + ] + + # Trying to save pcap with the same user which cuckoo is running. + user = getuser() + if user: + pargs.extend(["-Z", user]) + + pargs.extend(["-w", file_path]) + pargs.extend(["host", self.machine.ip]) + + if self.task.options.get("sniffer.debug") != "1": + # Do not capture Agent traffic. + pargs.extend([ + "and", "not", "(", + "dst", "host", self.machine.ip, "and", + "dst", "port", "%s" % CUCKOO_GUEST_PORT, + ")", "and", "not", "(", + "src", "host", self.machine.ip, "and", + "src", "port", "%s" % CUCKOO_GUEST_PORT, + ")", + ]) + + # Do not capture ResultServer traffic. + pargs.extend([ + "and", "not", "(", + "dst", "host", self.machine.resultserver_ip, "and", + "dst", "port", "%s" % self.machine.resultserver_port, + ")", "and", "not", "(", + "src", "host", self.machine.resultserver_ip, "and", + "src", "port", "%s" % self.machine.resultserver_port, + ")", + ]) + + if bpf: + pargs.extend(["and", "(", bpf, ")"]) + + try: + self.proc = Popen( + pargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True + ) + except (OSError, ValueError): + log.exception( + "Failed to start sniffer (interface=%s, host=%s, pcap=%s)", + self.machine.interface, self.machine.ip, file_path, + ) + return False + + log.info( + "Started sniffer with PID %d (interface=%s, host=%s)", + self.proc.pid, self.machine.interface, self.machine.ip + ) + return True + + def _check_output(self, out, err): + if out: + raise CuckooOperationalError( + "Potential error while running tcpdump, did not expect " + "standard output, got: %r." % out + ) + + err_safelist_start = ( + "tcpdump: listening on ", + ) + + err_safelist_ends = ( + "packet captured", + "packets captured", + "packet received by filter", + "packets received by filter", + "packet dropped by kernel", + "packets dropped by kernel", + "packet dropped by interface", + "packets dropped by interface", + "dropped privs to root", + ) + + for line in err.split("\n"): + if not line or line.startswith(err_safelist_start): + continue + + if line.endswith(err_safelist_ends): + continue + + raise CuckooOperationalError( + "Potential error while running tcpdump, did not expect " + "the following standard error output: %r." % line + ) + + def stop(self): + """Stop sniffing. + @return: operation status. + """ + # The tcpdump process was never started in the first place. + if not self.proc: + return + + # The tcpdump process has already quit, generally speaking this + # indicates an error such as "permission denied". + if self.proc.poll(): + out, err = self.proc.communicate() + raise CuckooOperationalError( + "Error running tcpdump to sniff the network traffic during " + "the analysis; stdout = %r and stderr = %r. Did you enable " + "the extra capabilities to allow running tcpdump as non-root " + "user and disable AppArmor properly (the latter only applies " + "to Ubuntu-based distributions with AppArmor, see also %s)?" % + (out, err, faq("permission-denied-for-tcpdump")) + ) + + try: + self.proc.terminate() + except: + try: + if not self.proc.poll(): + log.debug("Killing sniffer") + self.proc.kill() + except OSError as e: + log.debug("Error killing sniffer: %s. Continue", e) + except Exception as e: + log.exception("Unable to stop the sniffer with pid %d: %s", + self.proc.pid, e) + + # Ensure expected output was received from tcpdump. + out, err = self.proc.communicate() + self._check_output(out, err) diff --git a/cuckoo/common/__init__.py b/cuckoo/common/__init__.py new file mode 100644 index 0000000000..57edaa85f5 --- /dev/null +++ b/cuckoo/common/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. diff --git a/cuckoo/common/abstracts.py b/cuckoo/common/abstracts.py new file mode 100644 index 0000000000..1b163b8885 --- /dev/null +++ b/cuckoo/common/abstracts.py @@ -0,0 +1,1493 @@ +# Copyright (C) 2012-2013 Claudio Guarnieri. +# Copyright (C) 2014-2019 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import logging +import os +import re +import time +import xml.etree.ElementTree as ET + +from cuckoo.common.config import config +from cuckoo.common.exceptions import CuckooCriticalError +from cuckoo.common.exceptions import CuckooDependencyError +from cuckoo.common.exceptions import CuckooMachineError +from cuckoo.common.exceptions import CuckooOperationalError +from cuckoo.common.exceptions import CuckooReportError +from cuckoo.common.files import Folders +from cuckoo.common.objects import Dictionary +from cuckoo.core.database import Database +from cuckoo.misc import cwd, make_list + +try: + import libvirt + HAVE_LIBVIRT = True +except ImportError: + HAVE_LIBVIRT = False + +log = logging.getLogger(__name__) + +class Configuration(object): + skip = ( + "family", "extra", + ) + # Single entry values. + keywords1 = ( + "type", "version", "magic", "campaign", + ) + # Multiple entry values. + keywords2 = ( + "cnc", "url", "mutex", "user_agent", "referrer", + ) + # Encryption key values. + keywords3 = ( + "des3key", "rc4key", "xorkey", "pubkey", "privkey", "iv", + ) + # Normalize keys. + mapping = { + "cncs": "cnc", + "urls": "url", + "user-agent": "user_agent", + } + + def __init__(self): + self.entries = [] + self.order = [] + self.families = {} + + def add(self, entry): + self.entries.append(entry) + + if entry["family"] not in self.families: + self.families[entry["family"]] = { + "family": entry["family"], + } + self.order.append(entry["family"]) + family = self.families[entry["family"]] + + for key, value in entry.items(): + if key in self.skip or not value: + continue + key = self.mapping.get(key, key) + if key in self.keywords1: + if family.get(key) and family[key] != value: + log.error( + "Duplicate value for %s => %r vs %r", + key, family[key], value + ) + continue + family[key] = value + elif key in self.keywords2: + if key not in family: + family[key] = [] + for value in make_list(value): + if value and value not in family[key]: + family[key].append(value) + elif key in self.keywords3: + if "key" not in family: + family["key"] = {} + if key not in family["key"]: + family["key"][key] = [] + if value not in family["key"][key]: + family["key"][key].append(value) + elif key not in family.get("extra", {}): + if "extra" not in family: + family["extra"] = {} + family["extra"][key] = [value] + elif value not in family["extra"][key]: + family["extra"][key].append(value) + + def get(self, family, *keys): + r = self.families.get(family, {}) + for key in keys: + r = r.get(key, {}) + return r or None + + def family(self, name): + return self.families.get(name) or {} + + def results(self): + ret = [] + for family in self.order: + ret.append(self.families[family]) + return ret + +class Auxiliary(object): + """Base abstract class for auxiliary modules.""" + + def __init__(self): + self.task = None + self.machine = None + self.guest_manager = None + self.options = None + + @classmethod + def init_once(cls): + pass + + def set_task(self, task): + self.task = task + + def set_machine(self, machine): + self.machine = machine + + def set_guest_manager(self, guest_manager): + self.guest_manager = guest_manager + + def set_options(self, options): + self.options = Dictionary(options) + + def start(self): + raise NotImplementedError + + def stop(self): + raise NotImplementedError + +class Machinery(object): + """Base abstract class for machinery modules.""" + + # Default label used in machinery configuration file to supply virtual + # machine name/label/vmx path. Override it if you dubbed it in another + # way. + LABEL = "label" + + def __init__(self): + self.options = None + self.db = Database() + self.remote_control = False + + # Machine table is cleaned to be filled from configuration file + # at each start. + self.db.clean_machines() + + @classmethod + def init_once(cls): + pass + + def pcap_path(self, task_id): + """Return the .pcap path for this task id.""" + return cwd("storage", "analyses", "%s" % task_id, "dump.pcap") + + def set_options(self, options): + """Set machine manager options. + @param options: machine manager options dict. + """ + self.options = options + + def initialize(self, module_name): + """Read, load, and verify machines configuration. + @param module_name: module name. + """ + # Load. + self._initialize(module_name) + + # Run initialization checks. + self._initialize_check() + + def _initialize(self, module_name): + """Read configuration. + @param module_name: module name. + """ + machinery = self.options.get(module_name) + for vmname in machinery["machines"]: + options = self.options.get(vmname) + + # If configured, use specific network interface for this + # machine, else use the default value. + if options.get("interface"): + interface = options["interface"] + else: + interface = machinery.get("interface") + + if options.get("resultserver_ip"): + ip = options["resultserver_ip"] + else: + ip = config("cuckoo:resultserver:ip") + + if options.get("resultserver_port"): + port = options["resultserver_port"] + else: + # The ResultServer port might have been dynamically changed, + # get it from the ResultServer singleton. Also avoid import + # recursion issues by importing ResultServer here. + from cuckoo.core.resultserver import ResultServer + port = ResultServer().port + + self.db.add_machine( + name=vmname, + label=options[self.LABEL], + ip=options.ip, + platform=options.platform, + options=options.get("options", ""), + tags=options.tags, + interface=interface, + snapshot=options.snapshot, + resultserver_ip=ip, + resultserver_port=port + ) + + def _initialize_check(self): + """Run checks against virtualization software when a machine manager + is initialized. + @note: in machine manager modules you may override or superclass + his method. + @raise CuckooMachineError: if a misconfiguration or a unkown vm state + is found. + """ + try: + configured_vms = self._list() + except NotImplementedError: + return + + for machine in self.machines(): + # If this machine is already in the "correct" state, then we + # go on to the next machine. + if machine.label in configured_vms and \ + self._status(machine.label) in [self.POWEROFF, self.ABORTED]: + continue + + # This machine is currently not in its correct state, we're going + # to try to shut it down. If that works, then the machine is fine. + try: + self.stop(machine.label) + except CuckooMachineError as e: + raise CuckooCriticalError( + "Please update your configuration. Unable to shut '%s' " + "down or find the machine in its proper state: %s" % + (machine.label, e) + ) + + if not config("cuckoo:timeouts:vm_state"): + raise CuckooCriticalError( + "Virtual machine state change timeout has not been set " + "properly, please update it to be non-null." + ) + + def machines(self): + """List virtual machines. + @return: virtual machines list + """ + return self.db.list_machines() + + def availables(self): + """Return how many machines are free. + @return: free machines count. + """ + return self.db.count_machines_available() + + def acquire(self, machine_id=None, platform=None, tags=None): + """Acquire a machine to start analysis. + @param machine_id: machine ID. + @param platform: machine platform. + @param tags: machine tags + @return: machine or None. + """ + if machine_id: + return self.db.lock_machine(label=machine_id) + elif platform: + return self.db.lock_machine(platform=platform, tags=tags) + else: + return self.db.lock_machine(tags=tags) + + def release(self, label=None): + """Release a machine. + @param label: machine name. + """ + self.db.unlock_machine(label) + + def running(self): + """Return running virtual machines. + @return: running virtual machines list. + """ + return self.db.list_machines(locked=True) + + def shutdown(self): + """Shutdown the machine manager and kill all alive machines. + @raise CuckooMachineError: if unable to stop machine. + """ + if len(self.running()) > 0: + log.info("Still %s guests alive. Shutting down...", + len(self.running())) + for machine in self.running(): + try: + self.stop(machine.label) + except CuckooMachineError as e: + log.warning("Unable to shutdown machine %s, please check " + "manually. Error: %s", machine.label, e) + + def set_status(self, label, status): + """Set status for a virtual machine. + @param label: virtual machine label + @param status: new virtual machine status + """ + self.db.set_machine_status(label, status) + + def start(self, label, task): + """Start a machine. + @param label: machine name. + @param task: task object. + @raise NotImplementedError: this method is abstract. + """ + raise NotImplementedError + + def stop(self, label=None): + """Stop a machine. + @param label: machine name. + @raise NotImplementedError: this method is abstract. + """ + raise NotImplementedError + + def _list(self): + """List virtual machines configured. + @raise NotImplementedError: this method is abstract. + """ + raise NotImplementedError + + def dump_memory(self, label, path): + """Take a memory dump of a machine. + @param path: path to where to store the memory dump. + """ + raise NotImplementedError + + def enable_remote_control(self, label): + """Enable remote control interface (RDP/VNC/SSH). + @param label: machine name. + @return: None + """ + raise NotImplementedError + + def disable_remote_control(self, label): + """Disable remote control interface (RDP/VNC/SSH). + @param label: machine name. + @return: None + """ + raise NotImplementedError + + def get_remote_control_params(self, label): + """Return connection details for remote control. + @param label: machine name. + @return: dict with keys: protocol, host, port + """ + raise NotImplementedError + + def _wait_status(self, label, *states): + """Wait for a vm status. + @param label: virtual machine name. + @param state: virtual machine status, accepts multiple states as list. + @raise CuckooMachineError: if default waiting timeout expire. + """ + # This block was originally suggested by Loic Jaquemet. + waitme = 0 + try: + current = self._status(label) + except NameError: + return + + while current not in states: + log.debug("Waiting %i cuckooseconds for machine %s to switch " + "to status %s", waitme, label, states) + if waitme > config("cuckoo:timeouts:vm_state"): + raise CuckooMachineError( + "Timeout hit while for machine %s to change status" % label + ) + + time.sleep(1) + waitme += 1 + current = self._status(label) + + @staticmethod + def version(): + """Return the version of the virtualization software""" + return None + +class LibVirtMachinery(Machinery): + """Libvirt based machine manager. + + If you want to write a custom module for a virtualization software + supported by libvirt you have just to inherit this machine manager and + change the connection string. + """ + + # VM states. + RUNNING = "running" + PAUSED = "paused" + POWEROFF = "poweroff" + ERROR = "machete" + ABORTED = "abort" + + def __init__(self): + if not HAVE_LIBVIRT: + raise CuckooDependencyError( + "The libvirt package has not been installed " + "(`pip install libvirt-python`)" + ) + + super(LibVirtMachinery, self).__init__() + + def initialize(self, module): + """Initialize machine manager module. Override default to set proper + connection string. + @param module: machine manager module + """ + super(LibVirtMachinery, self).initialize(module) + + def _initialize_check(self): + """Run all checks when a machine manager is initialized. + @raise CuckooMachineError: if libvirt version is not supported. + """ + # Version checks. + if not self._version_check(): + raise CuckooMachineError("Libvirt version is not supported, " + "please get an updated version") + + # Preload VMs + self.vms = self._fetch_machines() + + # Base checks. Also attempts to shutdown any machines which are + # currently still active. + super(LibVirtMachinery, self)._initialize_check() + + def start(self, label, task): + """Start a virtual machine. + @param label: virtual machine name. + @param task: task object. + @raise CuckooMachineError: if unable to start virtual machine. + """ + log.debug("Starting machine %s", label) + + if self._status(label) != self.POWEROFF: + msg = "Trying to start a virtual machine that has not " \ + "been turned off {0}".format(label) + raise CuckooMachineError(msg) + + conn = self._connect() + + vm_info = self.db.view_machine_by_label(label) + + snapshot_list = self.vms[label].snapshotListNames(flags=0) + + # If a snapshot is configured try to use it. + if vm_info.snapshot and vm_info.snapshot in snapshot_list: + # Revert to desired snapshot, if it exists. + log.debug("Using snapshot {0} for virtual machine " + "{1}".format(vm_info.snapshot, label)) + try: + vm = self.vms[label] + snapshot = vm.snapshotLookupByName(vm_info.snapshot, flags=0) + self.vms[label].revertToSnapshot(snapshot, flags=0) + except libvirt.libvirtError: + msg = "Unable to restore snapshot {0} on " \ + "virtual machine {1}".format(vm_info.snapshot, label) + raise CuckooMachineError(msg) + finally: + self._disconnect(conn) + elif self._get_snapshot(label): + snapshot = self._get_snapshot(label) + log.debug("Using snapshot {0} for virtual machine " + "{1}".format(snapshot.getName(), label)) + try: + self.vms[label].revertToSnapshot(snapshot, flags=0) + except libvirt.libvirtError: + raise CuckooMachineError("Unable to restore snapshot on " + "virtual machine {0}".format(label)) + finally: + self._disconnect(conn) + else: + self._disconnect(conn) + raise CuckooMachineError("No snapshot found for virtual machine " + "{0}".format(label)) + + # Check state. + self._wait_status(label, self.RUNNING) + + def stop(self, label): + """Stop a virtual machine. Kill them all. + @param label: virtual machine name. + @raise CuckooMachineError: if unable to stop virtual machine. + """ + log.debug("Stopping machine %s", label) + + if self._status(label) == self.POWEROFF: + raise CuckooMachineError("Trying to stop an already stopped " + "machine {0}".format(label)) + + # Force virtual machine shutdown. + conn = self._connect() + try: + if not self.vms[label].isActive(): + log.debug("Trying to stop an already stopped machine %s. " + "Skip", label) + else: + self.vms[label].destroy() # Machete's way! + except libvirt.libvirtError as e: + raise CuckooMachineError("Error stopping virtual machine " + "{0}: {1}".format(label, e)) + finally: + self._disconnect(conn) + # Check state. + self._wait_status(label, self.POWEROFF) + + def shutdown(self): + """Override shutdown to free libvirt handlers - they print errors.""" + super(LibVirtMachinery, self).shutdown() + + # Free handlers. + self.vms = None + + def dump_memory(self, label, path): + """Take a memory dump. + @param path: path to where to store the memory dump. + """ + log.debug("Dumping memory for machine %s", label) + + conn = self._connect() + try: + # Resolve permission issue as libvirt creates the file as + # root/root in mode 0600, preventing us from reading it. This + # supposedly still doesn't allow us to remove it, though.. + open(path, "wb").close() + self.vms[label].coreDump(path, flags=libvirt.VIR_DUMP_MEMORY_ONLY) + except libvirt.libvirtError as e: + raise CuckooMachineError("Error dumping memory virtual machine " + "{0}: {1}".format(label, e)) + finally: + self._disconnect(conn) + + def _status(self, label): + """Get current status of a vm. + @param label: virtual machine name. + @return: status string. + """ + log.debug("Getting status for %s", label) + + # Stetes mapping of python-libvirt. + # virDomainState + # VIR_DOMAIN_NOSTATE = 0 + # VIR_DOMAIN_RUNNING = 1 + # VIR_DOMAIN_BLOCKED = 2 + # VIR_DOMAIN_PAUSED = 3 + # VIR_DOMAIN_SHUTDOWN = 4 + # VIR_DOMAIN_SHUTOFF = 5 + # VIR_DOMAIN_CRASHED = 6 + # VIR_DOMAIN_PMSUSPENDED = 7 + + conn = self._connect() + try: + state = self.vms[label].state(flags=0) + except libvirt.libvirtError as e: + raise CuckooMachineError("Error getting status for virtual " + "machine {0}: {1}".format(label, e)) + finally: + self._disconnect(conn) + + if state: + if state[0] == 1: + status = self.RUNNING + elif state[0] == 3: + status = self.PAUSED + elif state[0] == 4 or state[0] == 5: + status = self.POWEROFF + else: + status = self.ERROR + + # Report back status. + if status: + self.set_status(label, status) + return status + else: + raise CuckooMachineError("Unable to get status for " + "{0}".format(label)) + + def _connect(self): + """Connect to libvirt subsystem. + @raise CuckooMachineError: when unable to connect to libvirt. + """ + # Check if a connection string is available. + if not self.dsn: + raise CuckooMachineError("You must provide a proper " + "connection string") + + try: + return libvirt.open(self.dsn) + except libvirt.libvirtError: + raise CuckooMachineError("Cannot connect to libvirt") + + def _disconnect(self, conn): + """Disconnect from libvirt subsystem. + @raise CuckooMachineError: if cannot disconnect from libvirt. + """ + try: + conn.close() + except libvirt.libvirtError: + raise CuckooMachineError("Cannot disconnect from libvirt") + + def _fetch_machines(self): + """Fetch machines handlers. + @return: dict with machine label as key and handle as value. + """ + vms = {} + for vm in self.machines(): + vms[vm.label] = self._lookup(vm.label) + return vms + + def _lookup(self, label): + """Search for a virtual machine. + @param conn: libvirt connection handle. + @param label: virtual machine name. + @raise CuckooMachineError: if virtual machine is not found. + """ + conn = self._connect() + try: + vm = conn.lookupByName(label) + except libvirt.libvirtError: + raise CuckooMachineError("Cannot find machine " + "{0}".format(label)) + finally: + self._disconnect(conn) + return vm + + def _list(self): + """List available virtual machines. + @raise CuckooMachineError: if unable to list virtual machines. + """ + conn = self._connect() + try: + names = conn.listDefinedDomains() + except libvirt.libvirtError: + raise CuckooMachineError("Cannot list domains") + finally: + self._disconnect(conn) + return names + + def _version_check(self): + """Check if libvirt release supports snapshots. + @return: True or false. + """ + if libvirt.getVersion() >= 8000: + return True + else: + return False + + def _get_snapshot(self, label): + """Get current snapshot for virtual machine + @param label: virtual machine name + @return None or current snapshot + @raise CuckooMachineError: if cannot find current snapshot or + when there are too many snapshots available + """ + def _extract_creation_time(node): + """Extracts creation time from a KVM vm config file. + @param node: config file node + @return: extracted creation time + """ + xml = ET.fromstring(node.getXMLDesc(flags=0)) + return xml.findtext("./creationTime") + + snapshot = None + conn = self._connect() + try: + vm = self.vms[label] + + # Try to get the currrent snapshot, otherwise fallback on the latest + # from config file. + if vm.hasCurrentSnapshot(flags=0): + snapshot = vm.snapshotCurrent(flags=0) + else: + log.debug("No current snapshot, using latest snapshot") + + # No current snapshot, try to get the last one from config file. + snapshot = sorted(vm.listAllSnapshots(flags=0), + key=_extract_creation_time, + reverse=True)[0] + except libvirt.libvirtError: + raise CuckooMachineError("Unable to get snapshot for " + "virtual machine {0}".format(label)) + finally: + self._disconnect(conn) + + return snapshot + + def enable_remote_control(self, label): + # TODO: we can't dynamically enable/disable this right now + pass + + def disable_remote_control(self, label): + pass + + def get_remote_control_params(self, label): + conn = self._connect() + + try: + vm = conn.lookupByName(label) + if not vm: + log.warning("No such VM: %s", label) + return {} + + port = 0 + desc = ET.fromstring(vm.XMLDesc()) + for elem in desc.findall("./devices/graphics"): + if elem.attrib.get("type") == "vnc": + # Future work: passwd, listen, socket (addr:port) + port = elem.attrib.get("port") + if port: + port = int(port) + break + finally: + self._disconnect(conn) + + if port <= 0: + log.error("VM %s does not have a valid VNC port", label) + return {} + + # TODO The Cuckoo Web Interface may be running at a different host + # than the actual Cuckoo daemon (and as such, the VMs). + return { + "protocol": "vnc", + "host": "127.0.0.1", + "port": port, + } + +class Processing(object): + """Base abstract class for processing module.""" + order = 1 + enabled = True + + def __init__(self): + self.analysis_path = "" + self.baseline_path = "" + self.logs_path = "" + self.task = None + self.machine = None + self.options = None + self.results = {} + + @classmethod + def init_once(cls): + pass + + def set_options(self, options): + """Set processing options. + @param options: processing options dict. + """ + self.options = Dictionary(options) + + def set_task(self, task): + """Add task information. + @param task: task dictionary. + """ + self.task = task + + def set_machine(self, machine): + """Add machine information.""" + self.machine = machine + + def set_baseline(self, baseline_path): + """Set the path to the baseline directory.""" + self.baseline_path = baseline_path + + def set_path(self, analysis_path): + """Set paths. + @param analysis_path: analysis folder path. + """ + self.analysis_path = analysis_path + self.log_path = os.path.join(self.analysis_path, "analysis.log") + self.cuckoolog_path = os.path.join(self.analysis_path, "cuckoo.log") + self.file_path = os.path.realpath(os.path.join(self.analysis_path, + "binary")) + self.dropped_path = os.path.join(self.analysis_path, "files") + self.dropped_meta_path = os.path.join(self.analysis_path, "files.json") + self.extracted_path = os.path.join(self.analysis_path, "extracted") + self.package_files = os.path.join(self.analysis_path, "package_files") + self.buffer_path = os.path.join(self.analysis_path, "buffer") + self.logs_path = os.path.join(self.analysis_path, "logs") + self.shots_path = os.path.join(self.analysis_path, "shots") + self.pcap_path = os.path.join(self.analysis_path, "dump.pcap") + self.pmemory_path = os.path.join(self.analysis_path, "memory") + self.memory_path = os.path.join(self.analysis_path, "memory.dmp") + self.mitmout_path = os.path.join(self.analysis_path, "mitm.log") + self.mitmerr_path = os.path.join(self.analysis_path, "mitm.err") + self.tlsmaster_path = os.path.join(self.analysis_path, "tlsmaster.txt") + self.suricata_path = os.path.join(self.analysis_path, "suricata") + self.network_path = os.path.join(self.analysis_path, "network") + self.taskinfo_path = os.path.join(self.analysis_path, "task.json") + + def set_results(self, results): + """Set the results - the fat dictionary.""" + self.results = results + + def run(self): + """Start processing. + @raise NotImplementedError: this method is abstract. + """ + raise NotImplementedError + +class Signature(object): + """Base class for Cuckoo signatures.""" + name = "" + description = "" + severity = 1 + order = 1 + categories = [] + families = [] + authors = [] + references = [] + platform = None + alert = False + enabled = True + minimum = None + maximum = None + ttp = [] + + # Maximum amount of marks to record. + markcount = 50 + + # Basic filters to reduce the amount of events sent to this signature. + filter_apinames = [] + filter_categories = [] + + def __init__(self, caller): + """ + @param caller: calling object. Stores results in caller.results + """ + self.marks = [] + self.matched = False + self._caller = caller + + # These are set by the caller, they represent the process identifier + # and call index respectively. + self.pid = None + self.cid = None + self.call = None + + @classmethod + def init_once(cls): + pass + + def _check_value(self, pattern, subject, regex=False, all=False): + """Check a pattern against a given subject. + @param pattern: string or expression to check for. + @param subject: target of the check. + @param regex: boolean representing if the pattern is a regular + expression or not and therefore should be compiled. + @return: boolean with the result of the check. + """ + ret = set() + if regex: + exp = re.compile(pattern, re.IGNORECASE) + if isinstance(subject, list): + for item in subject: + if exp.match(item): + ret.add(item) + else: + if exp.match(subject): + ret.add(subject) + else: + if isinstance(subject, list): + for item in subject: + if item.lower() == pattern.lower(): + ret.add(item) + else: + if subject == pattern: + ret.add(subject) + + # Return all elements. + if all: + return list(ret) + # Return only the first element, if available. Otherwise return None. + elif ret: + return ret.pop() + + def get_results(self, key=None, default=None): + if key: + return self._caller.results.get(key, default) + + return self._caller.results + + def get_processes(self, name=None): + """Get a list of processes. + + @param name: If set only return processes with that name. + @return: List of processes or empty list + """ + for item in self.get_results("behavior", {}).get("processes", []): + if name is None or item["process_name"] == name: + yield item + + def get_process_by_pid(self, pid=None): + """Get a process by its process identifier. + + @param pid: pid to search for. + @return: process. + """ + for item in self.get_results("behavior", {}).get("processes", []): + if item["pid"] == pid: + return item + + def get_summary(self, key=None, default=[]): + """Get one or all values related to the global summary.""" + summary = self.get_results("behavior", {}).get("summary", {}) + return summary.get(key, default) if key else summary + + def get_summary_generic(self, pid, actions): + """Get generic info from summary. + + @param pid: pid of the process. None for all + @param actions: A list of actions to get + """ + ret = [] + for process in self.get_results("behavior", {}).get("generic", []): + if pid is not None and process["pid"] != pid: + continue + + for action in actions: + if action in process["summary"]: + ret += process["summary"][action] + return ret + + def get_files(self, pid=None, actions=None): + """Get files read, queried, or written to optionally by a + specific process. + + @param pid: the process or None for all + @param actions: actions to search for. None is all + @return: yields files + + """ + if actions is None: + actions = [ + "file_opened", "file_written", + "file_read", "file_deleted", + "file_exists", "file_failed", + ] + + return self.get_summary_generic(pid, actions) + + def get_dll_loaded(self, pid=None): + """Get DLLs loaded by a specific process. + + @param pid: the process or None for all + @return: yields DLLs loaded + + """ + return self.get_summary_generic(pid, ["dll_loaded"]) + + def get_keys(self, pid=None, actions=None): + """Get registry keys. + + @param pid: The pid to look in or None for all. + @param actions: the actions as a list. + @return: yields registry keys + + """ + if actions is None: + actions = [ + "regkey_opened", "regkey_written", + "regkey_read", "regkey_deleted", + ] + + return self.get_summary_generic(pid, actions) + + def check_file(self, pattern, regex=False, actions=None, pid=None, + all=False): + """Check for a file being opened. + @param pattern: string or expression to check for. + @param regex: boolean representing if the pattern is a regular + expression or not and therefore should be compiled. + @param actions: a list of key actions to use. + @param pid: The process id to check. If it is set to None, all + processes will be checked. + @return: boolean with the result of the check. + """ + if actions is None: + actions = [ + "file_opened", "file_written", + "file_read", "file_deleted", + "file_exists", "file_failed", + ] + + return self._check_value(pattern=pattern, + subject=self.get_files(pid, actions), + regex=regex, + all=all) + + def check_dll_loaded(self, pattern, regex=False, actions=None, pid=None, + all=False): + """Check for DLLs being loaded. + @param pattern: string or expression to check for. + @param regex: boolean representing if the pattern is a regular + expression or not and therefore should be compiled. + @param pid: The process id to check. If it is set to None, all + processes will be checked. + @return: boolean with the result of the check. + """ + return self._check_value(pattern=pattern, + subject=self.get_dll_loaded(pid), + regex=regex, + all=all) + + def check_command_line(self, pattern, regex=False, all=False): + """Check for a command line being opened. + @param pattern: string or expression to check for. + @param regex: boolean representing if the pattern is a regular + expression or not and therefore should be compiled. + @return: boolean with the result of the check. + """ + return self._check_value(pattern=pattern, + subject=self.get_summary("command_line"), + regex=regex, + all=all) + + def check_key(self, pattern, regex=False, actions=None, pid=None, + all=False): + """Check for a registry key being accessed. + @param pattern: string or expression to check for. + @param regex: boolean representing if the pattern is a regular + expression or not and therefore should be compiled. + @param actions: a list of key actions to use. + @param pid: The process id to check. If it is set to None, all + processes will be checked. + @return: boolean with the result of the check. + """ + if actions is None: + actions = [ + "regkey_written", "regkey_opened", + "regkey_read", "regkey_deleted", + ] + + return self._check_value(pattern=pattern, + subject=self.get_keys(pid, actions), + regex=regex, + all=all) + + def get_mutexes(self, pid=None): + """ + @param pid: Pid to filter for + @return:List of mutexes + """ + return self.get_summary_generic(pid, ["mutex"]) + + def check_mutex(self, pattern, regex=False, all=False): + """Check for a mutex being opened. + @param pattern: string or expression to check for. + @param regex: boolean representing if the pattern is a regular + expression or not and therefore should be compiled. + @return: boolean with the result of the check. + """ + return self._check_value(pattern=pattern, + subject=self.get_mutexes(), + regex=regex, + all=all) + + def get_command_lines(self): + """Retrieve all command lines used.""" + return self.get_summary("command_line") + + def get_wmi_queries(self): + """Retrieve all executed WMI queries.""" + return self.get_summary("wmi_query") + + def get_net_generic(self, subtype): + """Generic getting network data. + + @param subtype: subtype string to search for. + """ + return self.get_results("network", {}).get(subtype, []) + + def get_net_hosts(self): + """Return a list of all hosts.""" + return self.get_net_generic("hosts") + + def get_net_domains(self): + """Return a list of all domains.""" + return self.get_net_generic("domains") + + def get_net_http(self): + """Return a list of all http data.""" + return self.get_net_generic("http") + + def get_net_http_ex(self): + """Return a list of all http data.""" + return \ + self.get_net_generic("http_ex") + self.get_net_generic("https_ex") + + def get_net_udp(self): + """Return a list of all udp data.""" + return self.get_net_generic("udp") + + def get_net_icmp(self): + """Return a list of all icmp data.""" + return self.get_net_generic("icmp") + + def get_net_irc(self): + """Return a list of all irc data.""" + return self.get_net_generic("irc") + + def get_net_smtp(self): + """Return a list of all smtp data.""" + return self.get_net_generic("smtp") + + def get_net_smtp_ex(self): + """"Return a list of all smtp data""" + return self.get_net_generic("smtp_ex") + + def get_virustotal(self): + """Return the information retrieved from virustotal.""" + return self.get_results("virustotal", {}) + + def get_volatility(self, module=None): + """Return the data that belongs to the given module.""" + volatility = self.get_results("memory", {}) + return volatility if module is None else volatility.get(module, {}) + + def get_apkinfo(self, section=None, default={}): + """Return the apkinfo results for this analysis.""" + apkinfo = self.get_results("apkinfo", {}) + return apkinfo if section is None else apkinfo.get(section, default) + + def get_droidmon(self, section=None, default={}): + """Return the droidmon results for this analysis.""" + droidmon = self.get_results("droidmon", {}) + return droidmon if section is None else droidmon.get(section, default) + + def get_googleplay(self, section=None, default={}): + """Return the Google Play results for this analysis.""" + googleplay = self.get_results("googleplay", {}) + return googleplay if section is None else googleplay.get(section, default) + + def check_ip(self, pattern, regex=False, all=False): + """Check for an IP address being contacted. + @param pattern: string or expression to check for. + @param regex: boolean representing if the pattern is a regular + expression or not and therefore should be compiled. + @return: boolean with the result of the check. + """ + return self._check_value(pattern=pattern, + subject=self.get_net_hosts(), + regex=regex, + all=all) + + def check_domain(self, pattern, regex=False, all=False): + """Check for a domain being contacted. + @param pattern: string or expression to check for. + @param regex: boolean representing if the pattern is a regular + expression or not and therefore should be compiled. + @return: boolean with the result of the check. + """ + domains = set() + for item in self.get_net_domains(): + domains.add(item["domain"]) + + return self._check_value(pattern=pattern, + subject=list(domains), + regex=regex, + all=all) + + def check_url(self, pattern, regex=False, all=False): + """Check for a URL being contacted. + @param pattern: string or expression to check for. + @param regex: boolean representing if the pattern is a regular + expression or not and therefore should be compiled. + @return: boolean with the result of the check. + """ + urls = set() + for item in self.get_net_http(): + urls.add(item["uri"]) + + return self._check_value(pattern=pattern, + subject=list(urls), + regex=regex, + all=all) + + def check_suricata_alerts(self, pattern): + """Check for pattern in Suricata alert signature + @param pattern: string or expression to check for. + @return: True/False + """ + for alert in self.get_results("suricata", {}).get("alerts", []): + if re.findall(pattern, alert.get("signature", ""), re.I): + return True + return False + + def init(self): + """Allow signatures to initialize themselves.""" + + def mark_call(self, *args, **kwargs): + """Mark the current call as explanation as to why this signature + matched.""" + mark = { + "type": "call", + "pid": self.pid, + "cid": self.cid, + "call": self.call, + } + + if args or kwargs: + log.warning( + "You have provided extra arguments to the mark_call() method " + "which no longer supports doing so. Please report explicit " + "IOCs through mark_ioc()." + ) + + self.marks.append(mark) + + def mark_ioc(self, category, ioc, description=None): + """Mark an IOC as explanation as to why the current signature + matched.""" + mark = { + "type": "ioc", + "category": category, + "ioc": ioc, + "description": description, + } + + # Prevent duplicates. + if mark not in self.marks: + self.marks.append(mark) + + def mark_vol(self, plugin, **kwargs): + """Mark output of a Volatility plugin as explanation as to why the + current signature matched.""" + mark = { + "type": "volatility", + "plugin": plugin, + } + mark.update(kwargs) + self.marks.append(mark) + + def mark_config(self, config): + """Mark configuration from this malware family.""" + if not isinstance(config, dict) or "family" not in config: + raise CuckooCriticalError("Invalid call to mark_config().") + + self.marks.append({ + "type": "config", + "config": config, + }) + + def mark(self, **kwargs): + """Mark arbitrary data.""" + mark = { + "type": "generic", + } + mark.update(kwargs) + self.marks.append(mark) + + def has_marks(self, count=None): + """Return true if this signature has one or more marks.""" + if count is not None: + return len(self.marks) >= count + return not not self.marks + + def on_call(self, call, process): + """Notify signature about API call. Return value determines + if this signature is done or could still match. + + Only called if signature is "active". + + @param call: logged API call. + @param process: proc object. + """ + raise NotImplementedError + + def on_signature(self, signature): + """Event yielded when another signatures has matched. Some signatures + only take effect when one or more other signatures have matched as + well. + + @param signature: The signature that just matched + """ + + def on_process(self, process): + """Called on process change. + + Can be used for cleanup of flags, re-activation of the signature, etc. + + @param process: dictionary describing this process + """ + + def on_yara(self, category, filepath, match): + """Called on YARA match. + @param category: yara match category + @param filepath: path to the file that matched + @param match: yara match information + + The Yara match category can be one of the following. + extracted: an extracted PE image from a process memory dump + procmem: a process memory dump + dropped: a dropped file + """ + + def on_extract(self, match): + """Called on an Extracted match. + @param match: extracted match information + """ + + def on_complete(self): + """Signature is notified when all API calls have been processed.""" + + def extend_ttp(self): + """Find the short and long descriptions for the TTPs of a signature""" + d = {} + for t in self.ttp: + d[t] = self._caller.ttp_descriptions.get(t) + return d + + def results(self): + """Turn this signature into actionable results.""" + return dict(name=self.name, + ttp=self.extend_ttp(), + description=self.description, + severity=self.severity, + families=self.families, + references=self.references, + marks=self.marks[:self.markcount], + markcount=len(self.marks)) + + @property + def cfgextr(self): + return self._caller.c + +class Report(object): + """Base abstract class for reporting module.""" + order = 1 + + def __init__(self): + self.analysis_path = "" + self.reports_path = "" + self.task = None + self.options = None + + @classmethod + def init_once(cls): + pass + + def _get_analysis_path(self, subpath): + return os.path.join(self.analysis_path, subpath) + + def set_path(self, analysis_path): + """Set analysis folder path. + @param analysis_path: analysis folder path. + """ + self.analysis_path = analysis_path + self.file_path = os.path.realpath(self._get_analysis_path("binary")) + self.reports_path = self._get_analysis_path("reports") + self.shots_path = self._get_analysis_path("shots") + self.pcap_path = self._get_analysis_path("dump.pcap") + + try: + Folders.create(self.reports_path) + except CuckooOperationalError as e: + raise CuckooReportError(e) + + def set_options(self, options): + """Set report options. + @param options: report options dict. + """ + self.options = Dictionary(options) + + def set_task(self, task): + """Add task information. + @param task: task dictionary. + """ + self.task = task + + def run(self, results): + """Start report processing. + @raise NotImplementedError: this method is abstract. + """ + raise NotImplementedError + +class BehaviorHandler(object): + """Base class for behavior handlers inside of BehaviorAnalysis.""" + key = "undefined" + + # Behavior event types this handler is interested in. + event_types = [] + + def __init__(self, behavior_analysis): + self.analysis = behavior_analysis + + def handles_path(self, logpath): + """Needs to return True for the log files this handler wants to + process.""" + return False + + def parse(self, logpath): + """Called after handles_path succeeded, should generate behavior + events.""" + raise NotImplementedError + + def handle_event(self, event): + """Handle an event that gets passed down the stack.""" + raise NotImplementedError + + def run(self): + """Return the handler specific structure, gets placed into + behavior[self.key].""" + raise NotImplementedError + + +class ProtocolHandler(object): + """Abstract class for protocol handlers coming out of the analysis.""" + def __init__(self, task_id, ctx, version=None): + self.task_id = task_id + self.handler = ctx + self.fd = None + self.version = version + + def __enter__(self): + self.init() + + def __exit__(self, type, value, traceback): + self.close() + + def close(self): + if self.fd: + self.fd.close() + self.fd = None + + def handle(self): + raise NotImplementedError + + +class Extractor(object): + """One piece in a series of recursive extractors & unpackers.""" + yara_rules = [] + # Minimum and maximum supported version in Cuckoo. + minimum = None + maximum = None + + @classmethod + def init_once(cls): + pass + + def __init__(self, parent): + self.parent = parent + + def handle_yara(self, filepath, match): + raise NotImplementedError + + def push_command_line(self, cmdline, process=None): + self.parent.push_command_line(cmdline, process) + + def push_script(self, process, command): + self.parent.push_script(process, command) + + def push_script_recursive(self, command): + self.parent.push_script_recursive(command) + + def push_shellcode(self, sc): + self.parent.push_shellcode(sc) + + def push_blob(self, blob, category, externals, info=None): + self.parent.push_blob(blob, category, externals, info) + + def push_blob_noyara(self, blob, category, info=None): + self.parent.push_blob_noyara(blob, category, info) + + def push_config(self, config): + self.parent.push_config(config) + + def enhance(self, filepath, key, value): + self.parent.enhance(filepath, key, value) diff --git a/lib/cuckoo/common/colors.py b/cuckoo/common/colors.py similarity index 91% rename from lib/cuckoo/common/colors.py rename to cuckoo/common/colors.py index e04ae8f21a..bfa725a1bf 100644 --- a/lib/cuckoo/common/colors.py +++ b/cuckoo/common/colors.py @@ -1,4 +1,5 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. diff --git a/cuckoo/common/compare.py b/cuckoo/common/compare.py new file mode 100644 index 0000000000..ac31ae9338 --- /dev/null +++ b/cuckoo/common/compare.py @@ -0,0 +1,99 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import os +import collections + +from cuckoo.misc import cwd + +ANALYSIS_ROOT = cwd("storage", "analyses") + +def behavior_categories_percent(calls): + catcounts = collections.defaultdict(lambda: 0) + + for call in calls: + catcounts[call.get("category", "none")] += 1 + + return dict(catcounts) + +def combine_behavior_percentages(stats): + # get all categories present + cats = set() + for v in stats.values(): + for v2 in v.values(): + cats |= set(v2.keys()) + + sums = {} + for tid in stats: + sums[tid] = {} + for cat in cats: + sums[tid][cat] = sum(j.get(cat, 0) for j in stats[tid].values()) + + totals = dict((k, sum(v.values())) for k, v in sums.items()) + + percentages = {} + for tid in stats: + percentages[tid] = {} + for cat in cats: + percentages[tid][cat] = round(sums[tid][cat] * 1.0 / totals[tid] * 100, 2) + + return percentages + +def iter_task_process_logfiles(tid): + tpath = os.path.join(ANALYSIS_ROOT, str(tid), "logs") + + for fname in os.listdir(tpath): + fpath = os.path.join(tpath, fname) + pid = int(fname.split(".")[0]) + yield (pid, fpath) + +def helper_percentages_storage(tid1, tid2): + counts = {} + + for tid in [tid1, tid2]: + counts[tid] = {} + + for pid, fpath in iter_task_process_logfiles(tid): + # ppl = ParseProcessLog(fpath) + # category_counts = behavior_categories_percent(ppl.calls) + category_counts = None + + counts[tid][pid] = category_counts + + return combine_behavior_percentages(counts) + +def helper_percentages_mongo(results_db, tid1, tid2, ignore_categories=["misc"]): + counts = {} + + for tid in[tid1, tid2]: + counts[tid] = {} + + pids_calls = results_db.analysis.find_one( + { + "info.id": int(tid), + }, + { + "behavior.processes.pid": 1, + "behavior.processes.calls": 1 + } + ) + + if not pids_calls: + continue + + for pdoc in pids_calls["behavior"]["processes"]: + pid = pdoc["pid"] + counts[tid][pid] = {} + + for coid in pdoc["calls"]: + chunk = results_db.calls.find_one({"_id": coid}, {"calls.category": 1}) + category_counts = behavior_categories_percent(chunk["calls"]) + for cat, count in category_counts.items(): + if cat in ignore_categories: + continue + + counts[tid][pid][cat] = counts[tid][pid].get(cat, 0) + count + + return combine_behavior_percentages(counts) diff --git a/cuckoo/common/config.py b/cuckoo/common/config.py new file mode 100644 index 0000000000..7728d513d6 --- /dev/null +++ b/cuckoo/common/config.py @@ -0,0 +1,1224 @@ +# Copyright (C) 2012-2013 Claudio Guarnieri. +# Copyright (C) 2014-2019 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import ConfigParser +import click +import os +import logging +import re + +from cuckoo.common.exceptions import CuckooConfigurationError +from cuckoo.common.objects import Dictionary +from cuckoo.common.utils import parse_bool +from cuckoo.misc import cwd + +log = logging.getLogger(__name__) + +_cache = {} + +class Type(object): + """Base Class for Type Definitions""" + + def __init__(self, default=None, required=True, sanitize=False, + allow_empty=False): + self.required = required + self.sanitize = sanitize + self.allow_empty = allow_empty + self.default = self.parse(default) + + def parse(self, value): + """Parse a raw input value.""" + + def check(self, value): + """Checks the type of the value.""" + + def emit(self, value): + """String-readable version of this object""" + +class Int(Type): + """Integer Type Definition class.""" + + def parse(self, value): + if isinstance(value, (int, long)): + return value + + if isinstance(value, basestring) and value.isdigit(): + return int(value) + + def check(self, value): + if self.allow_empty and not value: + return True + + try: + click.INT(value) + return True + except: + return False + + def emit(self, value): + return "%d" % value if value is not None else "" + +class String(Type): + """String Type Definition class.""" + + def parse(self, value): + return value.strip() if value else None + + def check(self, value): + if self.allow_empty and not value: + return True + + return isinstance(value, basestring) + + def emit(self, value): + return value or "" + +class Path(String): + """Path Type Definition class.""" + + def __init__(self, default=None, exists=False, writable=False, + readable=False, required=True, allow_empty=False, + sanitize=False): + self.exists = exists + self.writable = writable + self.readable = readable + super(Path, self).__init__(default, required, sanitize, allow_empty) + + def parse(self, value): + if self.allow_empty and not value: + return + + try: + c = click.Path( + exists=self.exists, + writable=self.writable, + readable=self.readable + ) + return c.convert(value, None, None) + except Exception: + return value + + def check(self, value): + if self.allow_empty and not value: + return True + + try: + c = click.Path( + exists=self.exists, + writable=self.writable, + readable=self.readable + ) + c.convert(value, None, None) + return True + except: + return False + + def emit(self, value): + return value or "" + +class Boolean(Type): + """Boolean Type Definition class.""" + + def parse(self, value): + try: + return parse_bool(value) + except: + log.error("Incorrect Boolean %s", value) + + def check(self, value): + try: + parse_bool(value) + return True + except: + return False + + def emit(self, value): + return "yes" if value else "no" + +class UUID(Type): + """UUID Type Definition class.""" + + def parse(self, value): + try: + c = click.UUID(value) + return str(c) + except: + log.error("Incorrect UUID %s", value) + + def check(self, value): + """Check if the value is of type UUID.""" + try: + click.UUID(value) + return True + except: + return False + + def emit(self, value): + return value + +class List(Type): + """List Type Definition class.""" + + def __init__(self, subclass, default, sep=",", strip=True): + self.subclass = subclass + self.sep = sep + self.strip = strip + super(List, self).__init__(default) + + def parse(self, value): + if value is None: + return [] + + try: + ret = [] + + if isinstance(value, (tuple, list)): + for entry in value: + ret.append(self.subclass().parse(entry)) + return ret + + for entry in re.split("[%s]" % self.sep, value): + if self.strip: + entry = entry.strip() + if not entry: + continue + + ret.append(self.subclass().parse(entry)) + return ret + except: + log.error("Incorrect list: %s", value) + + def check(self, value): + try: + value.split(self.sep) + return True + except: + return False + + def emit(self, value): + return (", " if self.sep[0] == "," else self.sep[0]).join(value or "") + +class Config(object): + """Configuration file parser.""" + + configuration = { + "cuckoo": { + "cuckoo": { + "version_check": Boolean(True), + "ignore_vulnerabilities": Boolean(False, required=False), + "delete_original": Boolean(False), + "delete_bin_copy": Boolean(False), + "machinery": String("virtualbox"), + "memory_dump": Boolean(False), + "terminate_processes": Boolean(False), + "reschedule": Boolean(False), + "process_results": Boolean(True), + "max_analysis_count": Int(0), + "max_machines_count": Int(0), + "max_vmstartup_count": Int(10), + "freespace": Int(1024), + "tmppath": Path( + exists=True, writable=True, readable=False, + allow_empty=True + ), + "api_token": String( + allow_empty=True, sanitize=True, required=False + ), + "web_secret": String( + allow_empty=True, sanitize=True, required=False + ), + "rooter": Path( + "/tmp/cuckoo-rooter", + exists=False, writable=False, readable=False + ), + }, + "feedback": { + "enabled": Boolean(False), + "name": String(), + "company": String(), + "email": String(), + }, + "resultserver": { + "ip": String("192.168.56.1"), + "port": Int(2042), + "force_port": Boolean(False, False), # Unused + "pool_size": Int(0, False), + "upload_max_size": Int(128 * 1024 * 1024), + }, + "processing": { + "analysis_size_limit": Int(128 * 1024 * 1024), + "resolve_dns": Boolean(True), + "sort_pcap": Boolean(True), + }, + "database": { + "connection": String(sanitize=True), + "timeout": Int(60, allow_empty=True), + }, + "timeouts": { + "default": Int(120), + "critical": Int(60), + "vm_state": Int(60), + }, + "remotecontrol": { + "enabled": Boolean(False), + "guacd_host": String("localhost"), + "guacd_port": Int(4822), + }, + }, + "virtualbox": { + "virtualbox": { + "mode": String("headless"), + "path": Path( + "/usr/bin/VBoxManage", + exists=False, writable=False, readable=True + ), + "interface": String("vboxnet0"), + "machines": List(String, "cuckoo1"), + "controlports": String("5000-5050", required=False), + }, + "*": { + "__section__": "cuckoo1", + "label": String("cuckoo1"), + "platform": String("windows"), + "ip": String("192.168.56.101"), + "snapshot": String(), + "interface": String(), + "resultserver_ip": String(), + "resultserver_port": Int(), + "tags": String(), + "options": List(String, None, ",\\s"), + "osprofile": String(required=False), + }, + "__star__": ("virtualbox", "machines"), + }, + "auxiliary": { + "sniffer": { + "enabled": Boolean(True), + "tcpdump": Path( + "/usr/sbin/tcpdump", + exists=False, writable=False, readable=True + ), + "bpf": String(), + }, + "mitm": { + "enabled": Boolean(False), + "mitmdump": Path( + "/usr/local/bin/mitmdump", + exists=False, writable=False, readable=True + ), + "port_base": Int(50000), + "script": Path( + "stuff/mitm.py", + exists=False, writable=False, readable=True + ), + "certificate": Path( + "bin/cert.p12", + exists=False, writable=False, readable=True + ), + }, + "replay": { + "enabled": Boolean(True, required=False), + "mitmdump": Path( + "/usr/local/bin/mitmdump", exists=False, + writable=False, readable=True, required=False + ), + "port_base": Int(51000, required=False), + "certificate": Path( + "bin/cert.p12", exists=False, + writable=False, readable=True, required=False + ), + }, + "services": { + "enabled": Boolean(False), + "services": String("honeyd"), + "timeout": Int(0), + }, + "reboot": { + "enabled": Boolean(True), + }, + }, + "avd": { + "avd": { + "mode": String("headless"), + "emulator_path": Path( + "/home/cuckoo/android-sdk-linux/tools/emulator", + exists=True, writable=False, readable=True + ), + "adb_path": Path( + "/home/cuckoo/android-sdk-linux/platform-tools/adb", + exists=True, writable=False, readable=True + ), + "avd_path": Path( + "/home/cuckoo/.android/avd", + exists=True, writable=False, readable=True + ), + "reference_machine": String("cuckoo-bird"), + "machines": List(String, "cuckoo1"), + }, + "*": { + "__section__": "cuckoo1", + "label": String("cuckoo1"), + "platform": String("android"), + "ip": String("127.0.0.1"), + "emulator_port": Int(5554), + "resultserver_ip": String("10.0.2.2"), + "resultserver_port": Int(2042), + "osprofile": String(required=False), + }, + "__star__": ("avd", "machines"), + }, + "esx": { + "esx": { + "dsn": String("esx://127.0.0.1/?no_verify=1"), + "username": String("username_goes_here"), + "password": String("password_goes_here", sanitize=True), + "machines": List(String, "analysis1"), + "interface": String("eth0"), + }, + "*": { + "__section__": "analysis1", + "label": String("cuckoo1"), + "platform": String("windows"), + "ip": String("192.168.122.101"), + "snapshot": String("clean_snapshot"), + "interface": String(), + "resultserver_ip": String(), + "resultserver_port": Int(), + "tags": String(), + "osprofile": String(required=False), + }, + "__star__": ("esx", "machines"), + }, + "kvm": { + "kvm": { + "dsn": String("qemu:///system", required=False), + "interface": String("virbr0"), + "machines": List(String, "cuckoo1"), + }, + "*": { + "__section__": "cuckoo1", + "label": String("cuckoo1"), + "platform": String("windows"), + "ip": String("192.168.122.101"), + "snapshot": String(), + "interface": String(), + "resultserver_ip": String(), + "resultserver_port": Int(), + "tags": String(), + "osprofile": String(required=False), + }, + "__star__": ("kvm", "machines"), + }, + "memory": { + "basic": { + "guest_profile": String("WinXPSP2x86"), + "delete_memdump": Boolean(False), + }, + "malfind": { + "enabled": Boolean(True), + "filter": Boolean(True), + }, + "apihooks": { + "enabled": Boolean(False), + "filter": Boolean(True), + }, + "pslist": { + "enabled": Boolean(True), + "filter": Boolean(False), + }, + "psxview": { + "enabled": Boolean(True), + "filter": Boolean(False), + }, + "callbacks": { + "enabled": Boolean(True), + "filter": Boolean(False), + }, + "idt": { + "enabled": Boolean(True), + "filter": Boolean(False), + }, + "timers": { + "enabled": Boolean(True), + "filter": Boolean(False), + }, + "messagehooks": { + "enabled": Boolean(False), + "filter": Boolean(False), + }, + "getsids": { + "enabled": Boolean(True), + "filter": Boolean(False), + }, + "privs": { + "enabled": Boolean(True), + "filter": Boolean(False), + }, + "dlllist": { + "enabled": Boolean(True), + "filter": Boolean(True), + }, + "handles": { + "enabled": Boolean(True), + "filter": Boolean(True), + }, + "ldrmodules": { + "enabled": Boolean(True), + "filter": Boolean(True), + }, + "mutantscan": { + "enabled": Boolean(True), + "filter": Boolean(True), + }, + "devicetree": { + "enabled": Boolean(True), + "filter": Boolean(True), + }, + "svcscan": { + "enabled": Boolean(True), + "filter": Boolean(True), + }, + "modscan": { + "enabled": Boolean(True), + "filter": Boolean(True), + }, + "yarascan": { + "enabled": Boolean(True), + "filter": Boolean(True), + }, + "ssdt": { + "enabled": Boolean(True), + "filter": Boolean(True), + }, + "gdt": { + "enabled": Boolean(True), + "filter": Boolean(True), + }, + "sockscan": { + "enabled": Boolean(True), + "filter": Boolean(False), + }, + "netscan": { + "enabled": Boolean(True), + "filter": Boolean(False), + }, + "mask": { + "enabled": Boolean(False), + "pid_generic": List(String, None), + }, + }, + "physical": { + "physical": { + "machines": List(String, "physical1"), + "user": String("username"), + "password": String("password", sanitize=True), + "interface": String("eth0"), + }, + "fog": { + "hostname": String("none"), + "username": String("fog"), + "password": String("password", sanitize=True), + }, + "*": { + "__section__": "physical1", + "label": String("physical1"), + "platform": String("windows"), + "ip": String("192.168.56.101"), + "osprofile": String(required=False), + }, + "__star__": ("physical", "machines"), + }, + "processing": { + "analysisinfo": { + "enabled": Boolean(True), + }, + "apkinfo": { + "enabled": Boolean(False), + "decompilation_threshold": Int(5000000), + }, + "baseline": { + "enabled": Boolean(False), + }, + "behavior": { + "enabled": Boolean(True), + }, + "buffer": { + "enabled": Boolean(True), + }, + "debug": { + "enabled": Boolean(True), + }, + "droidmon": { + "enabled": Boolean(False), + }, + "dropped": { + "enabled": Boolean(True), + }, + "dumptls": { + "enabled": Boolean(True), + }, + "extracted": { + "enabled": Boolean(True, required=False), + }, + "googleplay": { + "enabled": Boolean(False), + "android_id": String(), + "google_login": String(), + "google_password": String(sanitize=True), + }, + "memory": { + "enabled": Boolean(False), + }, + "misp": { + "enabled": Boolean(False), + "url": String(), + "apikey": String(sanitize=True), + "maxioc": Int(100), + }, + "network": { + "enabled": Boolean(True), + "safelist_dns": Boolean(False), + "allowed_dns": String(), + }, + "procmemory": { + "enabled": Boolean(True), + "idapro": Boolean(False), + "extract_img": Boolean(True), + "extract_dll": Boolean(False), + "dump_delete": Boolean(False), + }, + "procmon": { + "enabled": Boolean(True), + }, + "screenshots": { + "enabled": Boolean(True), + "tesseract": String("no"), + }, + "snort": { + "enabled": Boolean(False), + "snort": Path( + "/usr/local/bin/snort", + exists=False, writable=False, readable=True + ), + "conf": Path( + "/etc/snort/snort.conf", + exists=False, writable=False, readable=True + ), + }, + "static": { + "enabled": Boolean(True), + "pdf_timeout": Int(60), + }, + "strings": { + "enabled": Boolean(True), + }, + "suricata": { + "enabled": Boolean(False), + "suricata": Path( + "/usr/bin/suricata", + exists=True, writable=False, readable=True + ), + "conf": Path( + "/etc/suricata/suricata.yaml", + exists=True, writable=False, readable=True + ), + "eve_log": Path( + "eve.json", + exists=False, writable=True, readable=False + ), + "files_log": Path( + "files-json.log", + exists=False, writable=True, readable=False + ), + "files_dir": Path( + "files", + exists=False, writable=False, readable=True + ), + "socket": Path( + exists=True, writable=False, readable=True, + allow_empty=True + ), + }, + "targetinfo": { + "enabled": Boolean(True), + }, + "virustotal": { + "enabled": Boolean(False), + "timeout": Int(60), + "scan": Boolean(False), + "key": String("a0283a2c3d55728300d064874239b5346fb991317e8449fe43c902879d758088", sanitize=True), + }, + "irma": { + "enabled": Boolean(False), + "timeout": Int(60), + "scan": Boolean(False), + "force": Boolean(False), + "url": String(), + "probes": String(required=False), + }, + }, + "qemu": { + "qemu": { + "path": Path( + "/usr/bin/qemu-system-x86_64", + exists=True, writable=False, readable=True + ), + "interface": String("qemubr"), + "machines": List(String, "vm1,vm2,vm3"), + }, + "*": [ + { + "__section__": "vm1", + "label": String("vm1"), + "image": Path( + "/home/rep/vms/qvm_wheezy64_1.qcow2", + exists=True, writable=False, readable=True + ), + "snapshot": String(required=False), + "arch": String(), + "enable_kvm": Boolean(False), + "platform": String("linux"), + "ip": String("192.168.55.2"), + "interface": String("qemubr"), + "resultserver_ip": String("192.168.55.1"), + "resultserver_port": Int(), + "tags": String("debian_wheezy,64_bit"), + "kernel": String(), + "initrd": String(), + "osprofile": String(required=False), + }, { + "__section__": "vm2", + "label": String("vm2"), + "image": Path( + "/home/rep/vms/qvm_wheezy64_1.qcow2", + exists=True, writable=False, readable=True + ), + "snapshot": String(required=False), + "arch": String("mipsel"), + "enable_kvm": Boolean(False), + "platform": String("linux"), + "ip": String("192.168.55.3"), + "interface": String("qemubr"), + "resultserver_ip": String("192.168.55.1"), + "resultserver_port": Int(), + "tags": String("debian_wheezy,mipsel"), + "kernel": String( + "{imagepath}/vmlinux-3.16.0-4-4kc-malta-mipsel" + ), + "osprofile": String(""), + }, { + "__section__": "vm3", + "label": String("vm3"), + "image": Path( + "/home/rep/vms/qvm_wheezy64_1.qcow2", + exists=True, writable=False, readable=True + ), + "snapshot": String(required=False), + "arch": String("arm"), + "enable_kvm": Boolean(False), + "platform": String("linux"), + "ip": String("192.168.55.4"), + "interface": String("qemubr"), + "tags": String("debian_wheezy,arm"), + "kernel": String( + "{imagepath}/vmlinuz-3.2.0-4-versatile-arm" + ), + "initrd": String( + "{imagepath}/initrd-3.2.0-4-versatile-arm" + ), + "osprofile": String(""), + }, + ], + "__star__": ("qemu", "machines"), + }, + "reporting": { + "feedback": { + "enabled": Boolean(False), + }, + "jsondump": { + "enabled": Boolean(True), + "indent": Int(4), + "calls": Boolean(True), + }, + "singlefile": { + "enabled": Boolean(False), + "html": Boolean(False), + "pdf": Boolean(False), + }, + "misp": { + "enabled": Boolean(False), + "url": String(), + "apikey": String(sanitize=True), + "mode": String("maldoc ipaddr hashes url"), + "distribution": Int(0, required=False), + "analysis": Int(0, required=False), + "threat_level": Int(4, required=False), + "min_malscore": Int(0, required=False), + "tag": String("Cuckoo", required=False), + "upload_sample": Boolean(False, required=False), + }, + "mongodb": { + "enabled": Boolean(False), + "host": String("127.0.0.1"), + "port": Int(27017), + "db": String("cuckoo"), + "store_memdump": Boolean(True), + "paginate": Int(100), + "username": String(), + "password": String(), + }, + "elasticsearch": { + "enabled": Boolean(False), + "hosts": List(String, "127.0.0.1"), + "timeout": Int(300), + "calls": Boolean(False), + "index": String("cuckoo"), + "index_time_pattern": String("yearly"), + "cuckoo_node": String(), + }, + "moloch": { + "enabled": Boolean(False), + "host": String(), + "insecure": Boolean(False), + "moloch_capture": Path( + "/data/moloch/bin/moloch-capture", + exists=True, writable=False, readable=True + ), + "conf": Path( + "/data/moloch/etc/config.ini", + exists=True, writable=False, readable=True + ), + "instance": String("cuckoo"), + }, + "notification": { + "enabled": Boolean(False), + "url": String(), + "identifier": String(), + }, + "mattermost": { + "enabled": Boolean(False), + "username": String("cuckoo"), + "url": String(), + "myurl": String(), + "show_virustotal": Boolean(False), + "show_signatures": Boolean(False), + "show_urls": Boolean(False), + "hash_filename": Boolean(False), + "hash_url": Boolean(False), + }, + }, + "routing": { + "routing": { + "route": String("none"), + "internet": String("none"), + "rt_table": String("main"), + "auto_rt": Boolean(True), + "drop": Boolean(False), + }, + "inetsim": { + "enabled": Boolean(False), + "server": String("192.168.56.1"), + "ports": String(), + }, + "tor": { + "enabled": Boolean(False), + "dnsport": Int(5353), + "proxyport": Int(9040), + }, + "vpn": { + "enabled": Boolean(False), + "vpns": List(String, "vpn0"), + }, + "*": { + "__section__": "vpn0", + "name": String("vpn0"), + "description": String("Spain, Europe"), + "interface": String("tun0"), + "rt_table": String("tun0"), + }, + "__star__": ("vpn", "vpns"), + }, + "vmware": { + "vmware": { + "mode": String("gui"), + "path": Path( + "/usr/bin/vmrun", + exists=True, writable=False, readable=True + ), + "interface": String("virbr0"), + "machines": List(String, "cuckoo1"), + }, + "*": { + "__section__": "cuckoo1", + "vmx_path": Path( + "../cuckoo1/cuckoo1.vmx", + exists=True, writable=False, readable=True + ), + "snapshot": String("Snapshot1"), + "platform": String("windows"), + "ip": String("192.168.54.111"), + "interface": String(), + "resultserver_ip": String(), + "resultserver_port": Int(), + "tags": String(), + "osprofile": String(required=False), + }, + "__star__": ("vmware", "machines"), + }, + "vsphere": { + "vsphere": { + "host": String("10.0.0.1"), + "port": Int(443), + "user": String("username_goes_here"), + "pwd": String("password_goes_here", sanitize=True), + "interface": String("eth0"), + "machines": List(String, "analysis1"), + "unverified_ssl": Boolean(False), + }, + "*": { + "__section__": "analysis1", + "label": String("cuckoo1"), + "platform": String("windows"), + "snapshot": String("snapshot_name"), + "ip": String("192.168.122.101"), + "interface": String(), + "resultserver_ip": String(required=False), + "resultserver_port": Int(required=False), + "tags": String(required=False), + "osprofile": String(required=False), + }, + "__star__": ("vsphere", "machines"), + }, + "xenserver": { + "xenserver": { + "user": String("root"), + "password": String("changeme", sanitize=True), + "url": String("https://xenserver"), + "interface": String("virbr0"), + "machines": List(String, "cuckoo1"), + }, + "*": { + "__section__": "cuckoo1", + "uuid": UUID("00000000-0000-0000-0000-000000000000"), + "snapshot": String(), + "platform": String("windows"), + "ip": String("192.168.54.111"), + "interface": String(), + "resultserver_ip": String(), + "resultserver_port": Int(), + "tags": String(), + "osprofile": String(required=False), + }, + "__star__": ("xenserver", "machines"), + }, + } + + def get_section_types(self, file_name, section, strict=False, loose=False): + """Get types for a section entry.""" + section_types = get_section_types(file_name, section) + if not section_types and not loose: + log.error( + "Config section %s:%s not found!", file_name, section + ) + if strict: + raise CuckooConfigurationError( + "Config section %s:%s not found!", file_name, section + ) + return + return section_types + + def __init__(self, file_name="cuckoo", cfg=None, strict=False, + loose=False, raw=False): + """ + @param file_name: file name without extension. + @param cfg: configuration file path. + """ + env = {} + for key, value in os.environ.items(): + if key.startswith("CUCKOO_"): + env[key] = value + + env["CUCKOO_CWD"] = cwd() + env["CUCKOO_APP"] = os.environ.get("CUCKOO_APP", "") + config = ConfigParser.ConfigParser(env) + + self.env_keys = [] + for key in env.keys(): + self.env_keys.append(key.lower()) + + self.sections = {} + + try: + config.read(cfg or cwd("conf", "%s.conf" % file_name)) + except ConfigParser.ParsingError as e: + raise CuckooConfigurationError( + "There was an error reading in the $CWD/conf/%s.conf " + "configuration file. Most likely there are leading " + "whitespaces in front of one of the key=value lines defined. " + "More information from the original exception: %s" % + (file_name, e) + ) + + if file_name not in self.configuration and not loose: + log.error("Unknown config file %s.conf", file_name) + return + + for section in config.sections(): + types = self.get_section_types(file_name, section, strict, loose) + if types is None: + continue + + self.sections[section] = Dictionary() + setattr(self, section, self.sections[section]) + + try: + items = config.items(section) + except ConfigParser.InterpolationMissingOptionError as e: + log.error("Missing environment variable(s): %s", e) + raise CuckooConfigurationError( + "Missing environment variable: %s" % e + ) + except ValueError as e: + if e.message == "incomplete format key": + raise CuckooConfigurationError( + "One of the fields that you've filled out in " + "$CWD/conf/%s contains the sequence '%(' which is " + "interpreted as environment variable sequence, e.g., " + "'%(PGPASSWORD)s' would locate a PostgreSQL " + "password. Please update the field to correctly " + "state the environment variable or change it in a " + "way that '%(' is no longer in the variable." + ) + raise + + for name, raw_value in items: + if name in self.env_keys: + continue + + if "\n" in raw_value: + wrong_key = "???" + try: + wrong_key = raw_value.split("\n", 1)[1].split()[0] + except: + pass + + raise CuckooConfigurationError( + "There was an error reading in the $CWD/conf/%s.conf " + "configuration file. Namely, there are one or more " + "leading whitespaces before the definition of the " + "'%s' key/value pair in the '%s' section. Please " + "remove those leading whitespaces as Python's default " + "configuration parser is unable to handle those " + "properly." % (file_name, wrong_key, section) + ) + + if not raw and name in types: + # TODO Is this the area where we should be checking the + # configuration values? + # if not types[name].check(raw_value): + # print file_name, section, name, raw_value + # raise + + value = types[name].parse(raw_value) + else: + if not loose: + log.error( + "Type of config parameter %s:%s:%s not found! " + "This may indicate that you've incorrectly filled " + "out the Cuckoo configuration, please double " + "check it.", file_name, section, name + ) + value = raw_value + + self.sections[section][name] = value + + def get(self, section): + """Get option. + @param section: section to fetch. + @raise CuckooConfigurationError: if section not found. + @return: option value. + """ + if section not in self.sections: + raise CuckooConfigurationError( + "Option %s is not found in configuration" % section + ) + + return self.sections[section] + + @staticmethod + def from_confdir(dirpath, loose=False, sanitize=False): + """Read all the configuration from a configuration directory. If + `sanitize` is set, then black out sensitive fields.""" + ret = {} + for filename in os.listdir(dirpath): + if not filename.endswith(".conf"): + continue + + config_name = filename.rsplit(".", 1)[0] + cfg = Config( + config_name, cfg=os.path.join(dirpath, filename), loose=loose + ) + + ret[config_name] = {} + for section, values in cfg.sections.items(): + ret[config_name][section] = {} + types = cfg.get_section_types( + config_name, section, loose=loose + ) or {} + for key, value in values.items(): + if sanitize and key in types and types[key].sanitize: + value = "*"*8 + + ret[config_name][section][key] = value + return ret + +def parse_options(options): + """Parse the analysis options field to a dictionary.""" + ret = {} + for field in options.split(","): + if "=" not in field: + continue + + key, value = field.split("=", 1) + ret[key.strip()] = value.strip() + return ret + +def emit_options(options): + """Emit the analysis options from a dictionary to a string.""" + return ",".join("%s=%s" % (k, v) for k, v in sorted(options.items())) + +def config(s, cfg=None, strict=False, raw=False, loose=False, check=False): + """Fetch a configuration value, denoted as file:section:key.""" + if s.count(":") != 2: + raise RuntimeError("Invalid configuration entry: %s" % s) + + file_name, section, key = s.split(":") + + if check: + strict = raw = loose = True + + type_ = Config.configuration.get(file_name, {}).get(section, {}).get(key) + if strict and type_ is None: + raise CuckooConfigurationError( + "No such configuration value exists: %s" % s + ) + + required = type_ is not None and type_.required + index = file_name, cfg, cwd(), strict, raw, loose + + if index not in _cache: + _cache[index] = Config( + file_name, cfg=cfg, strict=strict, raw=raw, loose=loose + ) + + config = _cache[index] + + if strict and required and section not in config.sections: + raise CuckooConfigurationError( + "Configuration value %s not present! This may indicate that " + "you've incorrectly filled out the Cuckoo configuration, " + "please double check it." % s + ) + + section = config.sections.get(section, {}) + if strict and required and key not in section: + raise CuckooConfigurationError( + "Configuration value %s not present! This may indicate that " + "you've incorrectly filled out the Cuckoo configuration, " + "please double check it." % s + ) + + value = section.get(key, type_.default if type_ else None) + + if check and not type_.check(value): + raise CuckooConfigurationError( + "The configuration value %r found for %s is invalid. Please " + "update your configuration!" % (value, s) + ) + + return value + +def get_section_types(file_name, section, strict=False): + if section in Config.configuration.get(file_name, {}): + return Config.configuration[file_name][section] + + if "__star__" not in Config.configuration.get(file_name, {}): + return {} + + if strict: + section_, key = Config.configuration[file_name]["__star__"] + if section not in config("%s:%s:%s" % (file_name, section_, key)): + return {} + + if "*" in Config.configuration.get(file_name, {}): + section_types = Config.configuration[file_name]["*"] + # If multiple default values have been provided, pick one. + if isinstance(section_types, (tuple, list)): + section_types = section_types[0] + return section_types + return {} + +def config2(file_name, section): + keys = get_section_types(file_name, section, strict=True) + if not keys: + raise CuckooConfigurationError( + "No such configuration section exists: %s:%s" % + (file_name, section) + ) + + ret = Dictionary() + for key in keys: + if key == "__star__" or key == "*": + continue + ret[key] = config("%s:%s:%s" % (file_name, section, key)) + return ret + +def cast(s, value): + """Cast a configuration value as per its type.""" + if s.count(":") != 2: + raise RuntimeError("Invalid configuration entry: %s" % s) + + file_name, section, key = s.split(":") + type_ = get_section_types(file_name, section).get(key) + if type_ is None: + raise CuckooConfigurationError( + "No such configuration value exists: %s" % s + ) + + return type_.parse(value) + +def read_kv_conf(filepath): + """Read a flat Cuckoo key/value configuration file.""" + ret = {} + for line in open(filepath, "rb"): + line = line.strip() + if not line or line.startswith("#"): + continue + + if "=" not in line: + raise CuckooConfigurationError( + "Invalid flat configuration line: %s (missing '=' character)" % + line + ) + + key, raw_value = line.split("=", 1) + key, raw_value = key.replace(".", ":").strip(), raw_value.strip() + try: + value = cast(key, raw_value) + except (CuckooConfigurationError, RuntimeError) as e: + raise CuckooConfigurationError( + "Invalid flat configuration line: %s (error %s)" % (line, e) + ) + + if raw_value and value is None: + raise CuckooConfigurationError( + "Invalid flat configuration entry: %s is None" % key + ) + + a, b, c = key.split(":") + ret[a] = ret.get(a, {}) + ret[a][b] = ret[a].get(b, {}) + ret[a][b][c] = value + return ret diff --git a/cuckoo/common/constants.py b/cuckoo/common/constants.py new file mode 100644 index 0000000000..596045df34 --- /dev/null +++ b/cuckoo/common/constants.py @@ -0,0 +1,16 @@ +# Copyright (C) 2011-2013 Claudio Guarnieri. +# Copyright (C) 2014-2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +CUCKOO_GUEST_PORT = 8000 +CUCKOO_GUEST_INIT = 0x001 +CUCKOO_GUEST_RUNNING = 0x002 +CUCKOO_GUEST_COMPLETED = 0x003 +CUCKOO_GUEST_FAILED = 0x004 +GITHUB_URL = "https://github.com/cuckoosandbox/cuckoo" +ISSUES_PAGE_URL = "https://github.com/cuckoosandbox/cuckoo/issues" +DOCS_URL = "https://cuckoo.sh/docs" + +def faq(entry): + return "%s/faq/index.html#%s" % (DOCS_URL, entry) diff --git a/cuckoo/common/defines.py b/cuckoo/common/defines.py new file mode 100644 index 0000000000..9c311a4b16 --- /dev/null +++ b/cuckoo/common/defines.py @@ -0,0 +1,21 @@ +# Copyright (C) 2013 Claudio Guarnieri. +# Copyright (C) 2014-2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +REG_NONE = 0 +REG_SZ = 1 +REG_EXPAND_SZ = 2 +REG_BINARY = 3 +REG_DWORD_LITTLE_ENDIAN = 4 +REG_DWORD = 4 +REG_DWORD_BIG_ENDIAN = 5 + +# Windows specific process rights +# https://msdn.microsoft.com/en-us/library/ms684880(v=vs.85).aspx +WIN_PROCESS_QUERY_INFORMATION = 0x0400 + +# Windows specific error codes +# https://msdn.microsoft.com/en-us/library/windows/desktop +# /ms683189(v=vs.85).aspx +WIN_ERR_STILL_ALIVE = 259 diff --git a/lib/cuckoo/common/dns.py b/cuckoo/common/dns.py similarity index 90% rename from lib/cuckoo/common/dns.py rename to cuckoo/common/dns.py index 8069d6b116..a13bce6081 100644 --- a/lib/cuckoo/common/dns.py +++ b/cuckoo/common/dns.py @@ -1,4 +1,5 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. @@ -12,10 +13,10 @@ except: HAVE_CARES = False -#try: +# try: # import gevent, gevent.socket # HAVE_GEVENT = True -#except: +# except: HAVE_GEVENT = False @@ -35,15 +36,17 @@ def set_timeout_value(value): # standard gethostbyname in thread # http://code.activestate.com/recipes/473878/ def with_timeout(func, args=(), kwargs={}): - """This function will spawn a thread and run the given function - using the args, kwargs and return the given default value if the - timeout_duration is exceeded. + """Spawn a thread and run the given function using the args, + kwargs and return the given default value if the timeout_duration + is exceeded. """ class ResultThread(threading.Thread): daemon = True + def __init__(self): threading.Thread.__init__(self) self.result, self.error = None, None + def run(self): try: self.result = func(*args, **kwargs) @@ -106,7 +109,7 @@ class Resultholder: # gevent based resolver with timeout -def resolve_gevent(name): +"""def resolve_gevent(name): result = resolve_gevent_real(name) # if it failed, do this a second time because of strange libevent behavior # basically sometimes the Timeout fires immediately instead of after @@ -124,14 +127,14 @@ def resolve_gevent_real(name): pass return result - +""" # choose resolver automatically def resolve(name): if HAVE_CARES: return resolve_cares(name) - elif HAVE_GEVENT: - return resolve_gevent(name) + # elif HAVE_GEVENT: + # return resolve_gevent(name) else: return resolve_thread(name) diff --git a/cuckoo/common/elastic.py b/cuckoo/common/elastic.py new file mode 100644 index 0000000000..d4a8668455 --- /dev/null +++ b/cuckoo/common/elastic.py @@ -0,0 +1,52 @@ +# Copyright (C) 2017-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +from cuckoo.common.config import config +from cuckoo.common.exceptions import CuckooOperationalError + +class Elastic(object): + def __init__(self): + self.client = None + + self.enabled = None + self.hosts = None + self.calls = None + self.index = None + self.index_time_pattern = None + self.cuckoo_node = None + + def init(self): + self.enabled = config("reporting:elasticsearch:enabled") + self.hosts = config("reporting:elasticsearch:hosts") + self.timeout = config("reporting:elasticsearch:timeout") + self.calls = config("reporting:elasticsearch:calls") + self.index = config("reporting:elasticsearch:index") + self.index_time_pattern = config( + "reporting:elasticsearch:index_time_pattern" + ) + self.cuckoo_node = config("reporting:elasticsearch:cuckoo_node") + return self.enabled + + def connect(self): + # TODO Option to throw an exception? + if not self.enabled: + return + + import elasticsearch + + try: + self.client = elasticsearch.Elasticsearch( + self.hosts, timeout=self.timeout + ) + except TypeError as e: + raise CuckooOperationalError( + "Unable to connect to ElasticSearch due to an invalid ip:port " + "pair: %s" % e + ) + except elasticsearch.ConnectionError as e: + raise CuckooOperationalError( + "Unable to connect to ElasticSearch: %s" % e + ) + +elastic = Elastic() diff --git a/cuckoo/common/exceptions.py b/cuckoo/common/exceptions.py new file mode 100644 index 0000000000..e35550786c --- /dev/null +++ b/cuckoo/common/exceptions.py @@ -0,0 +1,58 @@ +# Copyright (C) 2012-2013 Claudio Guarnieri. +# Copyright (C) 2014-2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +class CuckooCriticalError(Exception): + """Cuckoo struggle in a critical error.""" + +class CuckooStartupError(CuckooCriticalError): + """Error starting up Cuckoo.""" + +class CuckooDatabaseError(CuckooCriticalError): + """Cuckoo database error.""" + +class CuckooDependencyError(CuckooCriticalError): + """Missing dependency error.""" + +class CuckooConfigurationError(CuckooCriticalError): + """Invalid configuration error.""" + +class CuckooOperationalError(Exception): + """Cuckoo operation error.""" + +class CuckooMachineError(CuckooOperationalError): + """Error managing analysis machine.""" + +class CuckooMissingMachineError(CuckooMachineError): + """No such machine exists.""" + +class CuckooMachineSnapshotError(CuckooMachineError): + """Error restoring snapshot from machine.""" + +class CuckooAnalysisError(CuckooOperationalError): + """Error during analysis.""" + +class CuckooProcessingError(CuckooOperationalError): + """Error in processor module.""" + +class CuckooReportError(CuckooOperationalError): + """Error in reporting module.""" + +class CuckooGuestError(CuckooOperationalError): + """Cuckoo guest agent error.""" + +class CuckooGuestCriticalTimeout(CuckooGuestError): + """The Host was unable to connect to the Guest.""" + +class CuckooResultError(CuckooOperationalError): + """Cuckoo result server error.""" + +class CuckooDisableModule(CuckooOperationalError): + """Exception for disabling a module dynamically.""" + +class CuckooFeedbackError(CuckooOperationalError): + """Error in feedback module.""" + +class CuckooApiError(CuckooOperationalError): + """Error during API usage.""" diff --git a/cuckoo/common/files.py b/cuckoo/common/files.py new file mode 100644 index 0000000000..f67de03f15 --- /dev/null +++ b/cuckoo/common/files.py @@ -0,0 +1,190 @@ +# Copyright (C) 2016-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import os +import hashlib +import tempfile +import ntpath +import shutil +import errno + +from cuckoo.common.config import config +from cuckoo.common.exceptions import CuckooOperationalError +from cuckoo.misc import getuser + +def temppath(): + """Return the true temporary directory.""" + tmppath = config("cuckoo:cuckoo:tmppath") + + # Backwards compatibility with older configuration. + if not tmppath or tmppath == "/tmp": + return os.path.join( + tempfile.gettempdir(), "cuckoo-tmp-%s" % getuser() + ) + + return tmppath + +def open_exclusive(path, mode='wb', bufsize=-1): + """Open a file with O_EXCL, failing if it already exists + [In Python 3, use open with x]""" + fd = os.open(path, os.O_CREAT|os.O_EXCL|os.O_WRONLY, 0644) + try: + return os.fdopen(fd, mode, bufsize) + except: + os.close(fd) + raise + +class Storage(object): + @staticmethod + def get_filename_from_path(path): + """Cross-platform filename extraction from path. + @param path: file path. + @return: filename. + """ + dirpath, filename = ntpath.split(path) + return filename if filename else ntpath.basename(dirpath) + +class Folders(Storage): + @staticmethod + def create(root=".", folders=None): + """Create a directory or multiple directories. + @param root: root path. + @param folders: folders list to be created. + @raise CuckooOperationalError: if fails to create folder. + If folders is None, we try to create the folder provided by `root`. + """ + if isinstance(root, (tuple, list)): + root = os.path.join(*root) + + if folders is None: + folders = [""] + elif isinstance(folders, basestring): + folders = folders, + + for folder in folders: + folder_path = os.path.join(root, folder) + if not os.path.isdir(folder_path): + try: + os.makedirs(folder_path) + except OSError as e: + if e.errno == errno.EEXIST: + # Race condition, ignore + continue + raise CuckooOperationalError( + "Unable to create folder: %s" % folder_path + ) + + @staticmethod + def copy(src, dest): + if os.path.exists(dest): + shutil.rmtree(dest) + shutil.copytree(src, dest) + + @staticmethod + def create_temp(path=None): + return tempfile.mkdtemp(dir=path or temppath()) + + @staticmethod + def delete(*folder): + """Delete a folder and all its subdirectories. + @param folder: path or components to path to delete. + @raise CuckooOperationalError: if fails to delete folder. + """ + folder = os.path.join(*folder) + if os.path.exists(folder): + try: + shutil.rmtree(folder) + except OSError: + raise CuckooOperationalError( + "Unable to delete folder: %s" % folder + ) + +class Files(Storage): + @staticmethod + def temp_put(content, path=None): + """Store a temporary file or files. + @param content: the content of this file + @param path: directory path to store the file + """ + fd, filepath = tempfile.mkstemp( + prefix="upload_", dir=path or temppath() + ) + + if hasattr(content, "read"): + chunk = content.read(1024) + while chunk: + os.write(fd, chunk) + chunk = content.read(1024) + else: + os.write(fd, content) + + os.close(fd) + return filepath + + @staticmethod + def temp_named_put(content, filename, path=None): + """Store a named temporary file. + @param content: the content of this file + @param filename: filename that the file should have + @param path: directory path to store the file + @return: full path to the temporary file + """ + filename = Storage.get_filename_from_path(filename) + dirpath = tempfile.mkdtemp(dir=path or temppath()) + Files.create(dirpath, filename, content) + return os.path.join(dirpath, filename) + + @staticmethod + def create(root, filename, content): + if isinstance(root, (tuple, list)): + root = os.path.join(*root) + + filepath = os.path.join(root, filename) + with open(filepath, "wb") as f: + if hasattr(content, "read"): + chunk = content.read(1024 * 1024) + while chunk: + f.write(chunk) + chunk = content.read(1024 * 1024) + else: + f.write(content) + return filepath + + @staticmethod + def copy(path_target, path_dest): + """Copy a file. The destination may be a directory. + @param path_target: The + @param path_dest: path_dest + @return: path to the file or directory + """ + shutil.copy(src=path_target, dst=path_dest) + return os.path.join(path_dest, os.path.basename(path_target)) + + @staticmethod + def hash_file(method, filepath): + """Calculate a hash on a file by path. + @param method: callable hashing method + @param path: file path + @return: computed hash string + """ + f = open(filepath, "rb") + h = method() + while True: + buf = f.read(1024 * 1024) + if not buf: + break + h.update(buf) + return h.hexdigest() + + @staticmethod + def md5_file(filepath): + return Files.hash_file(hashlib.md5, filepath) + + @staticmethod + def sha1_file(filepath): + return Files.hash_file(hashlib.sha1, filepath) + + @staticmethod + def sha256_file(filepath): + return Files.hash_file(hashlib.sha256, filepath) diff --git a/lib/cuckoo/common/irc.py b/cuckoo/common/irc.py similarity index 79% rename from lib/cuckoo/common/irc.py rename to cuckoo/common/irc.py index 198682e3fa..ccd60145ed 100644 --- a/lib/cuckoo/common/irc.py +++ b/cuckoo/common/irc.py @@ -1,15 +1,14 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. # Copyright (C) 2012 JoseMi Holguin (@j0sm1) +# Copyright (C) 2013 Claudio Guarnieri. +# Copyright (C) 2014-2017 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. -"""IRC Protocol""" - import cStringIO import re import logging -from lib.cuckoo.common.utils import convert_to_printable +from cuckoo.common.utils import convert_to_printable log = logging.getLogger("Processing.Pcap.irc.protocol") @@ -18,20 +17,21 @@ class ircMessage(object): """IRC Protocol Request.""" # Client commands - __methods_client = dict.fromkeys(( "PASS", "JOIN", "USER", "OPER", "MODE", "SERVICE", "QUIT", "SQUIT", - "PART", "TOPIC", "NAMES", "LIST", "INVITE", - "KICK", "PRIVMSG", "NOTICE", "MOTD", "LUSERS", "VERSION", "STATS", "LINKS", "TIME", "CONNECT", - "TRACE", "ADMIN", "INFO", "SERVLIST", - "SQUERY", "WHO", "WHOIS", "WHOWAS", "KILL", "PING", "PONG", "ERROR", "AWAY", "REHASH", "DIE", "RESTART", - "SUMMON", "USERS", "WALLOPS", - "USERHOST", "NICK", "ISON" + __methods_client = dict.fromkeys(( + "PASS", "JOIN", "USER", "OPER", "MODE", "SERVICE", "QUIT", "SQUIT", + "PART", "TOPIC", "NAMES", "LIST", "INVITE", "KICK", "PRIVMSG", + "NOTICE", "MOTD", "LUSERS", "VERSION", "STATS", "LINKS", "TIME", + "CONNECT", "TRACE", "ADMIN", "INFO", "SERVLIST", "SQUERY", "WHO", + "WHOIS", "WHOWAS", "KILL", "PING", "PONG", "ERROR", "AWAY", "REHASH", + "DIE", "RESTART", "SUMMON", "USERS", "WALLOPS", "USERHOST", "NICK", + "ISON" )) def __init__(self): self._messages = [] - # Server commandis : prefix - command - params + # Server commandis : prefix - command - params self._sc = {} - # Client commands : command - params + # Client commands : command - params self._cc = {} def _unpack(self, buf): @@ -49,7 +49,9 @@ def _unpack(self, buf): if not re.match("^:", element) is None: command = "([a-zA-Z]+|[0-9]{3})" params = "(\x20.+)" - irc_server_msg = re.findall("(^:[\w+.{}!@|()]+\x20)"+command+params,element) + irc_server_msg = re.findall( + "(^:[\w+.{}!@|()]+\x20)" + command + params, element + ) if irc_server_msg: self._sc["prefix"] = convert_to_printable(irc_server_msg[0][0].strip()) self._sc["command"] = convert_to_printable(irc_server_msg[0][1].strip()) @@ -57,13 +59,15 @@ def _unpack(self, buf): self._sc["type"] = "server" self._messages.append(dict(self._sc)) else: - irc_client_msg = re.findall("([a-zA-Z]+\x20)(.+[\x0a\0x0d])",element) + irc_client_msg = re.findall( + "([a-zA-Z]+\x20)(.+[\x0a\0x0d])", element + ) if irc_client_msg and irc_client_msg[0][0].strip() in self.__methods_client: self._cc["command"] = convert_to_printable(irc_client_msg[0][0].strip()) self._cc["params"] = convert_to_printable(irc_client_msg[0][1].strip()) self._cc["type"] = "client" self._messages.append(dict(self._cc)) - + def getClientMessages(self, buf): """Get irc client commands of tcp streams. @buf: list of messages diff --git a/cuckoo/common/logo.py b/cuckoo/common/logo.py new file mode 100644 index 0000000000..f41382d0c7 --- /dev/null +++ b/cuckoo/common/logo.py @@ -0,0 +1,148 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import sys +import random + +from cuckoo.common.colors import color, yellow +from cuckoo.misc import version + +def logo(): + """Cuckoo asciiarts. + @return: asciiarts array. + """ + logos = [] + + logos.append(""" + _| + _|_|_| _| _| _|_|_| _| _| _|_| _|_| + _| _| _| _| _|_| _| _| _| _| + _| _| _| _| _| _| _| _| _| _| + _|_|_| _|_|_| _|_|_| _| _| _|_| _|_|""") + + logos.append(""" + __ + .----..--.--..----.| |--..-----..-----. + | __|| | || __|| < | _ || _ | + |____||_____||____||__|__||_____||_____|""") + + logos.append(""" + .: + :: + .-. , : .-. ;;.-. .-. .-. + ; ; ; ; ;; .' ; ;'; ;' + `;;;;'.'`..:;._`;;;;'_.'` `.`;;' `;;'""") + + logos.append(""" + eeee e e eeee e e eeeee eeeee + 8 8 8 8 8 8 8 8 8 88 8 88 + 8e 8e 8 8e 8eee8e 8 8 8 8 + 88 88 8 88 88 8 8 8 8 8 + 88e8 88ee8 88e8 88 8 8eee8 8eee8""") + + logos.append(""" + _____________________________________/\/\_______________________________ + ___/\/\/\/\__/\/\__/\/\____/\/\/\/\__/\/\__/\/\____/\/\/\______/\/\/\___ + _/\/\________/\/\__/\/\__/\/\________/\/\/\/\____/\/\__/\/\__/\/\__/\/\_ + _/\/\________/\/\__/\/\__/\/\________/\/\/\/\____/\/\__/\/\__/\/\__/\/\_ + ___/\/\/\/\____/\/\/\/\____/\/\/\/\__/\/\__/\/\____/\/\/\______/\/\/\___ + ________________________________________________________________________""") + + logos.append(""" + _______ _ _ _______ _ _ _____ _____ + | | | | |____/ | | | | + |_____ |_____| |_____ | \\_ |_____| |_____|""") + + logos.append(""" + _ + ____ _ _ ____| | _ ___ ___ + / ___) | | |/ ___) |_/ ) _ \ / _ \\ + ( (___| |_| ( (___| _ ( |_| | |_| | + \\____)____/ \\____)_| \\_)___/ \\___/""") + + logos.append(""" + ______ __ __ ______ ___ ___ ______ ______ + /_____/\\ /_/\\/_/\\ /_____/\\ /___/\\/__/\\ /_____/\\ /_____/\\ + \\:::__\\/ \\:\\ \\:\\ \\\\:::__\\/ \\::.\\ \\\\ \\ \\\\:::_ \\ \\\\:::_ \\ \\ + \\:\\ \\ __\\:\\ \\:\\ \\\\:\\ \\ __\\:: \\/_) \\ \\\\:\\ \\ \\ \\\\:\\ \\ \\ \\ + \\:\\ \\/_/\\\\:\\ \\:\\ \\\\:\\ \\/_/\\\\:. __ ( ( \\:\\ \\ \\ \\\\:\\ \\ \\ \\ + \\:\\_\\ \\ \\\\:\\_\\:\\ \\\\:\\_\\ \\ \\\\: \\ ) \\ \\ \\:\\_\\ \\ \\\\:\\_\\ \\ \\ + \\_____\\/ \\_____\\/ \\_____\\/ \\__\\/\\__\\/ \\_____\\/ \\_____\\/""") + + logos.append(""" + sSSs .S S. sSSs .S S. sSSs_sSSs sSSs_sSSs + d%%SP .SS SS. d%%SP .SS SS. d%%SP~YS%%b d%%SP~YS%%b + d%S' S%S S%S d%S' S%S S&S d%S' `S%b d%S' `S%b + S%S S%S S%S S%S S%S d*S S%S S%S S%S S%S + S&S S&S S&S S&S S&S .S*S S&S S&S S&S S&S + S&S S&S S&S S&S S&S_sdSSS S&S S&S S&S S&S + S&S S&S S&S S&S S&S~YSSY%b S&S S&S S&S S&S + S&S S&S S&S S&S S&S `S% S&S S&S S&S S&S + S*b S*b d*S S*b S*S S% S*b d*S S*b d*S + S*S. S*S. .S*S S*S. S*S S& S*S. .S*S S*S. .S*S + SSSbs SSSbs_sdSSS SSSbs S*S S& SSSbs_sdSSS SSSbs_sdSSS + YSSP YSSP~YSSY YSSP S*S SS YSSP~YSSY YSSP~YSSY + SP + Y""") + + logos.append(""" + _______ _____ _____ + /::\\ \\ /\\ \\ /\\ \\ + /::::\\ \\ /::\\____\\ /::\\ \\ + /::::::\\ \\ /::::| | /::::\\ \\ + /::::::::\\ \\ /:::::| | /::::::\\ \\ + /:::/~~\\:::\\ \\ /::::::| | /:::/\\:::\\ \\ + /:::/ \\:::\\ \\ /:::/|::| | /:::/ \\:::\\ \\ + /:::/ / \\:::\\ \\ /:::/ |::| | /:::/ \\:::\\ \\ + /:::/____/ \\:::\\____\\ /:::/ |::|___|______ /:::/ / \\:::\\ \\ + |:::| | |:::| | /:::/ |::::::::\\ \\ /:::/ / \\:::\\ ___\\ + |:::|____| |:::| |/:::/ |:::::::::\\____\\/:::/____/ ___\\:::| | + \\:::\\ \\ /:::/ / \\::/ / ~~~~~/:::/ /\\:::\\ \\ /\\ /:::|____| + \\:::\\ \\ /:::/ / \\/____/ /:::/ / \\:::\\ /::\\ \\::/ / + \\:::\\ /:::/ / /:::/ / \\:::\\ \\:::\\ \\/____/ + \\:::\\__/:::/ / /:::/ / \\:::\\ \\:::\\____\\ + \\::::::::/ / /:::/ / \\:::\\ /:::/ / + \\::::::/ / /:::/ / \\:::\\/:::/ / + \\::::/ / /:::/ / \\::::::/ / + \\::/____/ /:::/ / \\::::/ / + ~~ \\::/ / \\::/____/ + \\/____/ + it's Cuckoo!""") + + logos.append(""" + _ _ _ _ _ _ + /\\ \\ /\\_\\ /\\ \\ /\\_\\ /\\ \\ /\\ \\ + / \\ \\ / / / _ / \\ \\ / / / _ / \\ \\ / \\ \\ + / /\\ \\ \\ \\ \\ \\__ /\\_\\ / /\\ \\ \\ / / / /\\_\\ / /\\ \\ \\ / /\\ \\ \\ + / / /\\ \\ \\ \\ \\___\\ / / // / /\\ \\ \\ / / /__/ / / / / /\\ \\ \\ / / /\\ \\ \\ + / / / \\ \\_\\ \\__ / / / // / / \\ \\_\\ / /\\_____/ / / / / \\ \\_\\ / / / \\ \\_\\ + / / / \\/_/ / / / / / // / / \\/_/ / /\\_______/ / / / / / // / / / / / + / / / / / / / / // / / / / /\\ \\ \\ / / / / / // / / / / / + / / /________ / / /___/ / // / /________ / / / \\ \\ \\ / / /___/ / // / /___/ / / + / / /_________\\/ / /____\\/ // / /_________\\/ / / \\ \\ \\ / / /____\\/ // / /____\\/ / + \\/____________/\\/_________/ \\/____________/\\/_/ \\_\\_\\\\/_________/ \\/_________/""") + + logos.append(""" + ),-. / + Cuckoo Sandbox <(a `---',' + no chance for malwares! ( `-, ._> ) + ) _>.___/ + _/""") + + logos.append(""" + .-----------------. + | Cuckoo Sandbox? | + | OH NOES! |\\ '-.__.-' + '-----------------' \\ /oo |--.--,--,--. + \\_.-'._i__i__i_.' + \"\"\"\"\"\"\"\"\"""") + + print(color(random.choice(logos), random.randrange(31, 37))) + print + print(" Cuckoo Sandbox %s" % yellow(version)) + print(" www.cuckoosandbox.org") + print(" Copyright (c) 2010-2018") + print + sys.stdout.flush() diff --git a/cuckoo/common/mongo.py b/cuckoo/common/mongo.py new file mode 100644 index 0000000000..89b0f43a89 --- /dev/null +++ b/cuckoo/common/mongo.py @@ -0,0 +1,74 @@ +# Copyright (C) 2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import gridfs +import logging +import pymongo +import socket + +from cuckoo.common.config import config +from cuckoo.common.exceptions import CuckooCriticalError + +log = logging.getLogger(__name__) + +class Mongo(object): + def __init__(self): + self.client = None + self.db = None + + self.enabled = None + self.hostname = None + self.port = None + self.database = None + self.username = None + self.password = None + self.grid = None + + def init(self): + self.enabled = config("reporting:mongodb:enabled") + self.hostname = config("reporting:mongodb:host") + self.port = config("reporting:mongodb:port") + self.database = config("reporting:mongodb:db") + self.username = config("reporting:mongodb:username") + self.password = config("reporting:mongodb:password") + return self.enabled + + def drop(self): + self.client.drop_database(self.database) + + def close(self): + self.client.close() + + def connect(self): + if not self.enabled: + return + + # Warn the user that this may take a while if an instant connection + # could not be made with the MongoDB server. + try: + socket.create_connection((self.hostname, self.port), 1).close() + except socket.error: + log.warning( + "We're attempting to connect to MongoDB, but the connection " + "seems slow or MongoDB is simply offline. Please wait while " + "Cuckoo tries to connect.." + ) + + try: + self.client = pymongo.MongoClient(self.hostname, self.port) + self.db = self.client[self.database] + if self.username and self.password: + self.db.authenticate(self.username, self.password) + self.grid = gridfs.GridFS(self.db) + + # Fetch the collection names to force Mongo to connect. + self.collection_names = self.db.collection_names() + except pymongo.errors.PyMongoError as e: + raise CuckooCriticalError( + "Unable to connect to MongoDB: %s. In order to operate " + "Cuckoo as per your configuration, a running MongoDB server " + "is required." % e + ) + +mongo = Mongo() diff --git a/lib/cuckoo/common/netlog.py b/cuckoo/common/netlog.py similarity index 78% rename from lib/cuckoo/common/netlog.py rename to cuckoo/common/netlog.py index 1974a1686b..69f70cbf38 100644 --- a/lib/cuckoo/common/netlog.py +++ b/cuckoo/common/netlog.py @@ -1,29 +1,26 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2019 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. +import bson import datetime import hashlib import logging import os.path import struct -try: - import bson - HAVE_BSON = True -except ImportError: - HAVE_BSON = False -else: - # The BSON module provided by pymongo works through its "BSON" class. - if hasattr(bson, "BSON"): - bson_decode = lambda d: bson.BSON(d).decode() - # The BSON module provided by "pip install bson" works through the - # "loads" function (just like pickle etc.) - elif hasattr(bson, "loads"): - bson_decode = lambda d: bson.loads(d) - -from lib.cuckoo.common.utils import get_filename_from_path -from lib.cuckoo.common.exceptions import CuckooResultError +# The BSON module provided by pymongo works through its "BSON" class. +if hasattr(bson, "BSON"): + bson_decode = lambda d: bson.BSON(d).decode() +# The BSON module provided by "pip install bson" works through the +# "loads" function (just like pickle etc.) +elif hasattr(bson, "loads"): + bson_decode = lambda d: bson.loads(d) + +from cuckoo.common.files import Storage +from cuckoo.common.exceptions import CuckooResultError +from cuckoo.misc import cwd log = logging.getLogger(__name__) @@ -58,13 +55,14 @@ def default_converter_64bit(v): return v class BsonParser(object): - """Handle .bson logs from monitor. Basically we would like to directly pass through - the parsed data structures, but the .bson logs need a bit special handling to be more space efficient. + """Interprets .bson logs from the monitor. - Basically we get "info" messages that explain how the function arguments will come through later on. - This class remembers these info mappings and then transforms the api call messages accordingly. + The monitor provides us with "info" messages that explain how the function + arguments will come through later on. This class remembers these info + mappings and then transforms the api call messages accordingly. - Other message types typically get passed through after renaming the keys slightly. + Other message types typically get passed through after renaming the + keys slightly. """ converters_32bit = { None: default_converter_32bit, @@ -78,7 +76,7 @@ class BsonParser(object): "x": pointer_converter_32bit, } - def __init__(self, fd): + def __init__(self, fd, task_id=None): self.fd = fd self.infomap = {} self.flags_value = {} @@ -86,12 +84,7 @@ def __init__(self, fd): self.pid = None self.is_64bit = False self.buffer_sha1 = None - - if not HAVE_BSON: - log.critical("Starting BsonParser, but bson is not available! (install with `pip install bson`)") - - def close(self): - pass + self.task_id = task_id def resolve_flags(self, apiname, argdict, flags): # Resolve 1:1 values. @@ -124,9 +117,9 @@ def resolve_flags(self, apiname, argdict, flags): flags[argument] = "|".join(flags[argument]) def determine_unserializers(self, arginfo): - """Determines which unserializers (or converters) have to be used in - order to parse the various arguments for this function call. Keeps in - mind whether the current bson is 32-bit or 64-bit.""" + """Determine which unserializers (or converters) have to be used in + order to parse the various arguments for this function call. Maintains + whether the current bson is 32-bit or 64-bit.""" argnames, converters = [], [] for argument in arginfo: @@ -159,8 +152,10 @@ def __iter__(self): blen = struct.unpack("I", data)[0] if blen > MAX_MESSAGE_LENGTH: - log.critical("BSON message larger than MAX_MESSAGE_LENGTH, " - "stopping handler.") + log.critical( + "BSON message larger than MAX_MESSAGE_LENGTH, " + "stopping handler." + ) return data += self.fd.read(blen-4) @@ -171,8 +166,10 @@ def __iter__(self): try: dec = bson_decode(data) except Exception as e: - log.warning("BsonParser decoding problem {0} on " - "data[:50] {1}".format(e, repr(data[:50]))) + log.warning( + "BsonParser decoding problem %s on data[:50] %s", + e, repr(data[:50]) + ) return mtype = dec.get("type", "none") @@ -208,16 +205,11 @@ def __iter__(self): if sha1 != self.buffer_sha1: log.warning("Incorrect sha1 passed along for a buffer.") - # If the parent is netlogs ResultHandler then we actually dump - # it - this should only be the case during the analysis, any - # after proposing will then be ignored. - from lib.cuckoo.core.resultserver import ResultHandler - - if isinstance(self.fd, ResultHandler): - filepath = os.path.join(self.fd.storagepath, - "buffer", self.buffer_sha1) - with open(filepath, "wb") as f: - f.write(buf) + filepath = cwd( + "buffer", self.buffer_sha1, analysis=self.task_id + ) + with open(filepath, "wb") as f: + f.write(buf) continue @@ -231,9 +223,8 @@ def __iter__(self): } if mtype == "debug": - log.info("Debug message from monitor: {0}".format(dec.get("msg", ""))) parsed["message"] = dec.get("msg", "") - + log.info("Debug message from monitor: %s", parsed["message"]) else: # Regular api call from monitor if index not in self.infomap: @@ -245,9 +236,10 @@ def __iter__(self): args = dec.get("args", []) if len(args) != len(argnames): - log.warning("Inconsistent arg count (compared to arg names) " - "on {2}: {0} names {1}".format(dec, argnames, - apiname)) + log.warning( + "Inconsistent arg count (compared to arg names) " + "on %s: %s names %s", dec, argnames, apiname + ) continue argdict = {} @@ -280,7 +272,9 @@ def __iter__(self): modulepath = argdict["module_path"] else: - raise CuckooResultError("I don't recognise the bson log contents.") + raise CuckooResultError( + "I don't recognize the bson log contents." + ) # FILETIME is 100-nanoseconds from 1601 :/ vmtimeunix = (timelow + (timehigh << 32)) @@ -288,7 +282,7 @@ def __iter__(self): vmtime = datetime.datetime.fromtimestamp(vmtimeunix) parsed["first_seen"] = vmtime - procname = get_filename_from_path(modulepath) + procname = Storage.get_filename_from_path(modulepath) parsed["process_path"] = modulepath parsed["process_name"] = procname parsed["command_line"] = argdict.get("command_line") @@ -299,6 +293,7 @@ def __iter__(self): # Is this process being "tracked"? parsed["track"] = bool(argdict.get("track", 1)) + parsed["modules"] = argdict.get("modules", {}) self.pid = pid @@ -311,7 +306,9 @@ def __iter__(self): # msg = argdict["Message"] # self.handler.log_anomaly(subcategory, tid, msg) # return True - + elif apiname == "__action__": + parsed["type"] = "action" + parsed["action"] = argdict["action"] else: parsed["type"] = "apicall" parsed["pid"] = self.pid diff --git a/cuckoo/common/objects.py b/cuckoo/common/objects.py new file mode 100644 index 0000000000..f8b40d880b --- /dev/null +++ b/cuckoo/common/objects.py @@ -0,0 +1,474 @@ +# Copyright (C) 2012-2013 Claudio Guarnieri. +# Copyright (C) 2014-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import base64 +import binascii +import hashlib +import logging +import mmap +import os +import pefile +import re +import sflock +import shutil +import tempfile +import zipfile + +from cuckoo.common.safelist import is_safelisted_domain + +try: + import pydeep + HAVE_PYDEEP = True +except ImportError: + HAVE_PYDEEP = False + +log = logging.getLogger(__name__) + +FILE_CHUNK_SIZE = 16*1024*1024 + +URL_REGEX = ( + # HTTP/HTTPS. + "(https?:\\/\\/)" + "(([" + # IP address. + "(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\." + "(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\." + "(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\." + "(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])]|" + # Or domain name. + "[a-zA-Z0-9\\.-]+)" + # Optional port. + "(\\:\\d+)?" + # URI. + "(/[\\(\\)a-zA-Z0-9_:%?=/\\.-]*)?" +) + +PUBPRIVKEY = ( + "(" + "(?:-----BEGIN PUBLIC KEY-----" + "[a-zA-Z0-9\\n\\+/]+" + "-----END PUBLIC KEY-----)" + "|" + "(?:-----BEGIN RSA PRIVATE KEY-----" + "[a-zA-Z0-9\\n\\+/]+" + "-----END RSA PRIVATE KEY-----)" + ")" +) + +class Dictionary(dict): + """Cuckoo custom dict.""" + + def __getattr__(self, key): + return self.get(key, None) + + __setattr__ = dict.__setitem__ + __delattr__ = dict.__delitem__ + +class URL: + """URL base object.""" + + def __init__(self, url): + """@param url: URL""" + self.url = url + +class File(object): + """Basic file object class with all useful utilities.""" + # Given that ssdeep hashes are not really used much in practice we're just + # going to disable its warning by default for now. + notified_pydeep = True + + # The yara rules should not change during one Cuckoo run and as such we're + # caching 'em. This dictionary is filled during init_yara(). + yara_rules = {} + + def __init__(self, file_path, temporary=False): + """@param file_path: file path.""" + self.file_path = file_path + self.temporary = temporary + + # these will be populated when first accessed + self._file_data = None + self._crc32 = None + self._md5 = None + self._sha1 = None + self._sha256 = None + self._sha512 = None + + def __del__(self): + self.temporary and os.unlink(self.file_path) + + def get_name(self): + """Get file name. + @return: file name. + """ + file_name = os.path.basename(self.file_path) + return file_name + + def valid(self): + return ( + self.file_path and + os.path.exists(self.file_path) and + os.path.isfile(self.file_path) and + os.path.getsize(self.file_path) != 0 + ) + + def get_data(self): + """Read file contents. + @return: data. + """ + return self.file_data + + def get_chunks(self): + """Read file contents in chunks (generator).""" + + with open(self.file_path, "rb") as fd: + while True: + chunk = fd.read(FILE_CHUNK_SIZE) + if not chunk: + break + yield chunk + + def calc_hashes(self): + """Calculate all possible hashes for this file.""" + crc = 0 + md5 = hashlib.md5() + sha1 = hashlib.sha1() + sha256 = hashlib.sha256() + sha512 = hashlib.sha512() + + for chunk in self.get_chunks(): + crc = binascii.crc32(chunk, crc) + md5.update(chunk) + sha1.update(chunk) + sha256.update(chunk) + sha512.update(chunk) + + self._crc32 = "".join("%02X" % ((crc >> i) & 0xff) + for i in [24, 16, 8, 0]) + self._md5 = md5.hexdigest() + self._sha1 = sha1.hexdigest() + self._sha256 = sha256.hexdigest() + self._sha512 = sha512.hexdigest() + + @property + def file_data(self): + if not self._file_data: + self._file_data = open(self.file_path, "rb").read() + return self._file_data + + def get_size(self): + """Get file size. + @return: file size. + """ + return os.path.getsize(self.file_path) + + def get_crc32(self): + """Get CRC32. + @return: CRC32. + """ + if not self._crc32: + self.calc_hashes() + return self._crc32 + + def get_md5(self): + """Get MD5. + @return: MD5. + """ + if not self._md5: + self.calc_hashes() + return self._md5 + + def get_sha1(self): + """Get SHA1. + @return: SHA1. + """ + if not self._sha1: + self.calc_hashes() + return self._sha1 + + def get_sha256(self): + """Get SHA256. + @return: SHA256. + """ + if not self._sha256: + self.calc_hashes() + return self._sha256 + + def get_sha512(self): + """ + Get SHA512. + @return: SHA512. + """ + if not self._sha512: + self.calc_hashes() + return self._sha512 + + def get_ssdeep(self): + """Get SSDEEP. + @return: SSDEEP. + """ + if not HAVE_PYDEEP: + if not File.notified_pydeep: + File.notified_pydeep = True + log.warning("Unable to import pydeep (install with `pip install pydeep`)") + return None + + try: + return pydeep.hash_file(self.file_path) + except Exception: + return None + + def get_type(self): + """Get MIME file type. + @return: file type. + """ + return sflock.magic.from_file( + os.path.realpath(self.file_path) + ) + + def get_content_type(self): + """Get MIME content file type (example: image/jpeg). + @return: file content type. + """ + return sflock.magic.from_file( + os.path.realpath(self.file_path), mime=True + ) + + def get_exported_functions(self): + """Get the exported function names of this PE file.""" + filetype = self.get_type() + if "MS-DOS" not in filetype and "PE32" not in self.get_type(): + return + + try: + pe = pefile.PE(self.file_path) + if not hasattr(pe, "DIRECTORY_ENTRY_EXPORT"): + return + + for export in pe.DIRECTORY_ENTRY_EXPORT.symbols: + if export.name: + yield export.name + except Exception as e: + log.warning("Error enumerating exported functions: %s", e) + + def get_imported_functions(self): + """Get the imported functions of this PE file.""" + filetype = self.get_type() + if "MS-DOS" not in filetype and "PE32" not in self.get_type(): + return + + try: + pe = pefile.PE(self.file_path) + if not hasattr(pe, "DIRECTORY_ENTRY_IMPORT"): + return + + for imp in pe.DIRECTORY_ENTRY_IMPORT: + for entry in imp.imports: + yield dict(dll=imp.dll, + name=entry.name, + ordinal=entry.ordinal, + hint=entry.hint, + address=entry.address) + except Exception as e: + log.warning("Error enumerating imported functions: %s", e) + + def get_apk_entry(self): + """Get the entry point for this APK. The entry point is denoted by a + package and main activity name.""" + filetype = self.get_type() + if "Zip archive data" not in filetype and "Java archive data" not in filetype: + return "", "" + + from androguard.core.bytecodes.apk import APK + + try: + a = APK(self.file_path) + if not a.is_valid_APK(): + return "", "" + + package = a.get_package() + if not package: + log.warning("Unable to find the main package, this analysis " + "will probably fail.") + return "", "" + + main_activity = a.get_main_activity() + if main_activity: + log.debug("Picked package %s and main activity %s.", + package, main_activity) + return package, main_activity + + activities = a.get_activities() + for activity in activities: + if "main" in activity or "start" in activity: + log.debug("Choosing package %s and main activity due to " + "its name %s.", package, activity) + return package, activity + + if activities and activities[0]: + log.debug("Picked package %s and the first activity %s.", + package, activities[0]) + return package, activities[0] + except Exception as e: + log.warning("Error extracting package and main activity: %s.", e) + + return "", "" + + def get_yara(self, category="binaries", externals=None): + """Get Yara signatures matches. + @return: matched Yara signatures. + """ + if not os.path.getsize(self.file_path): + return [] + + try: + # TODO Once Yara obtains proper Unicode filepath support we can + # remove this check. See also the following Github issue: + # https://github.com/VirusTotal/yara-python/issues/48 + assert len(str(self.file_path)) == len(self.file_path) + except (UnicodeEncodeError, AssertionError): + log.warning( + "Can't run Yara rules on %r as Unicode paths are currently " + "not supported in combination with Yara!", self.file_path + ) + return [] + + results, rule = [], File.yara_rules[category] + for match in rule.match(self.file_path, externals=externals): + strings, offsets = set(), {} + for _, key, value in match.strings: + strings.add(base64.b64encode(value)) + offsets[key.lstrip("$")] = [] + + strings = sorted(strings) + for offset, key, value in match.strings: + offsets[key.lstrip("$")].append( + (offset, strings.index(base64.b64encode(value))) + ) + + meta = { + "description": "(no description)", + } + meta.update(match.meta) + + results.append({ + "name": match.rule, + "meta": meta, + "strings": strings, + "offsets": offsets, + }) + + return results + + def mmap(self, fileno): + if hasattr(mmap, "PROT_READ"): + access = mmap.PROT_READ + elif hasattr(mmap, "ACCESS_READ"): + access = mmap.ACCESS_READ + else: + log.warning( + "Regexing through a file is not supported on your OS!" + ) + return + + return mmap.mmap(fileno, 0, access=access) + + def get_urls(self): + """Extract all URLs embedded in this file through a simple regex.""" + if not os.path.getsize(self.file_path): + return [] + + # http://stackoverflow.com/a/454589 + urls, f = set(), open(self.file_path, "rb") + for url in re.findall(URL_REGEX, self.mmap(f.fileno())): + if not is_safelisted_domain(url[1]): + urls.add("".join(url)) + return list(urls) + + def get_keys(self): + """Get any embedded plaintext public and/or private keys.""" + if not os.path.getsize(self.file_path): + return [] + + f = open(self.file_path, "rb") + return list(set(re.findall(PUBPRIVKEY, self.mmap(f.fileno())))) + + def get_all(self): + """Get all information available. + @return: information dict. + """ + infos = {} + infos["name"] = self.get_name() + infos["path"] = self.file_path + infos["size"] = self.get_size() + infos["crc32"] = self.get_crc32() + infos["md5"] = self.get_md5() + infos["sha1"] = self.get_sha1() + infos["sha256"] = self.get_sha256() + infos["sha512"] = self.get_sha512() + infos["ssdeep"] = self.get_ssdeep() + infos["type"] = self.get_type() + infos["yara"] = self.get_yara() + infos["urls"] = self.get_urls() + return infos + +class Archive(object): + def __init__(self, filepath): + self.filepath = filepath + self.z = zipfile.ZipFile(filepath) + + def get_file(self, filename): + filepath = tempfile.mktemp() + shutil.copyfileobj(self.z.open(filename), open(filepath, "wb")) + return File(filepath, temporary=True) + +class Buffer(object): + """A brief wrapper around string buffers for quick Yara rule matching.""" + + def __init__(self, buffer): + self.buffer = buffer + + def get_yara_quick(self, category, externals=None): + results, rule = [], File.yara_rules[category] + for match in rule.match(data=self.buffer, externals=externals): + results.append(match.rule) + return results + +class YaraMatch(object): + def __init__(self, match, category=None): + self.name = match["name"] + self.meta = match["meta"] + self._decoded = {} + self.offsets = match["offsets"] + self.category = category + + self._strings = [] + for s in match["strings"]: + self._strings.append(s.decode("base64")) + + def string(self, identifier, index=0): + off, idx = self.offsets[identifier][index] + return self._strings[idx] + + def strings(self, identifier): + ret = [] + for off, idx in self.offsets[identifier]: + ret.append(self._strings[idx]) + return ret + +class ExtractedMatch(object): + def __init__(self, match): + self.category = match["category"] + self.program = match.get("program") + self.first_seen = match.get("first_seen") + self.pid = match.get("pid") + + self.yara = [] + for ym in match["yara"]: + self.yara.append(YaraMatch(ym)) + + # Raw payload. + self.raw = match.get("raw") + self.info = match["info"] diff --git a/cuckoo/common/safelist.py b/cuckoo/common/safelist.py new file mode 100644 index 0000000000..a9eaef08b6 --- /dev/null +++ b/cuckoo/common/safelist.py @@ -0,0 +1,70 @@ +# Copyright (C) 2015-2019 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import os.path + +from cuckoo.misc import cwd + +domains = set() +ips = set() +mispdomains = set() +mispips = set() +mispurls = set() +misphashes = set() + +def _load_safelist(wlset, wl_file): + wl_path = cwd("safelist", wl_file) + + if not os.path.isfile(wl_path): + wl_path = cwd("..", "data", "safelist", wl_file, private=True) + + with open(wl_path, "rb") as fp: + safelist = fp.read() + + for entry in safelist.split("\n"): + entry = entry.strip() + if entry and not entry.startswith("#"): + wlset.add(entry) + +def is_safelisted_domain(domain): + if not domains: + # Initialize the domain safelist. + _load_safelist(domains, "domain.txt") + + return domain in domains + +def is_safelisted_ip(ip): + if not ips: + # Initialize the ip safelist. + _load_safelist(ips, "ip.txt") + + return ip in ips + +def is_safelisted_mispdomain(domain): + if not mispdomains: + # Initialize the misp domain safelist. + _load_safelist(mispdomains, "mispdomain.txt") + + return domain in mispdomains + +def is_safelisted_mispip(ip): + if not mispips: + # Initialize the misp ip safelist. + _load_safelist(mispips, "mispip.txt") + + return ip in mispips + +def is_safelisted_mispurl(url): + if not mispurls: + # Initialize the misp url safelist. + _load_safelist(mispurls, "mispurl.txt") + + return url in mispurls + +def is_safelisted_misphash(hash): + if not misphashes: + # Initialize the misp hash safelist. + _load_safelist(misphashes, "misphash.txt") + + return hash in misphashes diff --git a/cuckoo/common/scripting.py b/cuckoo/common/scripting.py new file mode 100644 index 0000000000..0f12c3dd0b --- /dev/null +++ b/cuckoo/common/scripting.py @@ -0,0 +1,190 @@ +# Copyright (C) 2017-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import binascii +import logging +import re +import shlex + +log = logging.getLogger(__name__) + +class Scripting(object): + program = None + ext = None + + def __init__(self, parent=None): + self.parent = parent + self.args = {} + self.children = [] + + def shlex(self, cmdline): + if isinstance(cmdline, (tuple, list)): + return cmdline + try: + return shlex.split(cmdline, posix=False) + except ValueError: + log.warning( + "Error parsing command-line: %s", cmdline.encode("utf8") + ) + return [] + + def match_command(self, program): + return bool(re.match(self.EXE_REGEX, program, re.I)) + + def parse_command(self, cmdline): + cmdline = self.shlex(cmdline) + if not cmdline: + return + + for cls in Scripting.__subclasses__(): + obj = cls(self) + if obj.match_command(cmdline[0]): + obj.args = obj.parse_command_line(cmdline) + self.children.append(obj) + return obj + + def astree(self): + return { + "args": self.args, + "children": [child.astree() for child in self.children], + } + + def parse_command_line(self, cmdline): + raise NotImplementedError + + def get_script(self): + raise NotImplementedError + +class CmdExe(Scripting): + EXE_REGEX = ( + "([\"]?C:(\\\\)+Windows(\\\\)+System32(\\\\)+)?" + "cmd(\\.exe)?[\"]?$" + ) + + program = "cmd" + ext = "bat" + + def parse_command_line(self, cmdline): + cmdline = self.shlex(cmdline) + + idx, ret = 1, {} + + # Strip off surrounding quotes. + if len(cmdline) == 2 and cmdline[1].startswith('"'): + cmdline = self.shlex(cmdline[1][1:-1]) + idx = 0 + + while idx < len(cmdline): + if cmdline[idx] == "/c" or cmdline[idx] == "/C": + ret["remains"] = False + ret["command"] = cmdline[idx+1:] + self.parse_command(cmdline[idx+1:]) + break + + if cmdline[idx] == "/k" or cmdline[idx] == "/K": + ret["remains"] = True + ret["command"] = cmdline[idx+1:] + self.parse_command(cmdline[idx+1:]) + break + + if cmdline[idx] == "/q" or cmdline[idx] == "/Q": + ret["quiet"] = True + idx += 1 + continue + + log.warning( + "Unhandled cmd.exe command-line argument(s): %s", + cmdline[idx:] + ) + idx += 1 + + return ret + + def get_script(self): + return " ".join(self.args.get("command", [])) + +def ps1_cmdarg(s, minimum=1): + """Create an exactly matching PowerShell command line argument regex, + instead of a regex that matches anything with the same characters.""" + return "".join( + "([%s%s^]" % (ch.lower(), ch.upper()) for ch in s + ) + ")?"*(len(s)-minimum) + ")"*minimum + +class PowerShell(Scripting): + EXE_REGEX = ( + "([\"]?C:(\\\\)+Windows(\\\\)+(System32|syswow64|sysnative)" + "(\\\\)+WindowsPowerShell(\\\\)+v1\\.0(\\\\)+)?" + "powershell(_ise)?(\\.exe)?" + "[\"]?$" + ) + + program = "powershell" + ext = "ps1" + + CMDLINE_REGEX = { + "command": "\\-[\\^]?%s$" % ps1_cmdarg("command"), + "encodedcommand": "\\-[\\^]?%s$" % ps1_cmdarg("encodedcommand"), + "windowstyle": "\\-[\\^]?%s$" % ps1_cmdarg("windowstyle"), + "noninteractive": "\\-[\\^]?%s$" % ps1_cmdarg("noninteractive", 4), + "noprofile": "\\-[\\^]?%s$" % ps1_cmdarg("noprofile", 3), + "executionpolicy": ( + "\\-[\\^]?([eE][pP]|%s)$" % ps1_cmdarg("executionpolicy", 2) + ), + "sta": "\\-[\\^]?sta$", + "noexit": "\\-[\\^]?noexit$", + "nologo": "\\-[\\^]?%s$" % ps1_cmdarg("nologo", 3), + } + + def _cmdparse_command(self, cmdline, idx): + return len(cmdline)-idx, " ".join(cmdline[idx+1:]) + + def _cmdparse_encodedcommand(self, cmdline, idx): + try: + return 1, cmdline[idx+1].decode("base64").decode("utf16") + except (IndexError, binascii.Error, UnicodeDecodeError): + pass + return 1, None + + def _cmdparse_windowstyle(self, cmdline, idx): + try: + if re.match(ps1_cmdarg("hidden", 3), cmdline[idx+1]): + return 1, "hidden" + except IndexError: + pass + return 1, None + + def _cmdparse_executionpolicy(self, cmdline, idx): + try: + return 1, cmdline[idx+1].lower() + except IndexError: + pass + return 1, None + + def parse_command_line(self, cmdline): + cmdline = self.shlex(cmdline) + + idx, ret = 1, {} + + while idx < len(cmdline): + for key, regex in self.CMDLINE_REGEX.items(): + if not re.match(regex, cmdline[idx]): + continue + + fn = getattr(self, "_cmdparse_%s" % key, None) + used, value = fn(cmdline, idx) if fn else (0, True) + + ret[key] = value + idx += used + 1 + break + else: + break + + # Handle trailing fields which are interpreted as commands. + if idx < len(cmdline) and not self.get_script(): + ret["command"] = " ".join(cmdline[idx:]) + + return ret + + def get_script(self): + return self.args.get("command") or self.args.get("encodedcommand") diff --git a/cuckoo/common/shellcode.py b/cuckoo/common/shellcode.py new file mode 100644 index 0000000000..205ee329e9 --- /dev/null +++ b/cuckoo/common/shellcode.py @@ -0,0 +1,77 @@ +# Copyright (C) 2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import logging +import unicorn +import unicorn.x86_const as x86 + +log = logging.getLogger(__name__) + +archs = { + "x86": unicorn.UC_ARCH_X86, +} + +modes = { + 32: unicorn.UC_MODE_32, +} + +class UcX86(unicorn.Uc): + @property + def esp(self): + return self.reg_read(x86.UC_X86_REG_ESP) + + @esp.setter + def esp(self, value): + self.reg_write(x86.UC_X86_REG_ESP, value) + +class ShellcodeX86(object): + def __init__(self, arch, mode, sc=None): + self.emu = UcX86(arch, mode) + self.addr = None + self.sc = sc + self.init() + + def init(self): + pass + + def map_memory(self, addr=0x1000, memsize=2*1024*1024): + self.addr = addr + self.emu.mem_map(addr, memsize) + self.emu.mem_write(addr, self.sc) + self.emu.esp = addr + memsize / 2 + + def run(self, addr=None, end=0, count=None): + try: + self.emu.emu_start(addr or self.addr, end, count=count) + except unicorn.UcError as e: + log.error("Error emulating shellcode: %s", e) + +class ShikataX86(ShellcodeX86): + def init(self): + self.bblcount = 0 + self.start = None + self.emu.hook_add(unicorn.UC_HOOK_BLOCK, self.hook_block) + + def hook_block(self, uc, addr, size, user_data): + if not size: + return + + self.bblcount += 1 + if self.bblcount == 2: + self.start = addr + return False + + def result(self): + start = self.start or self.addr + return self.emu.mem_read(start, len(self.sc) - start + self.addr) + +def Shellcode(arch="x86", mode=32, sc=None, cls=ShellcodeX86): + # TODO For now only 32-bit x86 shellcode is supported. + return cls(archs[arch], modes[mode], sc) + +def shikata(sc): + s = Shellcode(sc=sc, cls=ShikataX86) + s.map_memory() + s.run(count=0x1000) + return str(s.result()) diff --git a/cuckoo/common/structures.py b/cuckoo/common/structures.py new file mode 100644 index 0000000000..1870d8d51d --- /dev/null +++ b/cuckoo/common/structures.py @@ -0,0 +1,63 @@ +# Copyright (C) 2016-2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import ctypes + +class Structure(ctypes.Structure): + def as_dict(self): + ret = {} + for field, _ in self._fields_: + value = getattr(self, field) + if isinstance(value, Structure): + ret[field] = value.as_dict() + elif hasattr(value, "value"): + ret[field] = value + elif hasattr(value, "__getitem__"): + ret[field] = value[:] + else: + ret[field] = value + return ret + +class LnkHeader(Structure): + _fields_ = [ + ("signature", ctypes.c_ubyte * 4), + ("guid", ctypes.c_ubyte * 16), + ("flags", ctypes.c_uint), + ("attrs", ctypes.c_uint), + ("creation", ctypes.c_ulonglong), + ("access", ctypes.c_ulonglong), + ("modified", ctypes.c_ulonglong), + ("target_len", ctypes.c_uint), + ("icon_len", ctypes.c_uint), + ("show_window", ctypes.c_uint), + ("hotkey", ctypes.c_uint), + ] + +class LnkEntry(Structure): + _fields_ = [ + ("length", ctypes.c_uint), + ("first_offset", ctypes.c_uint), + ("volume_flags", ctypes.c_uint), + ("local_volume", ctypes.c_uint), + ("base_path", ctypes.c_uint), + ("net_volume", ctypes.c_uint), + ("path_remainder", ctypes.c_uint), + ] + +class PUBLICKEYSTRUC(Structure): + _pack_ = 1 + _fields_ = [ + ("type", ctypes.c_ubyte), + ("version", ctypes.c_ubyte), + ("reserved", ctypes.c_ushort), + ("algid", ctypes.c_uint), + ] + +class RSAPUBKEY(Structure): + _pack_ = 1 + _fields_ = [ + ("magic", ctypes.c_uint), + ("bitlen", ctypes.c_uint), + ("pubexp", ctypes.c_uint), + ] diff --git a/cuckoo/common/utils.py b/cuckoo/common/utils.py new file mode 100644 index 0000000000..2e9e4c8c9d --- /dev/null +++ b/cuckoo/common/utils.py @@ -0,0 +1,368 @@ +# Copyright (C) 2012-2013 Claudio Guarnieri. +# Copyright (C) 2014-2019 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import base64 +import bs4 +import chardet +import datetime +import io +import jsbeautifier +import json +import logging +import operator +import os +import platform +import re +import string +import sys +import threading +import warnings +import xmlrpclib + +from distutils.version import StrictVersion, LooseVersion + +from cuckoo.common.constants import GITHUB_URL, ISSUES_PAGE_URL +from cuckoo.misc import cwd, version + +log = logging.getLogger(__name__) + +# Don't allow all characters in "string.printable", as newlines, carriage +# returns, tabs, \x0b, and \x0c may mess up reports. +PRINTABLE_CHARACTERS = ( + string.letters + string.digits + string.punctuation + " \t\r\n" +) + +def convert_char(c): + """Escape characters. + @param c: dirty char. + @return: sanitized char. + """ + if c in PRINTABLE_CHARACTERS: + return c + else: + return "\\x%02x" % ord(c) + +def is_printable(s): + """Test if a string is printable.""" + for c in s: + if c not in PRINTABLE_CHARACTERS: + return False + return True + +def convert_to_printable(s): + """Convert char to printable. + @param s: string. + @return: sanitized string. + """ + if is_printable(s): + return s + return "".join(convert_char(c) for c in s) + +def random_token(): + """Generate a random token that can be used as a secret/password.""" + token = base64.urlsafe_b64encode(os.urandom(16)) + return token.rstrip(b"=").decode("utf8") + +def constant_time_compare(a, b): + """Compare two secret strings in constant time.""" + if not a or not b or len(a) != len(b): + return False + result = 0 + for x, y in zip(a, b): + result |= ord(x) ^ ord(y) + return result == 0 + +def validate_hash(h): + """Validate a hash by length and contents.""" + if len(h) not in (32, 40, 64, 128): + return False + + return bool(re.match("[0-9a-fA-F]*$", h)) + +def validate_url(url, allow_invalid=False): + """Validate an URL using Django's built-in URL validator""" + from django.core.validators import URLValidator + val = URLValidator(schemes=["http", "https"]) + + try: + val(url) + return url + except: + pass + + if allow_invalid and "://" in url: + parts = url.split("://") + # In case of "http://https://example.com" this will take the + # "https://" part and not the "http://" part. + if parts[-2] == "http" or parts[-2] == "https": + return "%s://%s" % (parts[-2], parts[-1]) + + try: + val("http://%s" % url) + return "http://%s" % url + except: + pass + +class TimeoutServer(xmlrpclib.ServerProxy): + """Timeout server for XMLRPC. + XMLRPC + timeout - still a bit ugly - but at least gets rid of setdefaulttimeout + inspired by http://stackoverflow.com/questions/372365/set-timeout-for-xmlrpclib-serverproxy + (although their stuff was messy, this is cleaner) + @see: http://stackoverflow.com/questions/372365/set-timeout-for-xmlrpclib-serverproxy + """ + def __init__(self, *args, **kwargs): + timeout = kwargs.pop("timeout", None) + kwargs["transport"] = TimeoutTransport(timeout=timeout) + xmlrpclib.ServerProxy.__init__(self, *args, **kwargs) + + def _set_timeout(self, timeout): + t = self._ServerProxy__transport + t.timeout = timeout + # If we still have a socket we need to update that as well. + if hasattr(t, "_connection") and t._connection[1] and t._connection[1].sock: + t._connection[1].sock.settimeout(timeout) + +class TimeoutTransport(xmlrpclib.Transport): + def __init__(self, *args, **kwargs): + self.timeout = kwargs.pop("timeout", None) + xmlrpclib.Transport.__init__(self, *args, **kwargs) + + def make_connection(self, *args, **kwargs): + conn = xmlrpclib.Transport.make_connection(self, *args, **kwargs) + if self.timeout is not None: + conn.timeout = self.timeout + return conn + +class Singleton(type): + """Singleton. + @see: http://stackoverflow.com/questions/6760685/creating-a-singleton-in-python + """ + _instances = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) + return cls._instances[cls] + +class ThreadSingleton(type): + """Singleton per thread.""" + _instances = threading.local() + + def __call__(cls, *args, **kwargs): + if not getattr(cls._instances, "instance", None): + cls._instances.instance = super(ThreadSingleton, cls).__call__(*args, **kwargs) + return cls._instances.instance + +def to_unicode(s): + """Attempt to fix non uft-8 string into utf-8. It tries to guess input encoding, + if fail retry with a replace strategy (so undetectable chars will be escaped). + @see: fuller list of encodings at http://docs.python.org/library/codecs.html#standard-encodings + """ + + def brute_enc(s2): + """Trying to decode via simple brute forcing.""" + encodings = ("ascii", "utf8", "latin1") + for enc in encodings: + try: + return unicode(s2, enc) + except UnicodeDecodeError: + pass + return None + + def chardet_enc(s2): + """Guess encoding via chardet.""" + enc = chardet.detect(s2)["encoding"] + + try: + return unicode(s2, enc) + except UnicodeDecodeError: + pass + return None + + # If already in unicode, skip. + if isinstance(s, unicode): + return s + + # First try to decode against a little set of common encodings. + result = brute_enc(s) + + # Try via chardet. + if not result: + result = chardet_enc(s) + + # If not possible to convert the input string, try again with + # a replace strategy. + if not result: + result = unicode(s, errors="replace") + + return result + +def classlock(f): + """Classlock decorator (created for database.Database). + Used to put a lock to avoid sqlite errors. + """ + def inner(self, *args, **kwargs): + if not self._lock: + return f(self, *args, **kwargs) + self._lock.acquire() + try: + return f(self, *args, **kwargs) + finally: + self._lock.release() + + return inner + +GUIDS = {} + +def guid_name(guid): + if not GUIDS: + for line in open(cwd("guids.txt", private=True)): + try: + guid_, name, url = line.strip().split() + except: + log.debug("Invalid GUID entry: %s", line) + continue + + GUIDS["{%s}" % guid_] = name + + return GUIDS.get(guid) + +def exception_message(): + """Create a message describing an unhandled exception.""" + def get_os_release(): + """Returns detailed OS release.""" + if platform.linux_distribution()[0]: + return " ".join(platform.linux_distribution()) + elif platform.mac_ver()[0]: + return "%s %s" % (platform.mac_ver()[0], platform.mac_ver()[2]) + else: + return "Unknown" + + import pkg_resources + + msg = ( + "Oops! Cuckoo failed in an unhandled exception!\nSometimes bugs are " + "already fixed in the development release, it is therefore " + "recommended to retry with the latest development release available " + "%s\nIf the error persists please open a new issue at %s\n\n" % + (GITHUB_URL, ISSUES_PAGE_URL) + ) + + msg += "=== Exception details ===\n" + msg += "Cuckoo version: %s\n" % version + msg += "OS version: %s\n" % os.name + msg += "OS release: %s\n" % get_os_release() + msg += "Python version: %s\n" % platform.python_version() + msg += "Python implementation: %s\n" % platform.python_implementation() + msg += "Machine arch: %s\n" % platform.machine() + msg += "Modules: %s\n\n" % " ".join(sorted( + "%s:%s" % (package.key, package.version) + for package in pkg_resources.working_set + )) + return msg + +_jsbeautify_blocklist = [ + "", + "error: Unknown p.a.c.k.e.r. encoding.\n", +] + +_jsbeautify_lock = threading.Lock() + +def jsbeautify(javascript): + """Beautify Javascript through jsbeautifier and ignore some messages.""" + with _jsbeautify_lock: + origout, sys.stdout = sys.stdout, io.StringIO() + + try: + javascript = jsbeautifier.beautify(javascript) + except Exception as e: + log.exception("Unable to beautify javascript: %s", e) + + if sys.stdout.getvalue() not in _jsbeautify_blocklist: + log.warning( + "jsbeautifier returned error: %s", sys.stdout.getvalue() + ) + + sys.stdout = origout + return javascript + +def htmlprettify(html): + """Beautify HTML through BeautifulSoup4.""" + # The following ignores the following bs4 warning: + # UserWarning: "." looks like a filename, not markup. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", lineno=182) + return bs4.BeautifulSoup(html, "html.parser").prettify() + +def json_default(obj): + """JSON serialize objects not serializable by default json code""" + if hasattr(obj, "to_dict"): + return obj.to_dict() + + if isinstance(obj, datetime.datetime): + if obj.utcoffset() is not None: + obj = obj - obj.utcoffset() + return {"$dt": obj.isoformat()} + raise TypeError("Type not serializable") + +def json_hook(obj): + """JSON object hook, deserializing datetimes ($date)""" + if "$dt" in obj: + return datetime.datetime.strptime(obj["$dt"], "%Y-%m-%dT%H:%M:%S.%f") + return obj + +def json_encode(obj, **kwargs): + """JSON encoding wrapper that handles datetime objects""" + return json.dumps(obj, default=json_default, **kwargs) + +def json_decode(x): + """JSON decoder that does ugly first-level datetime handling""" + return json.loads(x, object_hook=json_hook) + +def parse_bool(value): + """Attempt to parse a boolean value.""" + if value in ("true", "True", "yes", "1", "on"): + return True + if value in ("false", "False", "None", "no", "0", "off"): + return False + return bool(int(value)) + +def supported_version(version, minimum, maximum): + """Check if a version number is supported as per the minimum and maximum + version numbers.""" + if minimum and StrictVersion(version) < StrictVersion(minimum): + return False + + if maximum and StrictVersion(version) > StrictVersion(maximum): + return False + + return True + +def list_of(l, cls): + if not isinstance(l, (tuple, list)): + return False + for value in l: + if not isinstance(value, cls): + return False + return True + +def list_of_ints(l): + return list_of(l, (int, long)) + +def list_of_strings(l): + return list_of(l, basestring) + +def cmp_version(first, second, op): + op_lookup = { + ">": operator.gt, + "<": operator.lt, + ">=": operator.ge, + "<=": operator.le, + "!=": operator.ne, + "==": operator.eq + } + op = op_lookup.get(op) + + return op(LooseVersion(first), LooseVersion(second)) diff --git a/cuckoo/common/virustotal.py b/cuckoo/common/virustotal.py new file mode 100644 index 0000000000..8be92b5b22 --- /dev/null +++ b/cuckoo/common/virustotal.py @@ -0,0 +1,213 @@ +# Copyright (C) 2015-2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import logging +import re +import requests + +# Disable requests/urllib3 debug & info messages. +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) + +from cuckoo.common.config import config +from cuckoo.common.exceptions import CuckooOperationalError +from cuckoo.common.objects import File + +class VirusTotalError(CuckooOperationalError): + """VirusTotal operational error""" + +class VirusTotalResourceNotScanned(CuckooOperationalError): + """This resource has not been scanned yet.""" + +class VirusTotalAPI(object): + """Wrapper to VirusTotal API.""" + + FILE_REPORT = "https://www.virustotal.com/vtapi/v2/file/report" + URL_REPORT = "https://www.virustotal.com/vtapi/v2/url/report" + FILE_SCAN = "https://www.virustotal.com/vtapi/v2/file/scan" + URL_SCAN = "https://www.virustotal.com/vtapi/v2/url/scan" + HASH_DOWNLOAD = "https://www.virustotal.com/vtapi/v2/file/download" + + VARIANT_BLOCKLIST = [ + "generic", "malware", "trojan", "agent", "win32", "multi", "w32", + "trojanclicker", "trojware", "win", "a variant of win32", "trj", + "susp", "dangerousobject", "backdoor", "clicker", "variant", "heur", + "gen", "virus", "dropper", "generic suspicious", "spyware", "program", + "suspectcrc", "corrupt", "behaveslike", "crypt", "adclicker", + "troj", "injector", "cryptor", "packed", "adware", "macro", "msil4", + "suspicious", "worm", "msil", "msword", "drop", "keygen", "office", + "password", "malpack", "lookslike", "banker", "riskware", "unwanted", + "unclassifiedmalware", "ransom", "trojan horse", "trjndwnlder", + "trojandwnldr", "autorun", "trojandownloader", "trojandwnldr", "text", + "download", "excel", "msilobfuscator", "rootkit", "application", + "a variant of win64", "w97m", "shellcode", "o97m", "exploit", + "x97m", "maliciousmacro", "downldr", "msexcel", "pp97m", "other", + "trojandropper", "crypter", "a variant of msil", "macrodown", + "trojanapt", "dwnldr", "downldexe", "dload", "trojanhorse", "toolbar", + "mailer", "obfus", "obfuscator", "suspicious file", "optional", + "suspected of trojan", "heuristic", "rogue", "virtool", "infostealer", + "generic downloader", "generic malware", "undef", "inject", "packer", + "generic backdoor", "word", "macosx", "hack", "unknown", "downloader", + "trojanspy", "dldr", "msoffice", "osx32", "script", "stealer", + "not a virus", "html", "expl", "shellkode", "downagent", "win64", + "applicunwnt", "heur2", "ddos", "avkill", "servstart", "normal", + "encoder", "w2km_dloader", "docdl", "w97m_dloadr", "mo97", "dloader", + "x2km_dloadr", "w2km_dload", "w2km_dloade", "x2km_droppr", "exedown", + "encodefeature", "docdrop", "mw97", "adload", "a variant of pp97m", + "a variant of w97m", "badmacro", "bkdr", "docdrp", "exedrop", + "generic trojan", "malcrypt", "malicious website", "ransomlock", + "ransomcrypt", "reputation", "trojanransom", "pepatch", "risk", + "adplugin", "webtoolbar", "malagent", "genmalicious", "vbinject", + "vbcrypt", "inject2", "mdropper", "download3", "keylogger", + "downloader11", "damaged", "file", "dldrop", "msil7", "injcrypt", + "patched", "patchfile", "downware", "dropped", + ] + + def __init__(self): + """Initialize VirusTotal API.""" + self.apikey = config("processing:virustotal:key") + self.timeout = config("processing:virustotal:timeout") + self.scan = config("processing:virustotal:scan") + + def _request_json(self, url, **kwargs): + """Wrapper around doing a request and parsing its JSON output.""" + try: + r = requests.post(url, timeout=self.timeout, **kwargs) + return r.json() if r.status_code == 200 else {} + except (requests.ConnectionError, ValueError) as e: + raise CuckooOperationalError("Unable to fetch VirusTotal " + "results: %r" % e.message) + + def _request_hash(self, file_hash, **kwargs): + """Wrapper around requesting a hash.""" + params = dict(hash=file_hash, apikey=self.apikey) + + try: + r = requests.get(self.HASH_DOWNLOAD, params=params, + timeout=self.timeout, **kwargs) + r.raise_for_status() # raise an exception for HTTP error codes + return r.content + except (requests.ConnectionError, ValueError, requests.HTTPError): + raise CuckooOperationalError("Could not fetch hash \"%s\" " + "from VirusTotal" % file_hash) + + def _get_report(self, url, resource, summary=False): + """Fetch the report of a file or URL.""" + data = dict(resource=resource, apikey=self.apikey) + + r = self._request_json(url, data=data) + + # This URL has not been analyzed yet - send a request to analyze it + # and return with the permalink. + if not r.get("response_code"): + if self.scan: + raise VirusTotalResourceNotScanned + else: + return { + "summary": { + "error": "resource has not been scanned yet", + } + } + + results = { + "summary": { + "positives": r.get("positives", 0), + "permalink": r.get("permalink"), + "scan_date": r.get("scan_date"), + }, + } + + # For backwards compatibility. + results.update(r) + + if not summary: + results["scans"] = {} + results["normalized"] = [] + + # Embed all VirusTotal results into the report. + for engine, signature in r.get("scans", {}).items(): + signature["normalized"] = self.normalize(signature["result"]) + results["scans"][engine.replace(".", "_")] = signature + + # Normalize each detected variant in order to try to find the + # exact malware family. + norm_lower = [] + for signature in results["scans"].values(): + for normalized in signature["normalized"]: + if normalized.lower() not in norm_lower: + results["normalized"].append(normalized) + norm_lower.append(normalized.lower()) + + return results + + def url_report(self, url, summary=False): + """Get the report of an existing URL scan. + @param url: URL + @param summary: if you want a summary report""" + return self._get_report(self.URL_REPORT, url, summary) + + def file_report(self, filepath, summary=False): + """Get the report of an existing file scan. + @param filepath: file path + @param summary: if you want a summary report""" + resource = File(filepath).get_md5() + return self._get_report(self.FILE_REPORT, resource, summary) + + def url_scan(self, url): + """Submit a URL to be scanned. + @param url: URL + """ + data = dict(apikey=self.apikey, url=url) + r = self._request_json(self.URL_SCAN, data=data) + return dict(summary=dict(permalink=r.get("permalink"))) + + def hash_fetch(self, file_hash): + return self._request_hash(file_hash=file_hash) + + def file_scan(self, filepath): + """Submit a file to be scanned. + @param filepath: file path + """ + data = dict(apikey=self.apikey) + files = {"file": open(filepath, "rb")} + r = self._request_json(self.FILE_SCAN, data=data, files=files) + return dict(summary=dict(permalink=r.get("permalink"))) + + def normalize(self, variant): + """Normalize the variant name provided by an Anti Virus engine. This + attempts to extract the useful parts of a variant name by stripping + all the boilerplate stuff from it.""" + if not variant: + return [] + + ret = [] + + # Handles "CVE-2012-1234", "CVE2012-1234". + cve = re.search("CVE[-_]?(\\d{4})[-_](\\d{4})", variant) + if cve: + ret.append("CVE-%s-%s" % (cve.group(1), cve.group(2))) + + # Handles "CVE121234". + cve = re.search("CVE(\\d{2})(\\d{4})", variant) + if cve: + ret.append("CVE-20%s-%s" % (cve.group(1), cve.group(2))) + + for word in re.split("[\\.\\,\\-\\(\\)\\[\\]/!:_]", variant): + word = word.strip() + if len(word) < 4: + continue + + if word.lower() in self.VARIANT_BLOCKLIST: + continue + + # Random hashes that are specific to this file. + if re.match("[a-fA-F0-9]+$", word): + continue + + # Family names followed by "potentially unwanted". + if re.match("[a-zA-Z]{1,2} potentially unwanted", word.lower()): + continue + + ret.append(word) + return ret diff --git a/cuckoo/compat/__init__.py b/cuckoo/compat/__init__.py new file mode 100644 index 0000000000..a865e3ef39 --- /dev/null +++ b/cuckoo/compat/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. diff --git a/cuckoo/compat/config.py b/cuckoo/compat/config.py new file mode 100644 index 0000000000..8e95e1f18e --- /dev/null +++ b/cuckoo/compat/config.py @@ -0,0 +1,769 @@ +# Copyright (C) 2016-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +from cuckoo.common.config import cast + +def _041_042(c): + c["cuckoo"]["cuckoo"]["analysis_size_limit"] = 104857600 + c["virtualbox"]["virtualbox"]["timeout"] = 300 + c["vmware"] = { + "vmware": { + "mode": "gui", + "path": "/usr/bin/vmrun", + "machines": ["cuckoo1"], + }, + "cuckoo1": { + "label": "../vmware-xp3.vmx,Snapshot1", + "platform": "windows", + "ip": "192.168.54.111", + }, + } + return c + +def _042_050(c): + analysis_timeout = c["cuckoo"]["cuckoo"].pop("analysis_timeout", None) + critical_timeout = c["cuckoo"]["cuckoo"].pop("critical_timeout", None) + c["cuckoo"]["cuckoo"]["version_check"] = True + c["cuckoo"]["cuckoo"]["memory_dump"] = False + c["cuckoo"]["processing"] = { + "analysis_size_limit": c["cuckoo"]["cuckoo"].pop( + "analysis_size_limit", None + ), + "resolve_dns": True, + } + c["cuckoo"]["database"] = { + "connection": None, + "timeout": None, + } + timeout = c["virtualbox"]["virtualbox"].pop("timeout", None) + c["cuckoo"]["timeouts"] = { + "default": cast("cuckoo:timeouts:default", analysis_timeout), + "critical": cast("cuckoo:timeouts:critical", critical_timeout), + "vm_state": cast("cuckoo:timeouts:vm_state", timeout), + } + sniffer = c["cuckoo"]["cuckoo"].pop("use_sniffer", None) + c["cuckoo"]["sniffer"] = { + "enabled": sniffer, + "tcpdump": "/usr/sbin/tcpdump", + "interface": "vboxnet0", + "bpf": None, + } + c["cuckoo"]["graylog"] = { + "enabled": False, + "host": "localhost", + "port": 12201, + "level": "error", + } + return c + +def _050_060(c): + c["cuckoo"]["resultserver"] = { + "ip": "192.168.56.1", + "port": 2042, + "store_csvs": False, + "upload_max_size": 10485760, + } + c["processing"] = { + "analysisinfo": { + "enabled": True, + }, + "behavior": { + "enabled": True, + }, + "debug": { + "enabled": True, + }, + "dropped": { + "enabled": True, + }, + "network": { + "enabled": True, + }, + "static": { + "enabled": True, + }, + "strings": { + "enabled": True, + }, + "targetinfo": { + "enabled": True, + }, + "virustotal": { + "enabled": True, + "key": "a0283a2c3d55728300d064874239b5346fb991317e8449fe43c902879d758088", + }, + } + return c + +def _060_100(c): + c["auxiliary"] = { + "sniffer": { + "enabled": cast( + "auxiliary:sniffer:enabled", + c["cuckoo"]["sniffer"]["enabled"] + ), + "tcpdump": c["cuckoo"]["sniffer"]["tcpdump"], + "interface": c["cuckoo"]["sniffer"]["interface"], + "bpf": c["cuckoo"]["sniffer"].get("bpf"), + }, + } + c["cuckoo"].pop("sniffer", None) + c["cuckoo"]["cuckoo"]["delete_bin_copy"] = False + machinery = c["cuckoo"]["cuckoo"].pop("machine_manager", None) + c["cuckoo"]["cuckoo"]["machinery"] = machinery + c["cuckoo"]["cuckoo"]["reschedule"] = False + c["cuckoo"]["cuckoo"]["process_results"] = True + c["cuckoo"]["cuckoo"]["max_analysis_count"] = 0 + c["cuckoo"]["cuckoo"]["freespace"] = 64 + c["cuckoo"].pop("graylog", None) + c["esx"] = { + "esx": { + "dsn": "esx://127.0.0.1/?no_verify=1", + "username": "username_goes_here", + "password": "password_goes_here", + "machines": ["analysis1"], + }, + "analysis1": { + "label": "cuckoo1", + "platform": "windows", + "snapshot": "clean_snapshot", + "ip": "192.168.122.105", + } + } + c["memory"] = { + "basic": { + "guest_profile": "WinXPSP2x86", + "delete_memdump": False, + }, + "malfind": { + "enabled": True, + "filter": True, + }, + "apihooks": { + "enabled": False, + "filter": True, + }, + "pslist": { + "enabled": True, + "filter": False, + }, + "psxview": { + "enabled": True, + "filter": False, + }, + "callbacks": { + "enabled": True, + "filter": False, + }, + "idt": { + "enabled": True, + "filter": False, + }, + "timers": { + "enabled": True, + "filter": False, + }, + "messagehooks": { + "enabled": False, + "filter": False, + }, + "getsids": { + "enabled": True, + "filter": False, + }, + "privs": { + "enabled": True, + "filter": False, + }, + "dlllist": { + "enabled": True, + "filter": True, + }, + "handles": { + "enabled": True, + "filter": True, + }, + "ldrmodules": { + "enabled": True, + "filter": True, + }, + "mutantscan": { + "enabled": True, + "filter": True, + }, + "devicetree": { + "enabled": True, + "filter": True, + }, + "svcscan": { + "enabled": True, + "filter": True, + }, + "modscan": { + "enabled": True, + "filter": True, + }, + "mask": { + "enabled": False, + "pid_generic": None, + }, + } + c["processing"]["memory"] = { + "enabled": False, + } + c["reporting"].pop("pickled", None) + c["reporting"]["mmdef"] = { + "enabled": False, + } + c["reporting"].pop("metadata", None) + c["reporting"].pop("maec11", None) + c["reporting"]["maec41"] = { + "enabled": False, + "mode": "overview", + "processtree": True, + "output_handles": False, + "static": True, + "strings": True, + "virustotal": True, + } + c["reporting"]["mongodb"]["host"] = "127.0.0.1" + c["reporting"]["mongodb"]["port"] = 27017 + + for machine in c["vmware"]["vmware"]["machines"]: + label, snapshot = c["vmware"][machine]["label"].split(",", 1) + c["vmware"][machine]["label"] = label + c["vmware"][machine]["snapshot"] = snapshot + return c + +def _100_110(c): + c["cuckoo"]["cuckoo"]["tmppath"] = "/tmp" + return c + +def _111_120(c): + c["cuckoo"]["cuckoo"]["terminate_processes"] = False + c["cuckoo"]["cuckoo"]["max_machines_count"] = 0 + c["cuckoo"]["processing"]["sort_pcap"] = True + c["memory"]["yarascan"] = { + "enabled": True, + "filter": True, + } + c["memory"]["ssdt"] = { + "enabled": True, + "filter": True, + } + c["memory"]["gdt"] = { + "enabled": True, + "filter": True, + } + c["physical"] = { + "physical": { + "machines": ["physical1"], + "user": "username", + "password": "password", + }, + "physical1": { + "label": "physical1", + "platform": "windows", + "ip": "192.168.56.101", + } + } + c["processing"]["procmemory"] = { + "enabled": True, + } + c["processing"]["virustotal"]["timeout"] = 60 + c["reporting"]["jsondump"]["indent"] = 4 + c["reporting"]["jsondump"]["encoding"] = "latin-1" + c["reporting"]["mongodb"]["db"] = "cuckoo" + c["reporting"]["mongodb"]["store_memdump"] = True + c["reporting"].pop("hpfclient", None) + + for machine in c["vmware"]["vmware"]["machines"]: + c["vmware"][machine]["vmx_path"] = ( + c["vmware"][machine].pop("label", None) + ) + + c["xenserver"] = { + "xenserver": { + "user": "root", + "password": "changeme", + "url": "https://xenserver", + "machines": ["cuckoo1"], + }, + "cuckoo1": { + "uuid": "00000000-0000-0000-0000-000000000000", + "platform": "windows", + "ip": "192.168.54.111", + } + } + return c + +def _120_20c1(c): + interface = c["auxiliary"]["sniffer"].pop("interface", "vboxnet0") + c["auxiliary"]["mitm"] = { + "enabled": False, + "mitmdump": "/usr/local/bin/mitmdump", + "port_base": 50000, + "script": "data/mitm.py", + "certificate": "bin/cert.p12", + } + c["auxiliary"]["services"] = { + "enabled": False, + "services": "honeyd", + "timeout": 0, + } + c["avd"] = { + "avd": { + "mode": "headless", + "emulator_path": "/home/cuckoo/android-sdk-linux/tools/emulator", + "adb_path": "/home/cuckoo/android-sdk-linux/platform-tools/adb", + "avd_path": "/home/cuckoo/.android/avd", + "reference_machine": "cuckoo-bird", + "machines": ["cuckoo1"], + }, + "cuckoo1": { + "label": "cuckoo1", + "platform": "android", + "ip": "127.0.0.1", + "emulator_port": 5554, + "resultserver_ip": "10.0.2.2", + "resultserver_port": 2042, + }, + } + c["cuckoo"]["cuckoo"]["max_vmstartup_count"] = 10 + c["cuckoo"]["cuckoo"]["rooter"] = "/tmp/cuckoo-rooter" + c["cuckoo"]["routing"] = { + "route": "none", + "internet": "none", + } + c["cuckoo"]["resultserver"].pop("store_csvs", None) + if c["cuckoo"]["timeouts"]["vm_state"] == 300: + c["cuckoo"]["timeouts"]["vm_state"] = 60 + c["esx"]["esx"]["interface"] = "eth0" + c["kvm"]["kvm"]["interface"] = "virbr0" + c["memory"]["sockscan"] = { + "enabled": True, + "filter": False, + } + c["memory"]["netscan"] = { + "enabled": True, + "filter": False, + } + c["physical"]["physical"]["interface"] = "eth0" + c["physical"]["fog"] = { + "hostname": "none", + "username": "fog", + "password": "password", + } + c["processing"]["apkinfo"] = { + "enabled": False, + "decompilation_threshold": 5000000, + } + c["processing"]["baseline"] = { + "enabled": False, + } + c["processing"]["buffer"] = { + "enabled": True, + } + c["processing"]["droidmon"] = { + "enabled": False, + } + c["processing"]["dumptls"] = { + "enabled": True, + } + c["processing"]["googleplay"] = { + "enabled": False, + "android_id": None, + "google_login": None, + "google_password": None, + } + c["processing"]["procmemory"]["idapro"] = False + c["processing"]["screenshots"] = { + "enabled": False, + "tesseract": "/usr/bin/tesseract", + } + c["processing"]["snort"] = { + "enabled": False, + "snort": "/usr/local/bin/snort", + "conf": "/etc/snort/snort.conf", + } + c["processing"]["suricata"] = { + "enabled": False, + "suricata": "/usr/bin/suricata", + "conf": "/etc/suricata/suricata.yaml", + "eve_log": "eve.json", + "files_log": "files-json.log", + "files_dir": "files", + "socket": None, + } + c["processing"]["virustotal"]["scan"] = False + c["qemu"] = { + "qemu": { + "path": "/usr/bin/qemu-system-x86_64", + "machines": ["vm1", "vm2"], + "interface": "qemubr", + }, + "vm1": { + "label": "vm1", + "image": "/home/rep/vms/qvm_wheezy64_1.qcow2", + "platform": "linux", + "ip": "192.168.55.2", + "interface": "qemubr", + "resultserver_ip": "192.168.55.1", + "tags": "debian_wheezy,64_bit", + }, + "vm2": { + "label": "vm2", + "image": "/home/rep/vms/qvm_wheezy64_1.qcow2", + "arch": "mipsel", + "kernel_path": "{imagepath}/vmlinux-3.16.0-4-4kc-malta-mipsel", + "platform": "linux", + "ip": "192.168.55.3", + "interface": "qemubr", + "tags": "debian_wheezy,mipsel", + }, + } + c["reporting"]["jsondump"]["calls"] = True + c["reporting"].pop("mmdef", None) + c["reporting"].pop("maec41", None) + c["reporting"]["reporthtml"]["enabled"] = False + c["reporting"]["mongodb"]["paginate"] = 100 + c["reporting"]["moloch"] = { + "enabled": False, + "host": None, + "moloch_capture": "/data/moloch/bin/moloch-capture", + "conf": "/data/moloch/etc/config.ini", + "instance": "cuckoo", + } + c["virtualbox"]["virtualbox"]["mode"] = "headless" + c["virtualbox"]["virtualbox"]["interface"] = interface + c["virtualbox"]["honeyd"] = { + "label": "honeyd", + "platform": "linux", + "ip": "192.168.56.102", + "tags": "service, honeyd", + "options": "nictrace noagent", + } + c["vmware"]["vmware"]["interface"] = "virbr0" + c["vpn"] = { + "vpn": { + "enabled": False, + "vpns": "vpn0", + }, + "vpn0": { + "name": "vpn0", + "description": "Spain, Europe", + "interface": "tun0", + }, + } + c["vsphere"] = { + "vsphere": { + "host": "10.0.0.1", + "port": 443, + "user": "username_goes_here", + "pwd": "password_goes_here", + "machines": ["analysis1"], + "interface": "eth0", + }, + "analysis1": { + "label": "cuckoo1", + "platform": "windows", + "snapshot": "cuckoo_ready_running", + "ip": "192.168.1.1", + }, + } + c["xenserver"]["xenserver"]["interface"] = "virbr0" + return c + +def _20c1_20c2(c): + c["auxiliary"]["reboot"] = { + "enabled": True, + } + c["cuckoo"]["routing"]["rt_table"] = "main" + c["cuckoo"]["routing"]["auto_rt"] = True + if c["cuckoo"]["timeouts"]["critical"] == 600: + c["cuckoo"]["timeouts"]["critical"] = 60 + c["processing"]["misp"] = { + "enabled": False, + "url": None, + "apikey": None, + "maxioc": 100, + } + c["processing"]["network"]["safelist-dns"] = False + c["processing"]["network"]["allowed-dns"] = None + c["processing"]["procmemory"]["extract_img"] = True + c["processing"]["procmemory"]["dump_delete"] = False + c["processing"]["procmon"] = { + "enabled": True, + } + c["processing"]["static"]["pdf_timeout"] = 60 + c["processing"]["irma"] = { + "enabled": False, + "timeout": 60, + "scan": False, + "force": False, + "url": None, + } + c["reporting"]["elasticsearch"] = { + "enabled": False, + "hosts": "127.0.0.1", + "calls": False, + "index": "cuckoo", + "index_time_pattern": "yearly", + "cuckoo_node": None, + } + c["reporting"]["notification"] = { + "enabled": False, + "url": None, + "identifier": None, + } + c["reporting"]["mattermost"] = { + "enabled": False, + "username": "cuckoo", + "url": None, + "myurl": None, + "show-virustotal": False, + "show-signatures": False, + "show-urls": False, + "hash-filename": False, + } + + c["vpn"]["vpn"].pop("auto_rt", None) + for vpn in c["vpn"]["vpn"]["vpns"].split(","): + if not vpn.strip(): + continue + + c["vpn"][vpn.strip()]["rt_table"] = c["vpn"][vpn.strip()]["interface"] + + return c + +def _20c2_200(c): + if c["auxiliary"]["mitm"]["script"] == "data/mitm.py": + c["auxiliary"]["mitm"]["script"] = "mitm.py" + if c["cuckoo"]["cuckoo"]["freespace"] == 64: + c["cuckoo"]["cuckoo"]["freespace"] = 1024 + if c["cuckoo"]["cuckoo"]["tmppath"] == "/tmp": + c["cuckoo"]["cuckoo"]["tmppath"] = None + if c["cuckoo"]["processing"]["analysis_size_limit"] == 100*1024*1024: + c["cuckoo"]["processing"]["analysis_size_limit"] = 128*1024*1024 + if c["cuckoo"]["resultserver"]["upload_max_size"] == 10*1024*1024: + c["cuckoo"]["resultserver"]["upload_max_size"] = 128*1024*1024 + c["cuckoo"]["feedback"] = { + "enabled": False, + "name": None, + "company": None, + "email": None, + } + c["processing"]["network"]["safelist_dns"] = cast( + "processing:network:safelist_dns", + c["processing"]["network"].pop("safelist-dns", None) + ) + c["processing"]["network"]["allowed_dns"] = cast( + "processing:network:allowed_dns", + c["processing"]["network"].pop("allowed-dns", None) + ) + c["processing"]["procmemory"]["extract_dll"] = False + # If default key, disable VirusTotal by default. + if c["processing"]["virustotal"]["key"] == "a0283a2c3d55728300d064874239b5346fb991317e8449fe43c902879d758088": + c["processing"]["virustotal"]["enabled"] = False + for vm in c["qemu"]: + if "kernel_path" in c["qemu"][vm]: + c["qemu"][vm]["kernel"] = c["qemu"][vm].pop("kernel_path") + if c["qemu"]["qemu"]["machines"] == ["vm1", "vm2"]: + c["qemu"]["qemu"]["machines"].append("vm3") + c["qemu"]["vm3"] = { + "label": "vm3", + "image": "/home/rep/vms/qvm_wheezy64_1.qcow2", + "arch": "arm", + "platform": "linux", + "ip": "192.168.55.4", + "interface": "qemubr", + "tags": "debian_wheezy,arm", + "kernel": "{imagepath}/vmlinuz-3.2.0-4-versatile-arm", + "initrd": "{imagepath}/initrd-3.2.0-4-versatile-arm", + } + c["reporting"]["elasticsearch"]["hosts"] = cast( + "reporting:elasticsearch:hosts", + c["reporting"]["elasticsearch"]["hosts"] + ) + c["reporting"]["elasticsearch"]["timeout"] = 300 + c["reporting"]["feedback"] = { + "enabled": False, + } + c["reporting"]["jsondump"].pop("encoding") + c["reporting"]["misp"] = { + "enabled": False, + "url": None, + "apikey": None, + "mode": "maldoc ipaddr hashes url", + } + c["reporting"]["mattermost"]["hash_url"] = False + old_items = ( + "show-virustotal", "show-signatures", "show-urls", "hash-filename", + ) + for old_item in old_items: + new_item = old_item.replace("-", "_") + c["reporting"]["mattermost"][new_item] = cast( + "reporting:mattermost:%s" % new_item, + c["reporting"]["mattermost"].pop(old_item, False) + ) + + c["reporting"]["moloch"]["insecure"] = False + c["reporting"]["mongodb"]["username"] = None + c["reporting"]["mongodb"]["password"] = None + + if "url" not in c["reporting"]["notification"]: + c["reporting"]["notification"]["url"] = None + + c["reporting"]["singlefile"] = { + "enabled": cast( + "reporting:singlefile:enabled", + c["reporting"]["reporthtml"]["enabled"] + ), + "html": cast( + "reporting:singlefile:html", + c["reporting"]["reporthtml"]["enabled"] + ), + "pdf": False, + } + c["reporting"].pop("reporthtml") + + c["routing"] = { + "routing": { + "drop": False, + }, + "inetsim": { + "enabled": False, + "server": "192.168.56.1", + }, + "tor": { + "enabled": False, + "dnsport": 5353, + "proxyport": 9040, + }, + "vpn": { + "enabled": cast( + "routing:vpn:enabled", c["vpn"]["vpn"].pop("enabled", None) + ), + "vpns": [], + }, + } + + for item in ("route", "internet", "rt_table", "auto_rt"): + c["routing"]["routing"][item] = cast( + "routing:routing:%s" % item, c["cuckoo"]["routing"].pop(item, None) + ) + + for vpn in c["vpn"]["vpn"]["vpns"].split(","): + if not vpn.strip(): + continue + + c["routing"]["vpn"]["vpns"].append(vpn.strip()) + c["routing"][vpn.strip()] = c["vpn"].pop(vpn.strip(), None) + + c.pop("vpn", None) + c["vsphere"]["vsphere"]["unverified_ssl"] = False + return c + +def _200_201(c): + c["memory"]["mask"]["pid_generic"] = cast( + "memory:mask:pid_generic", c["memory"]["mask"]["pid_generic"] + ) + return c + +def _201_202(c): + machineries = ( + "virtualbox", "avd", "esx", "kvm", "physical", "qemu", "vmware", + "vsphere", "xenserver", + ) + for machinery in machineries: + for machine in c[machinery][machinery]["machines"]: + c[machinery][machine]["osprofile"] = None + return c + +def _203_204(c): + c["processing"]["extracted"] = { + "enabled": True, + } + for machine in c["qemu"]["qemu"]["machines"]: + c["qemu"][machine]["snapshot"] = None + c["qemu"][machine]["enable_kvm"] = False + return c + +def _204_205(c): + if c["auxiliary"]["mitm"]["script"] == "mitm.py": + c["auxiliary"]["mitm"]["script"] = "stuff/mitm.py" + return c + +def _205_206(c): + c["auxiliary"]["replay"] = { + "enabled": True, + "mitmdump": "/usr/local/bin/mitmdump", + "port_base": 51000, + } + c["cuckoo"]["remotecontrol"] = { + "enabled": False, + "guacd_host": "localhost", + "guacd_port": 4822, + } + c["routing"]["inetsim"]["ports"] = None + c["virtualbox"]["controlports"] = "5000-5050" + return c + +def _206_207(c): + c["auxiliary"]["replay"]["certificate"] = "bin/cert.p12" + # We'd like to provide a secure default, but let's not inconvenience + # upgrading users. TODO Might need to revisited once we write back config. + c["cuckoo"]["cuckoo"]["api_token"] = None + c["cuckoo"]["cuckoo"]["web_secret"] = None + c["kvm"]["kvm"]["dsn"] = "qemu:///system" + c["processing"]["irma"]["probes"] = None + c["reporting"]["misp"]["distribution"] = 0 + c["reporting"]["misp"]["analysis"] = 0 + c["reporting"]["misp"]["threat_level"] = 4 + c["reporting"]["misp"]["min_malscore"] = 0 + c["reporting"]["misp"]["tag"] = "Cuckoo" + c["reporting"]["misp"]["upload_sample"] = False + + return c + +migrations = { + "0.4.0": ("0.4.1", None), + "0.4.1": ("0.4.2", _041_042), + "0.4.2": ("0.5.0", _042_050), + "0.5.0": ("0.6.0", _050_060), + "0.6.0": ("1.0.0", _060_100), + "1.0.0": ("1.1.0", _100_110), + "1.1.0": ("1.1.1", None), + "1.1.1": ("1.2.0", _111_120), + "1.2.0": ("2.0-rc1", _120_20c1), + "2.0-rc1": ("2.0-rc2", _20c1_20c2), + "2.0-rc2": ("2.0.0", _20c2_200), + "2.0.0": ("2.0.1", _200_201), + "2.0.1": ("2.0.2", _201_202), + "2.0.2": ("2.0.3", None), + "2.0.3": ("2.0.4", _203_204), + "2.0.4": ("2.0.5", _204_205), + "2.0.5": ("2.0.6", _205_206), + "2.0.6": ("2.0.7", _206_207), + + # We're also capable of migrating away from 2.0-dev which basically means + # that we might have to a partial migration from either 2.0-rc2 or 2.0-rc1. + # TODO Most likely we'll have to work out some tweaks in the migrations. + # TODO Provide the option to push out feedback to the Core Developers if + # an exception occurs during the configuration migration phase. + "2.0-dev": ("1.2.0", None), +} + +# Mapping from actual version numbers to "full" / beautified version numbers. +mapping = { + "0.4": "0.4.0", "0.5": "0.5.0", "0.6": "0.6.0", "1.0": "1.0.0", + "1.1": "1.1.0", "1.2": "1.2.0", +} + +def migrate(c, current, to=None): + """Upgrade the configuration 'c' from 'current' to 'to'.""" + while current != to and mapping.get(current, current) in migrations: + current, migration = migrations[mapping.get(current, current)] + c = migration(c) if migration else c + return c diff --git a/cuckoo/core/__init__.py b/cuckoo/core/__init__.py new file mode 100644 index 0000000000..57edaa85f5 --- /dev/null +++ b/cuckoo/core/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. diff --git a/cuckoo/core/database.py b/cuckoo/core/database.py new file mode 100644 index 0000000000..d4994fb9f5 --- /dev/null +++ b/cuckoo/core/database.py @@ -0,0 +1,1669 @@ +# Copyright (C) 2012-2013 Claudio Guarnieri. +# Copyright (C) 2014-2019 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import datetime +import json +import logging +import os +import sys +import threading + +from cuckoo.common.colors import green +from cuckoo.common.config import config, parse_options, emit_options +from cuckoo.common.exceptions import CuckooDatabaseError +from cuckoo.common.exceptions import CuckooOperationalError +from cuckoo.common.exceptions import CuckooDependencyError +from cuckoo.common.objects import File, URL, Dictionary +from cuckoo.common.utils import Singleton, classlock, json_encode, parse_bool +from cuckoo.misc import cwd, format_command + +from sqlalchemy import create_engine, Column, not_, func +from sqlalchemy import Integer, String, Boolean, DateTime, Enum +from sqlalchemy import ForeignKey, Text, Index, Table, TypeDecorator +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.exc import SQLAlchemyError, IntegrityError +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import sessionmaker, relationship, joinedload + +Base = declarative_base() + +log = logging.getLogger(__name__) + +SCHEMA_VERSION = "cb1024e614b7" +TASK_PENDING = "pending" +TASK_RUNNING = "running" +TASK_COMPLETED = "completed" +TASK_RECOVERED = "recovered" +TASK_REPORTED = "reported" +TASK_FAILED_ANALYSIS = "failed_analysis" +TASK_FAILED_PROCESSING = "failed_processing" +TASK_FAILED_REPORTING = "failed_reporting" + +status_type = Enum( + TASK_PENDING, TASK_RUNNING, TASK_COMPLETED, TASK_REPORTED, TASK_RECOVERED, + TASK_FAILED_ANALYSIS, TASK_FAILED_PROCESSING, TASK_FAILED_REPORTING, + name="status_type" +) + +# Secondary table used in association Machine - Tag. +machines_tags = Table( + "machines_tags", Base.metadata, + Column("machine_id", Integer, ForeignKey("machines.id")), + Column("tag_id", Integer, ForeignKey("tags.id")) +) + +# Secondary table used in association Task - Tag. +tasks_tags = Table( + "tasks_tags", Base.metadata, + Column("task_id", Integer, ForeignKey("tasks.id")), + Column("tag_id", Integer, ForeignKey("tags.id")) +) + +class JsonType(TypeDecorator): + """Custom JSON type.""" + impl = Text + + def process_bind_param(self, value, dialect): + return json.dumps(value) + + def process_result_value(self, value, dialect): + return json.loads(value) + +class JsonTypeList255(TypeDecorator): + """Custom JSON type.""" + impl = String(255) + + def process_bind_param(self, value, dialect): + return json.dumps(value) + + def process_result_value(self, value, dialect): + return json.loads(value) if value else [] + +class Machine(Base): + """Configured virtual machines to be used as guests.""" + __tablename__ = "machines" + + id = Column(Integer(), primary_key=True) + name = Column(String(255), nullable=False) + label = Column(String(255), nullable=False) + ip = Column(String(255), nullable=False) + platform = Column(String(255), nullable=False) + tags = relationship("Tag", secondary=machines_tags, single_parent=True, + backref="machine") + options = Column(JsonTypeList255(), nullable=True) + interface = Column(String(255), nullable=True) + snapshot = Column(String(255), nullable=True) + locked = Column(Boolean(), nullable=False, default=False) + locked_changed_on = Column(DateTime(timezone=False), nullable=True) + status = Column(String(255), nullable=True) + status_changed_on = Column(DateTime(timezone=False), nullable=True) + resultserver_ip = Column(String(255), nullable=False) + resultserver_port = Column(Integer(), nullable=False) + _rcparams = Column("rcparams", Text(), nullable=True) + + def __repr__(self): + return "".format(self.id, self.name) + + @hybrid_property + def rcparams(self): + if not self._rcparams: + return {} + return parse_options(self._rcparams) + + @rcparams.setter + def rcparams(self, value): + if isinstance(value, dict): + self._rcparams = emit_options(value) + else: + self._rcparams = value + + def to_dict(self): + """Convert object to dict. + @return: dict + """ + d = {} + for column in self.__table__.columns: + value = getattr(self, column.name) + if isinstance(value, datetime.datetime): + d[column.name] = value.strftime("%Y-%m-%d %H:%M:%S") + else: + d[column.name] = value + + # Tags are a relation so no column to iterate. + d["tags"] = [tag.name for tag in self.tags] + return d + + def to_json(self): + """Convert object to JSON. + @return: JSON data + """ + return json.dumps(self.to_dict()) + + def is_analysis(self): + """Is this an analysis machine? Generally speaking all machines are + analysis machines, however, this is not the case for service VMs. + Please refer to the services auxiliary module.""" + for tag in self.tags: + if tag.name == "service": + return + return True + + def __init__(self, name, label, ip, platform, options, interface, + snapshot, resultserver_ip, resultserver_port): + self.name = name + self.label = label + self.ip = ip + self.platform = platform + self.options = options + self.interface = interface + self.snapshot = snapshot + self.resultserver_ip = resultserver_ip + self.resultserver_port = resultserver_port + +class Tag(Base): + """Tag describing anything you want.""" + __tablename__ = "tags" + + id = Column(Integer(), primary_key=True) + name = Column(String(255), nullable=False, unique=True) + + def __repr__(self): + return "".format(self.id, self.name) + + def __init__(self, name): + self.name = name + +class Guest(Base): + """Tracks guest run.""" + __tablename__ = "guests" + + id = Column(Integer(), primary_key=True) + # TODO Replace the guest.status with a more generic Task.status solution. + status = Column(String(16), nullable=False) + name = Column(String(255), nullable=False) + label = Column(String(255), nullable=False) + manager = Column(String(255), nullable=False) + started_on = Column(DateTime(timezone=False), + default=datetime.datetime.now, + nullable=False) + shutdown_on = Column(DateTime(timezone=False), nullable=True) + task_id = Column(Integer, + ForeignKey("tasks.id"), + nullable=False, + unique=True) + + def __repr__(self): + return "".format(self.id, self.name) + + def to_dict(self): + """Convert object to dict. + @return: dict + """ + d = {} + for column in self.__table__.columns: + value = getattr(self, column.name) + if isinstance(value, datetime.datetime): + d[column.name] = value.strftime("%Y-%m-%d %H:%M:%S") + else: + d[column.name] = value + return d + + def to_json(self): + """Convert object to JSON. + @return: JSON data + """ + return json.dumps(self.to_dict()) + + def __init__(self, name, label, manager): + self.name = name + self.label = label + self.manager = manager + +class Submit(Base): + """Submitted files details.""" + __tablename__ = "submit" + + id = Column(Integer(), primary_key=True) + tmp_path = Column(Text(), nullable=True) + added = Column(DateTime, nullable=False, default=datetime.datetime.utcnow) + submit_type = Column(String(16), nullable=True) + data = Column(JsonType, nullable=True) + + def __init__(self, tmp_path, submit_type, data): + self.tmp_path = tmp_path + self.submit_type = submit_type + self.data = data + +class Sample(Base): + """Submitted files details.""" + __tablename__ = "samples" + + id = Column(Integer(), primary_key=True) + file_size = Column(Integer(), nullable=False) + file_type = Column(Text(), nullable=False) + md5 = Column(String(32), nullable=False) + crc32 = Column(String(8), nullable=False) + sha1 = Column(String(40), nullable=False) + sha256 = Column(String(64), nullable=False) + sha512 = Column(String(128), nullable=False) + ssdeep = Column(String(255), nullable=True) + __table_args__ = Index("hash_index", "md5", "crc32", "sha1", + "sha256", "sha512", unique=True), + + def __repr__(self): + return "".format(self.id, self.sha256) + + def to_dict(self): + """Convert object to dict. + @return: dict + """ + d = {} + for column in self.__table__.columns: + d[column.name] = getattr(self, column.name) + return d + + def to_json(self): + """Convert object to JSON. + @return: JSON data + """ + return json.dumps(self.to_dict()) + + def __init__(self, md5, crc32, sha1, sha256, sha512, + file_size, file_type, ssdeep): + self.md5 = md5 + self.sha1 = sha1 + self.crc32 = crc32 + self.sha256 = sha256 + self.sha512 = sha512 + self.file_size = file_size + self.file_type = file_type + self.ssdeep = ssdeep + +class Error(Base): + """Analysis errors.""" + __tablename__ = "errors" + + id = Column(Integer(), primary_key=True) + action = Column(String(64), nullable=True) + message = Column(Text(), nullable=False) + task_id = Column(Integer, ForeignKey("tasks.id"), nullable=False) + + def to_dict(self): + """Convert object to dict. + @return: dict + """ + d = {} + for column in self.__table__.columns: + d[column.name] = getattr(self, column.name) + return d + + def to_json(self): + """Convert object to JSON. + @return: JSON data + """ + return json.dumps(self.to_dict()) + + def __init__(self, message, task_id, action=None): + self.action = action + self.message = message + self.task_id = task_id + + def __repr__(self): + return "".format(self.id, self.message, self.task_id) + +class Task(Base): + """Analysis task queue.""" + __tablename__ = "tasks" + + id = Column(Integer(), primary_key=True) + target = Column(Text(), nullable=False) + category = Column(String(255), nullable=False) + timeout = Column(Integer(), server_default="0", nullable=False) + priority = Column(Integer(), server_default="1", nullable=False) + custom = Column(Text(), nullable=True) + owner = Column(String(64), nullable=True) + machine = Column(String(255), nullable=True) + package = Column(String(255), nullable=True) + tags = relationship("Tag", secondary=tasks_tags, single_parent=True, + backref="task", lazy="subquery") + _options = Column("options", Text(), nullable=True) + platform = Column(String(255), nullable=True) + memory = Column(Boolean, nullable=False, default=False) + enforce_timeout = Column(Boolean, nullable=False, default=False) + clock = Column(DateTime(timezone=False), + default=datetime.datetime.now, + nullable=False) + added_on = Column(DateTime(timezone=False), + default=datetime.datetime.now, + nullable=False) + started_on = Column(DateTime(timezone=False), nullable=True) + completed_on = Column(DateTime(timezone=False), nullable=True) + status = Column(status_type, server_default=TASK_PENDING, nullable=False) + sample_id = Column(Integer, ForeignKey("samples.id"), nullable=True) + submit_id = Column( + Integer, ForeignKey("submit.id"), nullable=True, index=True + ) + processing = Column(String(16), nullable=True) + route = Column(String(16), nullable=True) + sample = relationship("Sample", backref="tasks") + submit = relationship("Submit", backref="tasks") + guest = relationship("Guest", uselist=False, backref="tasks", cascade="save-update, delete") + errors = relationship("Error", backref="tasks", cascade="save-update, delete") + + def duration(self): + if self.started_on and self.completed_on: + return (self.completed_on - self.started_on).seconds + return -1 + + @hybrid_property + def options(self): + if not self._options: + return {} + return parse_options(self._options) + + @options.setter + def options(self, value): + if isinstance(value, dict): + self._options = emit_options(value) + else: + self._options = value + + def to_dict(self, dt=False): + """Convert object to dict. + @param dt: encode datetime objects + @return: dict + """ + d = Dictionary() + for column in self.__table__.columns: + value = getattr(self, column.name) + if dt and isinstance(value, datetime.datetime): + d[column.name] = value.strftime("%Y-%m-%d %H:%M:%S") + else: + d[column.name] = value + + # Tags are a relation so no column to iterate. + d["tags"] = [tag.name for tag in self.tags] + d["duration"] = self.duration() + d["guest"] = {} + + if self.guest: + # Get machine description. + d["guest"] = machine = self.guest.to_dict() + # Remove superfluous fields. + del machine["task_id"] + del machine["id"] + + return d + + def to_json(self): + """Convert object to JSON. + @return: JSON data + """ + return json_encode(self.to_dict()) + + def __init__(self, target=None, id=None, category=None): + self.target = target + self.id = id + self.category = category + + def __repr__(self): + return "".format(self.id, self.target) + +class AlembicVersion(Base): + """Table used to pinpoint actual database schema release.""" + __tablename__ = "alembic_version" + + version_num = Column(String(32), nullable=False, primary_key=True) + +class Database(object): + """Analysis queue database. + + This class handles the creation of the database user for internal queue + management. It also provides some functions for interacting with it. + """ + __metaclass__ = Singleton + + def __init__(self, schema_check=True, echo=False): + """ + @param dsn: database connection string. + @param schema_check: disable or enable the db schema version check. + @param echo: echo sql queries. + """ + self._lock = None + self.schema_check = schema_check + self.echo = echo + + def connect(self, schema_check=None, dsn=None, create=True): + """Connect to the database backend.""" + if schema_check is not None: + self.schema_check = schema_check + + if not dsn: + dsn = config("cuckoo:database:connection") + if not dsn: + dsn = "sqlite:///%s" % cwd("cuckoo.db") + + database_flavor = dsn.split(":", 1)[0].lower() + if database_flavor == "sqlite": + log.debug("Using database-wide lock for sqlite") + self._lock = threading.RLock() + + self._connect_database(dsn) + + # Disable SQL logging. Turn it on for debugging. + self.engine.echo = self.echo + + # Connection timeout. + self.engine.pool_timeout = config("cuckoo:database:timeout") + + # Get db session. + self.Session = sessionmaker(bind=self.engine) + + if create: + self._create_tables() + + def _create_tables(self): + """Create all database tables etc.""" + try: + Base.metadata.create_all(self.engine) + except SQLAlchemyError as e: + raise CuckooDatabaseError( + "Unable to create or connect to database: %s" % e + ) + + # Deal with schema versioning. + # TODO: it's a little bit dirty, needs refactoring. + tmp_session = self.Session() + if not tmp_session.query(AlembicVersion).count(): + # Set database schema version. + tmp_session.add(AlembicVersion(version_num=SCHEMA_VERSION)) + try: + tmp_session.commit() + except SQLAlchemyError as e: + raise CuckooDatabaseError( + "Unable to set schema version: %s" % e + ) + tmp_session.rollback() + finally: + tmp_session.close() + else: + # Check if db version is the expected one. + last = tmp_session.query(AlembicVersion).first() + tmp_session.close() + if last.version_num != SCHEMA_VERSION and self.schema_check: + log.warning( + "Database schema version mismatch: found %s, expected %s.", + last.version_num, SCHEMA_VERSION + ) + log.error( + "Optionally make a backup and then apply the latest " + "database migration(s) by running:" + ) + log.info("$ %s", green(format_command("migrate"))) + sys.exit(1) + + def __del__(self): + """Disconnects pool.""" + self.engine.dispose() + + def _connect_database(self, connection_string): + """Connect to a database. + @param connection_string: Connection string specifying the database + """ + try: + # TODO: this is quite ugly, should improve. + if connection_string.startswith("sqlite"): + # Using "check_same_thread" to disable sqlite safety check on multiple threads. + self.engine = create_engine(connection_string, connect_args={"check_same_thread": False}) + elif connection_string.startswith("postgres"): + # Disabling SSL mode to avoid some errors using sqlalchemy and multiprocesing. + # See: http://www.postgresql.org/docs/9.0/static/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS + # TODO Check if this is still relevant. Especially provided the + # fact that we're no longer using multiprocessing. + self.engine = create_engine(connection_string, connect_args={"sslmode": "disable"}) + else: + self.engine = create_engine(connection_string) + except ImportError as e: + lib = e.message.split()[-1] + + if lib == "MySQLdb": + raise CuckooDependencyError( + "Missing MySQL database driver (install with " + "`pip install mysql-python` on Linux or `pip install " + "mysqlclient` on Windows)" + ) + + if lib == "psycopg2": + raise CuckooDependencyError( + "Missing PostgreSQL database driver (install with " + "`pip install psycopg2`)" + ) + + raise CuckooDependencyError( + "Missing unknown database driver, unable to import %s" % lib + ) + + def _get_or_create(self, session, model, **kwargs): + """Get an ORM instance or create it if not exist. + @param session: SQLAlchemy session object + @param model: model to query + @return: row instance + """ + instance = session.query(model).filter_by(**kwargs).first() + return instance or model(**kwargs) + + @classlock + def drop(self): + """Drop all tables.""" + try: + Base.metadata.drop_all(self.engine) + except SQLAlchemyError as e: + raise CuckooDatabaseError( + "Unable to drop all tables of the database: %s" % e + ) + + @classlock + def clean_machines(self): + """Clean old stored machines and related tables.""" + # Secondary table. + # TODO: this is better done via cascade delete. + self.engine.execute(machines_tags.delete()) + + session = self.Session() + try: + session.query(Machine).delete() + session.commit() + except SQLAlchemyError as e: + log.exception("Database error cleaning machines: {0}".format(e)) + session.rollback() + finally: + session.close() + + @classlock + def add_machine(self, name, label, ip, platform, options, tags, interface, + snapshot, resultserver_ip, resultserver_port): + """Add a guest machine. + @param name: machine id + @param label: machine label + @param ip: machine IP address + @param platform: machine supported platform + @param tags: list of comma separated tags + @param interface: sniffing interface for this machine + @param snapshot: snapshot name to use instead of the current one, if configured + @param resultserver_ip: IP address of the Result Server + @param resultserver_port: port of the Result Server + """ + if options is None: + options = [] + if not isinstance(options, (tuple, list)): + options = options.split() + + session = self.Session() + machine = Machine(name=name, + label=label, + ip=ip, + platform=platform, + options=options, + interface=interface, + snapshot=snapshot, + resultserver_ip=resultserver_ip, + resultserver_port=resultserver_port) + + # Deal with tags format (i.e., foo,bar,baz) + if tags: + for tag in tags.split(","): + if tag.strip(): + tag = self._get_or_create(session, Tag, name=tag.strip()) + machine.tags.append(tag) + session.add(machine) + + try: + session.commit() + except SQLAlchemyError as e: + log.exception("Database error adding machine: {0}".format(e)) + session.rollback() + finally: + session.close() + + @classlock + def set_status(self, task_id, status): + """Set task status. + @param task_id: task identifier + @param status: status string + @return: operation status + """ + session = self.Session() + try: + row = session.query(Task).get(task_id) + if not row: + return + + row.status = status + + if status == TASK_RUNNING: + row.started_on = datetime.datetime.now() + elif status == TASK_COMPLETED: + row.completed_on = datetime.datetime.now() + + session.commit() + except SQLAlchemyError as e: + log.exception("Database error setting status: {0}".format(e)) + session.rollback() + finally: + session.close() + + @classlock + def set_route(self, task_id, route): + """Set the taken route of this task. + @param task_id: task identifier + @param route: route string + @return: operation status + """ + session = self.Session() + try: + row = session.query(Task).get(task_id) + if not row: + return + + row.route = route + session.commit() + except SQLAlchemyError as e: + log.exception("Database error setting route: {0}".format(e)) + session.rollback() + finally: + session.close() + + @classlock + def fetch(self, machine=None, service=True): + """Fetch a task waiting to be processed and lock it for running. + @return: None or task + """ + session = self.Session() + try: + q = session.query(Task).filter_by(status=TASK_PENDING) + + if machine: + q = q.filter_by(machine=machine) + + if not service: + q = q.filter(not_(Task.tags.any(name="service"))) + + row = q.order_by(Task.priority.desc(), Task.added_on).first() + if row: + self.set_status(task_id=row.id, status=TASK_RUNNING) + session.refresh(row) + + return row + except SQLAlchemyError as e: + log.exception("Database error fetching task: {0}".format(e)) + session.rollback() + finally: + session.close() + + @classlock + def guest_start(self, task_id, name, label, manager): + """Log guest start. + @param task_id: task identifier + @param name: vm name + @param label: vm label + @param manager: vm manager + @return: guest row id + """ + session = self.Session() + guest = Guest(name, label, manager) + try: + guest.status = "init" + session.query(Task).get(task_id).guest = guest + session.commit() + session.refresh(guest) + return guest.id + except SQLAlchemyError as e: + log.exception("Database error logging guest start: {0}".format(e)) + session.rollback() + return None + finally: + session.close() + + @classlock + def guest_get_status(self, task_id): + """Log guest start. + @param task_id: task id + @return: guest status + """ + session = self.Session() + try: + guest = session.query(Guest).filter_by(task_id=task_id).first() + return guest.status if guest else None + except SQLAlchemyError as e: + log.exception("Database error logging guest start: {0}".format(e)) + session.rollback() + return + finally: + session.close() + + @classlock + def guest_set_status(self, task_id, status): + """Log guest start. + @param task_id: task identifier + @param status: status + """ + session = self.Session() + try: + guest = session.query(Guest).filter_by(task_id=task_id).first() + guest.status = status + session.commit() + session.refresh(guest) + except SQLAlchemyError as e: + log.exception("Database error logging guest start: {0}".format(e)) + session.rollback() + return None + finally: + session.close() + + @classlock + def guest_remove(self, guest_id): + """Remove a guest start entry.""" + session = self.Session() + try: + guest = session.query(Guest).get(guest_id) + session.delete(guest) + session.commit() + except SQLAlchemyError as e: + log.exception("Database error logging guest remove: {0}".format(e)) + session.rollback() + return None + finally: + session.close() + + @classlock + def guest_stop(self, guest_id): + """Log guest stop. + @param guest_id: guest log entry id + """ + session = self.Session() + try: + guest = session.query(Guest).get(guest_id) + guest.status = "stopped" + guest.shutdown_on = datetime.datetime.now() + session.commit() + except SQLAlchemyError as e: + log.exception("Database error logging guest stop: {0}".format(e)) + session.rollback() + except TypeError: + log.warning("Data inconsistency in guests table detected, it might be a crash leftover. Continue") + session.rollback() + finally: + session.close() + + @classlock + def list_machines(self, locked=False): + """List virtual machines. + @return: list of virtual machines + """ + session = self.Session() + try: + if locked: + machines = session.query(Machine).options(joinedload("tags")).filter_by(locked=True).all() + else: + machines = session.query(Machine).options(joinedload("tags")).all() + return machines + except SQLAlchemyError as e: + log.exception("Database error listing machines: {0}".format(e)) + return [] + finally: + session.close() + + @classlock + def lock_machine(self, label=None, platform=None, tags=None): + """Place a lock on a free virtual machine. + @param label: optional virtual machine label + @param platform: optional virtual machine platform + @param tags: optional tags required (list) + @return: locked machine + """ + session = self.Session() + + # Preventive checks. + if label and platform: + # Wrong usage. + log.error("You can select machine only by label or by platform.") + return None + elif label and tags: + # Also wrong usage. + log.error("You can select machine only by label or by tags.") + return None + + try: + machines = session.query(Machine) + if label: + machines = machines.filter_by(label=label) + if platform: + machines = machines.filter_by(platform=platform) + if tags: + for tag in tags: + machines = machines.filter(Machine.tags.any(name=tag)) + + # Check if there are any machines that satisfy the + # selection requirements. + if not machines.count(): + raise CuckooOperationalError("No machines match selection criteria.") + + # Get the first free machine. + machine = machines.filter_by(locked=False).first() + except SQLAlchemyError as e: + log.exception("Database error locking machine: {0}".format(e)) + session.close() + return None + + if machine: + machine.locked = True + machine.locked_changed_on = datetime.datetime.now() + try: + session.commit() + session.refresh(machine) + except SQLAlchemyError as e: + log.exception("Database error locking machine: {0}".format(e)) + session.rollback() + return None + finally: + session.close() + else: + session.close() + + return machine + + @classlock + def unlock_machine(self, label): + """Remove a lock from a virtual machine. + @param label: virtual machine label + @return: unlocked machine + """ + session = self.Session() + try: + machine = session.query(Machine).filter_by(label=label).first() + except SQLAlchemyError as e: + log.exception("Database error unlocking machine: {0}".format(e)) + session.close() + return None + + if machine: + machine.locked = False + machine.locked_changed_on = datetime.datetime.now() + try: + session.commit() + session.refresh(machine) + except SQLAlchemyError as e: + log.exception("Database error locking machine: {0}".format(e)) + session.rollback() + return None + finally: + session.close() + + return machine + + @classlock + def count_machines_available(self): + """Return number of virtual machines ready for analysis. + @return: free virtual machines count + """ + session = self.Session() + try: + machines_count = session.query(Machine).filter_by(locked=False).count() + return machines_count + except SQLAlchemyError as e: + log.exception("Database error counting machines: {0}".format(e)) + return 0 + finally: + session.close() + + @classlock + def get_available_machines(self): + """Return machines that are available. + @return: free virtual machines + """ + session = self.Session() + try: + machines = session.query(Machine).options(joinedload("tags")).filter_by(locked=False).all() + return machines + except SQLAlchemyError as e: + log.exception( + "Database error getting available machines: {0}".format(e) + ) + return [] + finally: + session.close() + + @classlock + def set_machine_status(self, label, status): + """Set status for a virtual machine. + @param label: virtual machine label + @param status: new virtual machine status + """ + session = self.Session() + try: + machine = session.query(Machine).filter_by(label=label).first() + except SQLAlchemyError as e: + log.exception( + "Database error setting machine status: {0}".format(e) + ) + session.close() + return + + if machine: + machine.status = status + machine.status_changed_on = datetime.datetime.now() + try: + session.commit() + session.refresh(machine) + except SQLAlchemyError as e: + log.exception("Database error setting machine status: %s", e) + session.rollback() + finally: + session.close() + else: + session.close() + + @classlock + def set_machine_rcparams(self, label, rcparams): + """Set remote control connection params for a virtual machine. + @param label: virtual machine label + @param rcparams: dict with keys: protocol, host, port + """ + session = self.Session() + try: + machine = session.query(Machine).filter_by(label=label).first() + except SQLAlchemyError as e: + log.exception("Database error setting machine rcparams: %s", e) + session.close() + return + + if machine: + machine.rcparams = rcparams + try: + session.commit() + session.refresh(machine) + except SQLAlchemyError as e: + log.exception("Database error setting machine rcparams: %s", e) + session.rollback() + finally: + session.close() + else: + session.close() + + @classlock + def add_error(self, message, task_id, action=None): + """Add an error related to a task. + @param message: error message + @param task_id: ID of the related task + """ + session = self.Session() + error = Error(message=message, task_id=task_id, action=action) + session.add(error) + try: + session.commit() + except SQLAlchemyError as e: + log.exception("Database error adding error log: {0}".format(e)) + session.rollback() + finally: + session.close() + + # The following functions are mostly used by external utils. + + @classlock + def add(self, obj, timeout=0, package="", options="", priority=1, + custom="", owner="", machine="", platform="", tags=None, + memory=False, enforce_timeout=False, clock=None, category=None, + submit_id=None): + """Add a task to database. + @param obj: object to add (File or URL). + @param timeout: selected timeout. + @param options: analysis options. + @param priority: analysis priority. + @param custom: custom options. + @param owner: task owner. + @param machine: selected machine. + @param platform: platform. + @param tags: optional tags that must be set for machine selection + @param memory: toggle full memory dump. + @param enforce_timeout: toggle full timeout execution. + @param clock: virtual machine clock time + @return: cursor or None. + """ + # TODO: parameter `package` is not mentioned in the function docstring + session = self.Session() + + # Convert empty strings and None values to a valid int + if not timeout: + timeout = 0 + if not priority: + priority = 1 + + try: + memory = parse_bool(memory) + except ValueError: + memory = False + + try: + enforce_timeout = parse_bool(enforce_timeout) + except ValueError: + enforce_timeout = False + + if isinstance(obj, File): + sample = Sample(md5=obj.get_md5(), + crc32=obj.get_crc32(), + sha1=obj.get_sha1(), + sha256=obj.get_sha256(), + sha512=obj.get_sha512(), + file_size=obj.get_size(), + file_type=obj.get_type(), + ssdeep=obj.get_ssdeep()) + session.add(sample) + + try: + session.commit() + except IntegrityError: + session.rollback() + try: + sample = session.query(Sample).filter_by(md5=obj.get_md5()).first() + except SQLAlchemyError as e: + log.exception( + "Error querying sample for hash: {0}".format(e) + ) + session.close() + return None + except SQLAlchemyError as e: + log.exception("Database error adding task: {0}".format(e)) + session.close() + return None + + task = Task(obj.file_path) + task.sample_id = sample.id + elif isinstance(obj, URL): + task = Task(obj.url) + else: + task = Task("none") + + task.category = category + task.timeout = timeout + task.package = package + task.options = options + task.priority = priority + task.custom = custom + task.owner = owner + task.machine = machine + task.platform = platform + task.memory = memory + task.enforce_timeout = enforce_timeout + task.submit_id = submit_id + + if tags: + if isinstance(tags, basestring): + for tag in tags.split(","): + if tag.strip(): + task.tags.append(self._get_or_create( + session, Tag, name=tag.strip() + )) + + if isinstance(tags, (tuple, list)): + for tag in tags: + if isinstance(tag, basestring) and tag.strip(): + task.tags.append(self._get_or_create( + session, Tag, name=tag.strip() + )) + + if clock: + if isinstance(clock, basestring): + try: + task.clock = datetime.datetime.strptime(clock, "%m-%d-%Y %H:%M:%S") + except ValueError: + log.warning("The date you specified has an invalid format, using current timestamp.") + task.clock = datetime.datetime.now() + else: + task.clock = clock + + session.add(task) + + try: + session.commit() + task_id = task.id + except SQLAlchemyError as e: + log.exception("Database error adding task: {0}".format(e)) + session.rollback() + return None + finally: + session.close() + + return task_id + + def add_path(self, file_path, timeout=0, package="", options="", + priority=1, custom="", owner="", machine="", platform="", + tags=None, memory=False, enforce_timeout=False, clock=None, + submit_id=None): + """Add a task to database from file path. + @param file_path: sample path. + @param timeout: selected timeout. + @param options: analysis options. + @param priority: analysis priority. + @param custom: custom options. + @param owner: task owner. + @param machine: selected machine. + @param platform: platform. + @param tags: Tags required in machine selection + @param memory: toggle full memory dump. + @param enforce_timeout: toggle full timeout execution. + @param clock: virtual machine clock time + @return: cursor or None. + """ + if not file_path or not os.path.exists(file_path): + log.warning("File does not exist: %s.", file_path) + return None + + # Convert empty strings and None values to a valid int + if not timeout: + timeout = 0 + if not priority: + priority = 1 + + return self.add(File(file_path), timeout, package, options, priority, + custom, owner, machine, platform, tags, memory, + enforce_timeout, clock, "file", submit_id) + + def add_archive(self, file_path, filename, package, timeout=0, + options=None, priority=1, custom="", owner="", machine="", + platform="", tags=None, memory=False, + enforce_timeout=False, clock=None, submit_id=None): + """Add a task to the database that's packaged in an archive file.""" + if not file_path or not os.path.exists(file_path): + log.warning("File does not exist: %s.", file_path) + return None + + options = options or {} + options["filename"] = filename + + # Convert empty strings and None values to a valid int + if not timeout: + timeout = 0 + if not priority: + priority = 1 + + options = emit_options(options) + return self.add(File(file_path), timeout, package, options, priority, + custom, owner, machine, platform, tags, memory, + enforce_timeout, clock, "archive", submit_id) + + def add_url(self, url, timeout=0, package="", options="", priority=1, + custom="", owner="", machine="", platform="", tags=None, + memory=False, enforce_timeout=False, clock=None, + submit_id=None): + """Add a task to database from url. + @param url: url. + @param timeout: selected timeout. + @param options: analysis options. + @param priority: analysis priority. + @param custom: custom options. + @param owner: task owner. + @param machine: selected machine. + @param platform: platform. + @param tags: tags for machine selection + @param memory: toggle full memory dump. + @param enforce_timeout: toggle full timeout execution. + @param clock: virtual machine clock time + @return: cursor or None. + """ + + # Convert empty strings and None values to a valid int + if not timeout: + timeout = 0 + if not priority: + priority = 1 + + return self.add(URL(url), timeout, package, options, priority, + custom, owner, machine, platform, tags, memory, + enforce_timeout, clock, "url", submit_id) + + def add_baseline(self, timeout=0, owner="", machine="", memory=False): + """Add a baseline task to database. + @param timeout: selected timeout. + @param owner: task owner. + @param machine: selected machine. + @param memory: toggle full memory dump. + @return: cursor or None. + """ + return self.add(None, timeout=timeout or 0, priority=999, owner=owner, + machine=machine, memory=memory, category="baseline") + + def add_service(self, timeout, owner, tags): + """Add a service task to database. + @param timeout: selected timeout. + @param owner: task owner. + @param tags: task tags. + @return: cursor or None. + """ + return self.add(None, timeout=timeout, priority=999, owner=owner, + tags=tags, category="service") + + def add_reboot(self, task_id, timeout=0, options="", priority=1, + owner="", machine="", platform="", tags=None, memory=False, + enforce_timeout=False, clock=None, submit_id=None): + """Add a reboot task to database from an existing analysis. + @param task_id: task id of existing analysis. + @param timeout: selected timeout. + @param options: analysis options. + @param priority: analysis priority. + @param owner: task owner. + @param machine: selected machine. + @param platform: platform. + @param tags: tags for machine selection + @param memory: toggle full memory dump. + @param enforce_timeout: toggle full timeout execution. + @param clock: virtual machine clock time + @return: cursor or None. + """ + + # Convert empty strings and None values to a valid int + if not timeout: + timeout = 0 + if not priority: + priority = 1 + + task = self.view_task(task_id) + if not task or not os.path.exists(task.target): + log.error( + "Unable to add reboot analysis as the original task or its " + "sample has already been deleted." + ) + return + + # TODO Integrate the Reboot screen with the submission portal and + # pass the parent task ID through as part of the "options". + custom = "%s" % task_id + + return self.add(File(task.target), timeout, "reboot", options, + priority, custom, owner, machine, platform, tags, + memory, enforce_timeout, clock, "file", submit_id) + + @classlock + def add_submit(self, tmp_path, submit_type, data): + session = self.Session() + + submit = Submit( + tmp_path=tmp_path, submit_type=submit_type, data=data or {} + ) + session.add(submit) + try: + session.commit() + session.refresh(submit) + submit_id = submit.id + except SQLAlchemyError as e: + log.exception("Database error adding submit entry: %s", e) + session.rollback() + finally: + session.close() + return submit_id + + @classlock + def view_submit(self, submit_id, tasks=False): + session = self.Session() + try: + q = session.query(Submit) + if tasks: + q = q.options(joinedload("tasks")) + submit = q.get(submit_id) + except SQLAlchemyError as e: + log.exception("Database error viewing submit: %s", e) + return + finally: + session.close() + return submit + + @classlock + def reschedule(self, task_id, priority=None): + """Reschedule a task. + @param task_id: ID of the task to reschedule. + @return: ID of the newly created task. + """ + task = self.view_task(task_id) + if not task: + return + + if task.category == "file": + add = self.add_path + elif task.category == "url": + add = self.add_url + else: + return + + # Change status to recovered. + session = self.Session() + session.query(Task).get(task_id).status = TASK_RECOVERED + try: + session.commit() + except SQLAlchemyError as e: + log.exception("Database error rescheduling task: {0}".format(e)) + session.rollback() + return False + finally: + session.close() + + # Normalize tags. + if task.tags: + tags = ",".join(tag.name for tag in task.tags) + else: + tags = task.tags + + # Assign a new priority. + if priority: + task.priority = priority + + options = emit_options(task.options) + return add(task.target, task.timeout, task.package, options, + task.priority, task.custom, task.owner, task.machine, + task.platform, tags, task.memory, task.enforce_timeout, + task.clock) + + def list_tasks(self, limit=None, details=True, category=None, owner=None, + offset=None, status=None, sample_id=None, not_status=None, + completed_after=None, order_by=None): + """Retrieve list of task. + @param limit: specify a limit of entries. + @param details: if details about must be included + @param category: filter by category + @param owner: task owner + @param offset: list offset + @param status: filter by task status + @param sample_id: filter tasks for a sample + @param not_status: exclude this task status from filter + @param completed_after: only list tasks completed after this timestamp + @param order_by: definition which field to sort by + @return: list of tasks. + """ + session = self.Session() + try: + search = session.query(Task) + + if status: + search = search.filter_by(status=status) + if not_status: + search = search.filter(Task.status != not_status) + if category: + search = search.filter_by(category=category) + if owner: + search = search.filter_by(owner=owner) + if details: + search = search.options(joinedload("guest"), joinedload("errors"), joinedload("tags")) + if sample_id is not None: + search = search.filter_by(sample_id=sample_id) + if completed_after: + search = search.filter(Task.completed_on > completed_after) + + if order_by is not None: + search = search.order_by(order_by) + else: + search = search.order_by(Task.added_on.desc()) + + tasks = search.limit(limit).offset(offset).all() + return tasks + except SQLAlchemyError as e: + log.exception("Database error listing tasks: {0}".format(e)) + return [] + finally: + session.close() + + def minmax_tasks(self): + """Find tasks minimum and maximum. + @return: unix timestamps of minimum and maximum + """ + session = self.Session() + try: + _min = session.query(func.min(Task.started_on).label("min")).first() + _max = session.query(func.max(Task.completed_on).label("max")).first() + + if not isinstance(_min, DateTime) or not isinstance(_max, DateTime): + return + + return int(_min[0].strftime("%s")), int(_max[0].strftime("%s")) + except SQLAlchemyError as e: + log.exception("Database error counting tasks: {0}".format(e)) + return + finally: + session.close() + + @classlock + def count_tasks(self, status=None): + """Count tasks in the database. + @param status: apply a filter according to the task status + @return: number of tasks found + """ + session = self.Session() + try: + if status: + tasks_count = session.query(Task).filter_by(status=status).count() + else: + tasks_count = session.query(Task).count() + return tasks_count + except SQLAlchemyError as e: + log.exception("Database error counting tasks: {0}".format(e)) + return 0 + finally: + session.close() + + @classlock + def view_task(self, task_id, details=True): + """Retrieve information on a task. + @param task_id: ID of the task to query. + @return: details on the task. + """ + session = self.Session() + try: + if details: + task = session.query(Task).options( + joinedload("guest"), + joinedload("errors"), + joinedload("tags") + ).get(task_id) + else: + task = session.query(Task).get(task_id) + except SQLAlchemyError as e: + log.exception("Database error viewing task: {0}".format(e)) + return None + else: + if task: + session.expunge(task) + return task + finally: + session.close() + + @classlock + def view_tasks(self, task_ids): + """Retrieve information on a task. + @param task_id: ID of the task to query. + @return: details on the task. + """ + session = self.Session() + try: + tasks = session.query(Task).options( + joinedload("guest"), + joinedload("errors"), + joinedload("tags") + ).filter(Task.id.in_(task_ids)).order_by(Task.id).all() + except SQLAlchemyError as e: + log.exception("Database error viewing tasks: {0}".format(e)) + return [] + else: + for task in tasks: + session.expunge(task) + return tasks + finally: + session.close() + + @classlock + def delete_task(self, task_id): + """Delete information on a task. + @param task_id: ID of the task to query. + @return: operation status. + """ + session = self.Session() + try: + task = session.query(Task).get(task_id) + session.delete(task) + session.commit() + except SQLAlchemyError as e: + log.exception("Database error deleting task: {0}".format(e)) + session.rollback() + return False + finally: + session.close() + return True + + @classlock + def view_sample(self, sample_id): + """Retrieve information on a sample given a sample ID. + @param sample_id: ID of the sample to query. + @return: details on the sample used in sample: sample_id. + """ + session = self.Session() + try: + sample = session.query(Sample).get(sample_id) + except AttributeError: + return None + except SQLAlchemyError as e: + log.exception("Database error viewing task: {0}".format(e)) + return None + else: + if sample: + session.expunge(sample) + finally: + session.close() + + return sample + + @classlock + def find_sample(self, md5=None, sha256=None): + """Search samples by MD5. + @param md5: md5 string + @return: matches list + """ + session = self.Session() + try: + if md5: + sample = session.query(Sample).filter_by(md5=md5).first() + elif sha256: + sample = session.query(Sample).filter_by(sha256=sha256).first() + except SQLAlchemyError as e: + log.exception("Database error searching sample: {0}".format(e)) + return None + else: + if sample: + session.expunge(sample) + finally: + session.close() + return sample + + @classlock + def count_samples(self): + """Count number of samples in the database.""" + session = self.Session() + try: + sample_count = session.query(Sample).count() + except SQLAlchemyError as e: + log.exception("Database error counting samples: {0}".format(e)) + return 0 + finally: + session.close() + return sample_count + + @classlock + def view_machine(self, name): + """Show virtual machine. + @params name: virtual machine name + @return: virtual machine's details + """ + session = self.Session() + try: + machine = session.query(Machine).options(joinedload("tags")).filter_by(name=name).first() + except SQLAlchemyError as e: + log.exception("Database error viewing machine: {0}".format(e)) + return None + else: + if machine: + session.expunge(machine) + finally: + session.close() + return machine + + @classlock + def view_machine_by_label(self, label): + """Show virtual machine. + @params label: virtual machine label + @return: virtual machine's details + """ + session = self.Session() + try: + machine = session.query(Machine).options(joinedload("tags")).filter_by(label=label).first() + except SQLAlchemyError as e: + log.exception( + "Database error viewing machine by label: {0}".format(e) + ) + return None + else: + if machine: + session.expunge(machine) + finally: + session.close() + return machine + + @classlock + def view_errors(self, task_id): + """Get all errors related to a task. + @param task_id: ID of task associated to the errors + @return: list of errors. + """ + session = self.Session() + try: + q = session.query(Error).filter_by(task_id=task_id) + errors = q.order_by(Error.id).all() + except SQLAlchemyError as e: + log.exception("Database error viewing errors: {0}".format(e)) + return [] + finally: + session.close() + return errors + + def processing_get_task(self, instance): + """Get an available task for processing.""" + session = self.Session() + + # TODO We can get rid of the `processing` column once again by + # introducing a "reporting" status, but this requires annoying + # database migrations, so leaving that for another day. + + try: + # Fetch a task that has yet to be processed and make sure no other + # threads are allowed to access it through "for update". + q = session.query(Task).filter_by(status=TASK_COMPLETED) + q = q.filter_by(processing=None) + q = q.order_by(Task.priority.desc(), Task.id) + task = q.with_for_update().first() + + # There's nothing to process in the first place. + if not task: + return + + # Update the task so that it is processed by this instance. + session.query(Task).filter_by(id=task.id).update({ + "processing": instance, + }) + + session.commit() + session.refresh(task) + + # Only return the task if it was really assigned to this node. It + # could be, e.g., in sqlite3, that the locking is misbehaving. + if task.processing == instance: + return task.id + except SQLAlchemyError as e: + log.exception("Database error getting new processing tasks: %s", e) + finally: + session.close() diff --git a/cuckoo/core/extract.py b/cuckoo/core/extract.py new file mode 100644 index 0000000000..f7b1e0bbbb --- /dev/null +++ b/cuckoo/core/extract.py @@ -0,0 +1,203 @@ +# Copyright (C) 2017-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import egghatch +import logging +import os + +from cuckoo.common.abstracts import Extractor +from cuckoo.common.exceptions import CuckooCriticalError +from cuckoo.common.objects import File, YaraMatch, Buffer +from cuckoo.common.scripting import Scripting +from cuckoo.common.utils import supported_version +from cuckoo.misc import cwd, version + +log = logging.getLogger(__name__) + +class ExtractManager(object): + _instances = {} + extractors = [] + + @staticmethod + def for_task(task_id): + if task_id not in ExtractManager._instances: + ExtractManager._instances[task_id] = ExtractManager(task_id) + return ExtractManager._instances[task_id] + + def __init__(self, task_id): + self.task_id = task_id + self.items = [] + self.payloads = {} + + @classmethod + def init_once(cls): + cls.extractors = [] + + # Gather all up-to-date Extractors. TODO Also handle nested subclasses. + for ext in Extractor.__subclasses__(): + if not supported_version(version, ext.minimum, ext.maximum): + log.debug( + "You are running a version of Cuckoo that's not " + "compatible with this Extractor (either it's too old or " + "too new): cuckoo=%s extractor=%s minversion=%s " + "maxversion=%s", + version, ext.__name__, ext.minimum, ext.maximum + ) + continue + + cls.extractors.append(ext) + + # Turn str/unicode into a tuple of size one. + if isinstance(ext.yara_rules, basestring): + ext.yara_rules = ext.yara_rules, + + def __del__(self): + self._instances.pop(self.task_id, None) + + def write_extracted(self, ext, payload): + dirpath = cwd("extracted", analysis=self.task_id) + + # TODO We need to move this somewhere else. Just a temporary + # hack in case old reports are processed that don't have the + # "extracted" directory in-place yet. + if not os.path.exists(dirpath): + os.mkdir(dirpath) + + # Handle duplicate payloads. + if payload in self.payloads: + return + + self.payloads[payload] = True + + # TODO Implement some rate-limiting here. + + filepath = os.path.join(dirpath, "%d.%s" % (len(self.items), ext)) + open(filepath, "wb").write(payload) + return filepath + + def push_command_line(self, cmdline, process=None): + command = Scripting().parse_command(cmdline) + if command and command.get_script(): + self.push_script(process, command) + + def push_script(self, process, command): + filepath = self.write_extracted( + command.ext, command.get_script().encode("utf8") + ) + if not filepath: + return + + process = process or {} + + yara_matches = File(filepath).get_yara("scripts") + self.items.append({ + "category": "script", + "program": command.program, + "pid": process.get("pid"), + "first_seen": process.get("first_seen"), + "raw": filepath, + "yara": yara_matches, + "info": {}, + }) + for match in yara_matches: + match = YaraMatch(match, "script") + self.handle_yara(filepath, match) + + def push_script_recursive(self, command): + self.push_script(None, command) + for child in command.children: + self.push_script_recursive(child) + + def push_shellcode(self, sc): + filepath = self.write_extracted("bin", sc) + if not filepath: + return + + # This file contains a plaintext representation of the shellcode. + open("%s.txt" % filepath, "wb").write(egghatch.as_text(sc)) + + yara_matches = File(filepath).get_yara("shellcode") + self.items.append({ + "category": "shellcode", + "raw": filepath, + "yara": yara_matches, + "info": { + "pretty": "%s.txt" % filepath, + }, + }) + for match in yara_matches: + match = YaraMatch(match, "shellcode") + self.handle_yara(filepath, match) + + def push_blob(self, blob, category, externals, info=None): + filepath = self.write_extracted("blob", blob) + if not filepath: + return + + yara_matches = File(filepath).get_yara(category, externals) + + self.items.append({ + "category": category, + "raw": filepath, + "yara": yara_matches, + "info": info or {}, + }) + for match in yara_matches: + match = YaraMatch(match, category) + self.handle_yara(filepath, match) + + def push_blob_noyara(self, blob, category, info=None): + filepath = self.write_extracted("blob", blob) + if not filepath: + return + + self.items.append({ + "category": category, + "raw": filepath, + "yara": [], + "info": info or {}, + }) + + def push_config(self, config): + if not isinstance(config, dict) or "family" not in config: + raise CuckooCriticalError("Invalid call to push_config().") + + self.items.append({ + "category": "config", + "raw": None, + "yara": [], + "info": config, + }) + + def enhance(self, filepath, key, value): + for item in self.items: + if item["raw"] == filepath: + item["info"][key] = value + break + + def peek_office(self, files): + for filename, content in files.items(): + externals = { + "filename": filename, + } + if Buffer(content).get_yara_quick("office", externals): + self.push_blob(content, "office", externals) + + def peek_procmem(self, process): + for match in process["yara"]: + self.handle_yara(process["file"], YaraMatch(match)) + + def handle_yara(self, filepath, match): + for plugin in self.extractors: + if match.name in plugin.yara_rules: + try: + plugin(self).handle_yara(filepath, match) + except Exception as e: + log.exception( + "Exception in an Extractor's handle_yara: %s", e + ) + + def results(self): + # TODO Apply some sort of sorting here. + return self.items diff --git a/cuckoo/core/feedback.py b/cuckoo/core/feedback.py new file mode 100644 index 0000000000..758f62c8fb --- /dev/null +++ b/cuckoo/core/feedback.py @@ -0,0 +1,306 @@ +# Copyright (C) 2016-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import io +import json +import logging +import os +import requests +import traceback +import zipfile + +from cuckoo.common.config import Config, config +from cuckoo.common.exceptions import CuckooFeedbackError +from cuckoo.core.report import Report +from cuckoo.misc import version, cwd + +log = logging.getLogger(__name__) + +class CuckooFeedback(object): + """Contact Cuckoo HQ with feedback & optional analysis dump.""" + endpoint = "https://feedback.cuckoosandbox.org/api/submit/" + exc_safelist = ( + CuckooFeedbackError, + ) + + def enabled(self): + return config("cuckoo:feedback:enabled") + + def send_exception(self, exception, request): + """ + To be used during exception handling. + @param exception: The exception class + @param request: Django request object + @return: + """ + if not self.enabled(): + return + + feedback = CuckooFeedbackObject( + automated=True, message="Exception encountered: %s" % exception + ) + + if isinstance(exception, self.exc_safelist): + log.debug("A safelisted exception occurred: %s", exception) + return + + # Ignore 404 exceptions regarding ".map" development files. + from django.http import Http404 + if isinstance(exception, Http404) and ".map" in exception.message: + return + + from django.template import TemplateSyntaxError, TemplateDoesNotExist + if isinstance(exception, (TemplateSyntaxError, TemplateDoesNotExist)): + feedback.add_error( + "A Django-related exception occurred: %s" % exception + ) + + feedback.add_traceback() + + class options(object): + analysis = False + json_report = False + memdump = False + config = True + + if request: + if hasattr(request, "resolver_match") and request.resolver_match: + if request.method == "POST" and request.is_ajax(): + kwargs = json.loads(request.body) + else: + kwargs = request.resolver_match.kwargs + elif request.method == "GET": + kwargs = request.GET + elif request.method == "POST": + kwargs = request.POST + else: + kwargs = {} + + task_id = ( + kwargs.get("task_id", kwargs.get("analysis_id")) + ) + + if task_id: + options.analysis = True + options.json_report = True + + if options.json_report: + feedback.include_report_web(task_id) + + if feedback.report and options.analysis: + feedback.include_analysis(memdump=options.memdump) + + return self.send_feedback(feedback) + + def send_form(self, task_id=None, name=None, email=None, message=None, + company=None, json_report=False, memdump=False): + feedback = CuckooFeedbackObject( + name=name, company=company, email=email, + message=message, automated=False + ) + + if json_report: + if not task_id or not isinstance(task_id, int): + raise CuckooFeedbackError( + "An incorrect Task ID has been provided: %s!" % task_id + ) + + feedback.include_report_web(task_id) + feedback.include_analysis(memdump=memdump) + + return self.send_feedback(feedback) + + def send_feedback(self, feedback): + try: + feedback.validate() + except CuckooFeedbackError as e: + raise CuckooFeedbackError( + "Could not validate feedback object: %s" % e + ) + + headers = { + "Accept": "application/json", + "User-Agent": "Cuckoo %s" % version + } + + try: + r = requests.post( + self.endpoint, + data={ + "feedback": json.dumps(feedback.to_dict()), + }, + files=feedback.to_files(), + headers=headers + ) + r.raise_for_status() + + obj = r.json() + if not obj.get("status"): + raise CuckooFeedbackError(obj["message"]) + return obj["feedback_id"] + except requests.RequestException as e: + raise CuckooFeedbackError( + "Invalid response from Cuckoo feedback server: %s" % e + ) + except CuckooFeedbackError as e: + raise CuckooFeedbackError( + "Cuckoo feedback error while trying to send: %s" % e + ) + +class CuckooFeedbackObject(object): + """Feedback object.""" + export_files = [ + "analysis.log", + "cuckoo.log", + "dump.pcap", + "tlsmaster.txt", + ("logs", ".bson"), + ("shots", ".jpg"), + ] + + def __init__(self, message=None, email=None, name=None, company=None, + automated=False): + self.automated = automated + self.message = message + self.contact = { + "name": name or config("cuckoo:feedback:name"), + "company": company or config("cuckoo:feedback:company"), + "email": email or config("cuckoo:feedback:email"), + } + self.errors = [] + self.traceback = None + self.export = [] + self.info = {} + self.report = None + + def include_report(self, report): + # Any and all errors. + for error in report.errors: + self.add_error(error) + + # Analysis information. + if report.target["category"] == "file": + self.info["file"] = report.target["file"] + elif report.target["category"] == "url": + self.info["url"] = report.target["url"] + + self.info["category"] = report.target["category"] + self.report = report + + def include_report_web(self, task_id): + from cuckoo.web.controllers.analysis.analysis import AnalysisController + from django.http import Http404 + try: + report = Report(AnalysisController.get_report(task_id)["analysis"]) + except Http404: + # No report available so ignoring the rest of this function. + return + + return self.include_report(report) + + def gather_export_files(self, dirpath): + """Return a list of all files of interest from an analysis.""" + ret = [] + for name in self.export_files: + if isinstance(name, basestring): + filepath = os.path.join(dirpath, name) + if os.path.isfile(filepath): + ret.append((name, filepath)) + continue + + if isinstance(name, tuple): + dirname, ext = name + dirpath = os.path.join(dirpath, dirname) + if os.path.isdir(dirpath): + for filename in os.listdir(dirpath): + if not filename.endswith(ext): + continue + + filepath = os.path.join(dirpath, filename) + if not os.path.isfile(filepath): + continue + + filename = "%s/%s" % (dirname, filename) + ret.append((filename, filepath)) + continue + + raise RuntimeError( + "Unknown export file defined: %s!" % name + ) + self.export = ret + + def include_analysis(self, memdump=False): + if not self.report: + raise CuckooFeedbackError( + "Report must be included first when including the analysis." + ) + + if "analysis_path" not in self.report.info: + raise CuckooFeedbackError( + "Can't include the entire analysis for this analysis as the " + "analysis path isn't known." + ) + + if not os.path.isdir(self.report.info["analysis_path"]): + raise CuckooFeedbackError( + "Can't include the entire analysis for this analysis as the " + "analysis path doesn't exist." + ) + + # TODO Support for also including memory dumps (should we?) and/or + # behavioral logs or at least something like matched signatures. + self.gather_export_files(self.report.info["analysis_path"]) + + def add_error(self, error): + self.errors.append(error) + + def add_traceback(self, tb=None): + self.traceback = tb or traceback.format_exc() + + def validate(self): + if not self.contact.get("name"): + raise CuckooFeedbackError("Missing contact name") + + if not self.contact.get("email"): + raise CuckooFeedbackError("Missing contact email") + + from django.core.validators import validate_email, ValidationError + try: + validate_email(self.contact["email"]) + except ValidationError: + raise CuckooFeedbackError( + "Invalid email address: %s!" % self.contact["email"] + ) + + if not self.message: + raise CuckooFeedbackError("Missing feedback message") + + return True + + def to_dict(self): + return { + "version": version, + "errors": self.errors, + "traceback": self.traceback, + "contact": self.contact, + "automated": self.automated, + "message": self.message, + "info": self.info, + "cuckoo": { + "cwd": cwd(), + "app": os.environ.get("CUCKOO_APP"), + "config": Config.from_confdir(cwd("conf"), sanitize=True), + }, + } + + def to_files(self): + buf = io.BytesIO() + z = zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) + for filename, filepath in self.export: + z.write(filepath, filename) + z.close() + buf.seek(0) + return { + "file": buf, + } diff --git a/cuckoo/core/guest.py b/cuckoo/core/guest.py new file mode 100644 index 0000000000..a75f5a3950 --- /dev/null +++ b/cuckoo/core/guest.py @@ -0,0 +1,585 @@ +# Copyright (C) 2012-2013 Claudio Guarnieri. +# Copyright (C) 2014-2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import datetime +import io +import json +import logging +import os +import requests +import socket +import time +import xmlrpclib +import zipfile + +from cuckoo.common.config import config, parse_options +from cuckoo.common.constants import ( + CUCKOO_GUEST_PORT, CUCKOO_GUEST_INIT, CUCKOO_GUEST_COMPLETED, + CUCKOO_GUEST_FAILED +) +from cuckoo.common.exceptions import ( + CuckooGuestError, CuckooGuestCriticalTimeout +) +from cuckoo.common.utils import TimeoutServer +from cuckoo.core.database import Database +from cuckoo.misc import cwd + +log = logging.getLogger(__name__) +db = Database() + +def analyzer_zipfile(platform, monitor): + """Create the zip file that is sent to the Guest.""" + t = time.time() + + zip_data = io.BytesIO() + zip_file = zipfile.ZipFile(zip_data, "w", zipfile.ZIP_STORED) + + # Select the proper analyzer's folder according to the operating + # system associated with the current machine. + root = cwd("analyzer", platform) + root_len = len(os.path.abspath(root)) + + if not os.path.exists(root): + log.error("No valid analyzer found at path: %s", root) + raise CuckooGuestError( + "No valid analyzer found for %s platform!" % platform + ) + + # Walk through everything inside the analyzer's folder and write + # them to the zip archive. + for root, dirs, files in os.walk(root): + archive_root = os.path.abspath(root)[root_len:] + for name in files: + path = os.path.join(root, name) + archive_name = os.path.join(archive_root, name) + zip_file.write(path, archive_name) + + # Include the chosen monitoring component and any additional files. + if platform == "windows": + dirpath = cwd("monitor", monitor) + + # Generally speaking we should no longer be getting symbolic links for + # "latest" anymore, so in the case of a file; follow it. + if os.path.isfile(dirpath): + monitor = os.path.basename(open(dirpath, "rb").read().strip()) + dirpath = cwd("monitor", monitor) + + for name in os.listdir(dirpath): + zip_file.write( + os.path.join(dirpath, name), os.path.join("bin", name) + ) + + # Dump compiled "dumpmem" Yara rules for zer0m0n usage. + zip_file.write(cwd("stuff", "dumpmem.yarac"), "bin/rules.yarac") + + zip_file.close() + data = zip_data.getvalue() + + if time.time() - t > 10: + log.warning( + "It took more than 10 seconds to build the Analyzer Zip for the " + "Guest. This might be a serious performance penalty. Is your " + "analyzer/windows/ directory bloated with unnecessary files?" + ) + + return data + +class OldGuestManager(object): + """Old and deprecated Guest Manager. + + This class handles the communications with the old agent running in the + virtual machine. + """ + + def __init__(self, vm_id, ip, platform, task_id): + """@param ip: guest's IP address. + @param platform: guest's operating system type. + """ + self.id = vm_id + self.ip = ip + self.platform = platform + self.task_id = task_id + + # initialized in start_analysis so we can update the critical timeout + # TODO, pull options parameter into __init__ so we can do this here + self.timeout = None + self.server = None + self.do_run = True + + def wait(self, status): + """Waiting for status. + @param status: status. + @return: always True. + """ + log.debug("%s: waiting for status 0x%.04x", self.id, status) + + end = time.time() + self.timeout + self.server._set_timeout(self.timeout) + + while db.guest_get_status(self.task_id) == "starting" and self.do_run: + # Check if we've passed the timeout. + if time.time() > end: + raise CuckooGuestCriticalTimeout( + "Machine %s: the guest initialization hit the " + "critical timeout, analysis aborted." % self.id + ) + + try: + # If the server returns the given status, break the loop + # and return. + if self.server.get_status() == status: + log.debug("%s: status ready", self.id) + break + except: + pass + + log.debug("%s: not ready yet", self.id) + time.sleep(1) + + self.server._set_timeout(None) + return True + + def upload_analyzer(self, monitor): + """Upload analyzer to guest. + @return: operation status. + """ + zip_data = analyzer_zipfile(self.platform, monitor) + + log.debug( + "Uploading analyzer to guest (id=%s, ip=%s, monitor=%s, size=%d)", + self.id, self.ip, monitor, len(zip_data) + ) + + # Send the zip containing the analyzer to the agent running inside + # the guest. + try: + self.server.add_analyzer(xmlrpclib.Binary(zip_data)) + except socket.timeout: + raise CuckooGuestError("{0}: guest communication timeout: unable " + "to upload agent, check networking or try " + "to increase timeout".format(self.id)) + + def start_analysis(self, options, monitor): + """Start analysis. + @param options: options. + @return: operation status. + """ + # TODO Deal with unicode URLs, should probably try URL encoding. + # Unicode files are being taken care of. + + self.timeout = options["timeout"] + config("cuckoo:timeouts:critical") + + url = "http://{0}:{1}".format(self.ip, CUCKOO_GUEST_PORT) + self.server = TimeoutServer(url, allow_none=True, + timeout=self.timeout) + + try: + # Wait for the agent to respond. This is done to check the + # availability of the agent and verify that it's ready to receive + # data. + self.wait(CUCKOO_GUEST_INIT) + if not self.do_run: + return + + # Invoke the upload of the analyzer to the guest. + self.upload_analyzer(monitor) + + # Give the analysis options to the guest, so it can generate the + # analysis.conf inside the guest. + try: + self.server.add_config(options) + except: + raise CuckooGuestError( + "%s: unable to upload config to analysis machine" % + self.id + ) + + # If the target of the analysis is a file, upload it to the guest. + if options["category"] in ("file", "archive"): + try: + file_data = open(options["target"], "rb").read() + except (IOError, OSError) as e: + raise CuckooGuestError( + "Unable to read %s, error: %s" % + (options["target"], e) + ) + + data = xmlrpclib.Binary(file_data) + + try: + self.server.add_malware(data, options["file_name"]) + except Exception as e: + raise CuckooGuestError( + "#%s: unable to upload malware to analysis " + "machine: %s" % (self.id, e) + ) + + # Launch the analyzer. + pid = self.server.execute() + log.debug("%s: analyzer started with PID %d", self.id, pid) + # If something goes wrong when establishing the connection, raise an + # exception and abort the analysis. + except (socket.timeout, socket.error): + raise CuckooGuestError( + "%s: guest communication timeout, check networking or try " + "to increase timeout" % self.id + ) + + def wait_for_completion(self): + """Wait for analysis completion. + @return: operation status. + """ + log.debug("%s: waiting for completion", self.id) + + end = time.time() + self.timeout + self.server._set_timeout(self.timeout) + + while db.guest_get_status(self.task_id) == "running" and self.do_run: + time.sleep(1) + + # If the analysis hits the critical timeout, just return straight + # away and try to recover the analysis results from the guest. + if time.time() > end: + log.info("%s: end of analysis reached!", self.id) + return + + try: + status = self.server.get_status() + except Exception as e: + log.debug("%s: error retrieving status: %s", self.id, e) + continue + + # React according to the returned status. + if status == CUCKOO_GUEST_COMPLETED: + log.info("%s: analysis completed successfully", self.id) + break + elif status == CUCKOO_GUEST_FAILED: + error = self.server.get_error() + raise CuckooGuestError( + "Analysis failed: %s" % (error or "unknown error") + ) + else: + log.debug("%s: analysis not completed yet (status=%s)", + self.id, status) + + self.server._set_timeout(None) + +class GuestManager(object): + """This class represents the new Guest Manager. It operates on the new + Cuckoo Agent which features a more abstract but more feature-rich API.""" + + def __init__(self, vmid, ipaddr, platform, task_id, analysis_manager): + self.vmid = vmid + self.ipaddr = ipaddr + self.port = CUCKOO_GUEST_PORT + self.platform = platform + self.task_id = task_id + self.analysis_manager = analysis_manager + self.timeout = None + + # Just in case we have an old agent inside the Virtual Machine. This + # allows us to remain backwards compatible (for now). + self.old = OldGuestManager(vmid, ipaddr, platform, task_id) + self.is_old = False + + # We maintain the path of the Cuckoo Analyzer on the host. + self.analyzer_path = None + self.environ = {} + + self.options = {} + self.do_run = True + + @property + def aux(self): + return self.analysis_manager.aux + + def stop(self): + self.do_run = False + if self.is_old: + self.old.do_run = False + + def get(self, method, *args, **kwargs): + """Simple wrapper around requests.get().""" + do_raise = kwargs.pop("do_raise", True) + url = "http://%s:%s%s" % (self.ipaddr, self.port, method) + session = requests.Session() + session.trust_env = False + session.proxies = None + + try: + r = session.get(url, *args, **kwargs) + except requests.ConnectionError: + raise CuckooGuestError( + "Cuckoo Agent failed without error status, please try " + "upgrading to the latest version of agent.py (>= 0.10) and " + "notify us if the issue persists." + ) + + do_raise and r.raise_for_status() + return r + + def post(self, method, *args, **kwargs): + """Simple wrapper around requests.post().""" + url = "http://%s:%s%s" % (self.ipaddr, self.port, method) + session = requests.Session() + session.trust_env = False + session.proxies = None + + try: + r = session.post(url, *args, **kwargs) + except requests.ConnectionError: + raise CuckooGuestError( + "Cuckoo Agent failed without error status, please try " + "upgrading to the latest version of agent.py (>= 0.10) and " + "notify us if the issue persists." + ) + + r.raise_for_status() + return r + + def wait_available(self): + """Wait until the Virtual Machine is available for usage.""" + end = time.time() + self.timeout + + while db.guest_get_status(self.task_id) == "starting" and self.do_run: + try: + socket.create_connection((self.ipaddr, self.port), 1).close() + break + except socket.timeout: + log.debug("%s: not ready yet", self.vmid) + except socket.error: + log.debug("%s: not ready yet", self.vmid) + time.sleep(1) + + if time.time() > end: + raise CuckooGuestCriticalTimeout( + "Machine %s: the guest initialization hit the critical " + "timeout, analysis aborted." % self.vmid + ) + + def query_environ(self): + """Query the environment of the Agent in the Virtual Machine.""" + self.environ = self.get("/environ").json()["environ"] + + def determine_analyzer_path(self): + """Determine the path of the analyzer. Basically creating a temporary + directory in the systemdrive, i.e., C:\\.""" + systemdrive = self.determine_system_drive() + + options = parse_options(self.options["options"]) + if options.get("analpath"): + dirpath = systemdrive + options["analpath"] + r = self.post("/mkdir", data={"dirpath": dirpath}) + self.analyzer_path = dirpath + else: + r = self.post("/mkdtemp", data={"dirpath": systemdrive}) + self.analyzer_path = r.json()["dirpath"] + + def determine_system_drive(self): + if self.platform == "windows": + return "%s/" % self.environ["SYSTEMDRIVE"] + return "/" + + def determine_temp_path(self): + if self.platform == "windows": + return self.environ["TEMP"] + return "/tmp" + + def upload_analyzer(self, monitor): + """Upload the analyzer to the Virtual Machine.""" + zip_data = analyzer_zipfile(self.platform, monitor) + + log.debug( + "Uploading analyzer to guest (id=%s, ip=%s, monitor=%s, size=%d)", + self.vmid, self.ipaddr, monitor, len(zip_data) + ) + + self.determine_analyzer_path() + data = { + "dirpath": self.analyzer_path, + } + self.post("/extract", files={"zipfile": zip_data}, data=data) + + def add_config(self, options): + """Upload the analysis.conf for this task to the Virtual Machine.""" + config = [ + "[analysis]", + ] + for key, value in options.items(): + # Encode datetime objects the way xmlrpc encodes them. + if isinstance(value, datetime.datetime): + config.append("%s = %s" % (key, value.strftime("%Y%m%dT%H:%M:%S"))) + else: + config.append("%s = %s" % (key, value)) + + data = { + "filepath": os.path.join(self.analyzer_path, "analysis.conf"), + } + self.post("/store", files={"file": "\n".join(config)}, data=data) + + def start_analysis(self, options, monitor): + """Start the analysis by uploading all required files. + + @param options: the task options + @param monitor: identifier of the monitor to be used. + """ + log.info("Starting analysis #%s on guest (id=%s, ip=%s)", + self.task_id, self.vmid, self.ipaddr) + + self.options = options + self.timeout = options["timeout"] + config("cuckoo:timeouts:critical") + + # Wait for the agent to come alive. + self.wait_available() + if not self.do_run: + return + + # Could be beautified a bit, but basically we have to perform the + # same check here as we did in wait_available(). + if db.guest_get_status(self.task_id) != "starting": + return + + # Check whether this is the new Agent or the old one (by looking at + # the status code of the index page). + r = self.get("/", do_raise=False) + if r.status_code == 501: + # log.info("Cuckoo 2.0 features a new Agent which is more " + # "feature-rich. It is recommended to make new Virtual " + # "Machines with the new Agent, but for now falling back " + # "to backwards compatibility with the old agent.") + self.is_old = True + self.aux.callback("legacy_agent") + self.old.start_analysis(options, monitor) + return + + if r.status_code != 200: + log.critical( + "While trying to determine the Agent version that your VM is " + "running we retrieved an unexpected HTTP status code: %s. If " + "this is a false positive, please report this issue to the " + "Cuckoo Developers. HTTP response headers: %s", + r.status_code, json.dumps(dict(r.headers)), + ) + db.guest_set_status(self.task_id, "failed") + return + + try: + status = r.json() + version = status.get("version") + features = status.get("features", []) + except: + log.critical( + "We were unable to detect either the Old or New Agent in the " + "Guest VM, are you sure you have set it up correctly? Please " + "go through the documentation once more and otherwise inform " + "the Cuckoo Developers of your issue." + ) + db.guest_set_status(self.task_id, "failed") + return + + log.info("Guest is running Cuckoo Agent %s (id=%s, ip=%s)", + version, self.vmid, self.ipaddr) + + # Pin the Agent to our IP address so that it is not accessible by + # other Virtual Machines etc. + if "pinning" in features: + self.get("/pinning") + + # Obtain the environment variables. + self.query_environ() + + # Upload the analyzer. + self.upload_analyzer(monitor) + + # Pass along the analysis.conf file. + self.add_config(options) + + # Allow Auxiliary modules to prepare the Guest. + self.aux.callback("prepare_guest") + + # If the target is a file, upload it to the guest. + if options["category"] == "file" or options["category"] == "archive": + data = { + "filepath": os.path.join( + self.determine_temp_path(), options["file_name"] + ), + } + files = { + "file": ("sample.bin", open(options["target"], "rb")), + } + self.post("/store", files=files, data=data) + + if "execpy" in features: + data = { + "filepath": "%s/analyzer.py" % self.analyzer_path, + "async": "yes", + "cwd": self.analyzer_path, + } + self.post("/execpy", data=data) + else: + # Execute the analyzer that we just uploaded. + data = { + "command": "C:\\Python27\\pythonw.exe %s\\analyzer.py" % self.analyzer_path, + "async": "yes", + "cwd": self.analyzer_path, + } + self.post("/execute", data=data) + + def wait_for_completion(self): + if self.is_old: + self.old.wait_for_completion() + return + + count = 0 + end = time.time() + self.timeout + + while db.guest_get_status(self.task_id) == "running" and self.do_run: + if count >= 5: + log.debug( + "%s: analysis #%s still processing", self.vmid, + self.task_id + ) + count = 0 + + count += 1 + time.sleep(1) + + # If the analysis hits the critical timeout, just return straight + # away and try to recover the analysis results from the guest. + if time.time() > end: + log.info("%s: end of analysis reached!", self.vmid) + return + + try: + status = self.get("/status", timeout=5).json() + except CuckooGuestError: + # this might fail due to timeouts or just temporary network + # issues thus we don't want to abort the analysis just yet and + # wait for things to recover + log.warning( + "Virtual Machine /status failed. This can indicate the " + "guest losing network connectivity" + ) + continue + except Exception as e: + log.error("Virtual machine /status failed. %s", e) + continue + + if status["status"] == "complete": + log.info("%s: analysis completed successfully", self.vmid) + return + elif status["status"] == "exception": + log.warning( + "%s: analysis #%s caught an exception\n%s", + self.vmid, self.task_id, status["description"] + ) + return + + @property + def server(self): + """Currently the Physical machine manager is using GuestManager in + an incorrect way. This should be fixed up later but for now this + workaround will do.""" + return self.old.server diff --git a/cuckoo/core/init.py b/cuckoo/core/init.py new file mode 100644 index 0000000000..0fdb1b9da0 --- /dev/null +++ b/cuckoo/core/init.py @@ -0,0 +1,111 @@ +# Copyright (C) 2016-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import jinja2 +import os + +from cuckoo.common.config import Config +from cuckoo.common.exceptions import CuckooConfigurationError +from cuckoo.common.utils import random_token +from cuckoo.misc import cwd + +def write_supervisor_conf(username): + """Write supervisord.conf configuration file if it does not exist yet.""" + # TODO Handle updates? + if os.path.exists(cwd("supervisord.conf")): + return + + if os.environ.get("VIRTUAL_ENV"): + virtualenv = os.path.join(os.environ["VIRTUAL_ENV"], "bin") + python_path = os.path.join(virtualenv, "python") + cuckoo_path = os.path.join(virtualenv, "cuckoo") + else: + python_path = "python" + cuckoo_path = "cuckoo" + + template = jinja2.Environment().from_string( + open(cwd("cwd", "supervisord.jinja2", private=True), "rb").read() + ) + + with open(cwd("supervisord.conf"), "wb") as f: + f.write(template.render({ + "cwd": cwd, + "username": username, + "cuckoo_path": cuckoo_path, + "python_path": python_path, + }).rstrip().encode("utf8") + "\n") + +def write_cuckoo_conf(cfg=None): + if cfg is None: + cfg = {} + + # Merge any provided configuration with the defaults and emit their values. + raw = {} + for filename, sections in Config.configuration.items(): + cfg[filename] = cfg.get(filename, {}) + raw[filename] = {} + for section, entries in sections.items(): + if section == "__star__": + continue + + # Process each entry. + if not isinstance(entries, (tuple, list)): + entries = entries, + + for entry in entries: + real_section = entry.get("__section__", section) + entries = cfg[filename].get(section, {}) + entries.update(cfg[filename].get(real_section, {})) + cfg[filename][real_section] = entries + raw[filename][real_section] = {} + for key, value in entry.items(): + if key == "__section__": + continue + + raw_value = cfg[filename][real_section].get(key, value.default) + cfg[filename][real_section][key] = raw_value + raw[filename][real_section][key] = value.emit(raw_value) + + if "__star__" in sections: + section, key = sections["__star__"] + for entry in cfg[filename][section][key]: + if entry not in cfg[filename]: + raise CuckooConfigurationError( + "A section was defined that has not been found: " + "%s:%s" % (section, entry) + ) + + if isinstance(sections["*"], (tuple, list)): + section_types = sections["*"][0] + else: + section_types = sections["*"] + + raw[filename][entry] = {} + for key, value in section_types.items(): + if key == "__section__": + continue + + if key not in cfg[filename][entry]: + raw_value = cfg[filename][entry][key] = None + else: + raw_value = cfg[filename][entry][key] + + raw[filename][entry][key] = value.emit(raw_value) + + # Not the most beautiful, but create a random API token here. + cfg["cuckoo"]["cuckoo"]["api_token"] = random_token() + raw["cuckoo"]["cuckoo"]["api_token"] = random_token() + + def _config(s): + filename, section, key = s.split(":") + return cfg[filename][section][key] + + raw["config"] = _config + for filename in os.listdir(cwd("cwd", "conf", private=True)): + template = jinja2.Template( + open(cwd("cwd", "conf", filename, private=True), "rb").read() + ) + open(cwd("conf", filename), "wb").write( + template.render(raw).rstrip() + "\n" + ) diff --git a/cuckoo/core/log.py b/cuckoo/core/log.py new file mode 100644 index 0000000000..64437844d0 --- /dev/null +++ b/cuckoo/core/log.py @@ -0,0 +1,196 @@ +# Copyright (C) 2016-2019 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import copy +import json +import logging +import logging.handlers +import os +import threading +import time + +import gevent.thread + +from cuckoo.common.colors import red, yellow, cyan +from cuckoo.core.database import Database +from cuckoo.misc import cwd + +_task_threads = {} +_tasks = {} +_loggers = {} + +_tasks_lock = threading.Lock() + +# Current GMT+x. +if time.localtime().tm_isdst: + tz = time.altzone / -3600. +else: + tz = time.timezone / -3600. + +# The greenlet library (used by Gevent) also creates some state per thread, +# so we can (ab)use this for both multi-threading and Gevent code +task_key = gevent.thread.get_ident + +class DatabaseHandler(logging.Handler): + """Logging to database handler. + Used to log errors related to tasks in database. + """ + + def emit(self, record): + # TODO Should this also attempt to guess the task ID from _tasks? + if hasattr(record, "task_id"): + Database().add_error( + self.format(record), int(record.task_id), + getattr(record, "error_action", None) + ) + +class TaskHandler(logging.Handler): + """Per-task logger. + Used to log all task specific events to a per-task cuckoo.log log file. + """ + + def emit(self, record): + task = _tasks.get(task_key()) + if not task: + return + + task[1].write("%s\n" % self.format(record)) + +class ConsoleHandler(logging.StreamHandler): + """Logging to console handler.""" + + def emit(self, record): + colored = copy.copy(record) + + if record.levelname == "WARNING": + colored.msg = yellow(record.msg) + elif record.levelname == "ERROR" or record.levelname == "CRITICAL": + colored.msg = red(record.msg) + else: + if "analysis procedure completed" in record.msg: + colored.msg = cyan(record.msg) + else: + colored.msg = record.msg + + logging.StreamHandler.emit(self, colored) + +class JsonFormatter(logging.Formatter): + """Logging Cuckoo logs to JSON.""" + + def format(self, record): + action = record.__dict__.get("action") + status = record.__dict__.get("status") + task = _tasks.get(task_key()) + task_id = task[0] if task else record.__dict__.get("task_id") + d = { + "action": action, + "task_id": task_id, + "status": status, + "time": record.created, + "level": record.levelname.lower(), + "message": record.getMessage(), + } + base = logging.makeLogRecord({}) + for key, value in record.__dict__.items(): + if key not in base.__dict__: + d[key] = value + return json.dumps(d) + + def filter(self, record): + action = record.__dict__.get("action") + status = record.__dict__.get("status") + return action and status + +def task_log_start(task_id): + """Associate a thread with a task.""" + _tasks_lock.acquire() + try: + if task_id not in _task_threads: + task_path = cwd(analysis=task_id) + if not os.path.exists(task_path): + return + + _task_threads[task_id] = [] + fp = open(os.path.join(task_path, "cuckoo.log"), "a+b") + _tasks[task_key()] = (task_id, fp) + else: + existing_key = _task_threads[task_id][0] + _tasks[task_key()] = _tasks[existing_key] + + _task_threads[task_id].append(task_key()) + finally: + _tasks_lock.release() + +def task_log_stop(task_id): + """Disassociate a thread from a task.""" + _tasks_lock.acquire() + try: + thread_key = task_key() + if thread_key not in _tasks: + return + + _, fp = _tasks.pop(thread_key) + _task_threads[task_id].remove(thread_key) + if not _task_threads[task_id]: + fp.close() + _task_threads.pop(task_id) + finally: + _tasks_lock.release() + +def init_logger(name, level=None): + formatter = logging.Formatter( + "%(asctime)s [%(name)s] %(levelname)s: %(message)s" + ) + + if name == "cuckoo.log": + l = logging.handlers.WatchedFileHandler(cwd("log", "cuckoo.log")) + l.setFormatter(formatter) + l.setLevel(level) + + if name == "cuckoo.json": + j = JsonFormatter() + l = logging.handlers.WatchedFileHandler(cwd("log", "cuckoo.json")) + l.setFormatter(j) + l.addFilter(j) + + if name == "console": + l = ConsoleHandler() + l.setFormatter(formatter) + l.setLevel(level) + + if name == "database": + l = DatabaseHandler() + l.setLevel(logging.ERROR) + + if name == "task": + l = TaskHandler() + l.setFormatter(formatter) + + if name.startswith("process-") and name.endswith(".json"): + j = JsonFormatter() + l = logging.handlers.WatchedFileHandler(cwd("log", name)) + l.setFormatter(j) + l.addFilter(j) + + _loggers[name] = l + logging.getLogger().addHandler(l) + +def logger(message, *args, **kwargs): + """Log a message to specific logger instance.""" + logfile = kwargs.pop("logfile", None) + record = logging.LogRecord( + None, logging.INFO, None, None, message, args, None, None + ) + record.asctime = "%s,%03d" % ( + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(record.created)), + record.msecs + ) + record.message = record.getMessage() + record.__dict__.update(kwargs) + + for key, value in _loggers.items(): + if logfile and key == logfile: + value.handle(record) + if logfile is None and key.endswith(".json"): + value.handle(record) diff --git a/cuckoo/core/plugins.py b/cuckoo/core/plugins.py new file mode 100644 index 0000000000..83496641cb --- /dev/null +++ b/cuckoo/core/plugins.py @@ -0,0 +1,696 @@ +# Copyright (C) 2012-2013 Claudio Guarnieri. +# Copyright (C) 2014-2019 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import importlib +import inspect +import json +import logging +import os +import pkgutil +import sys + +import cuckoo + +from cuckoo.common.abstracts import Configuration, Signature +from cuckoo.common.config import config2 +from cuckoo.common.exceptions import ( + CuckooConfigurationError, CuckooProcessingError, CuckooReportError, + CuckooDependencyError, CuckooDisableModule, CuckooOperationalError +) +from cuckoo.common.objects import YaraMatch, ExtractedMatch +from cuckoo.common.utils import supported_version +from cuckoo.core.extract import ExtractManager +from cuckoo.misc import cwd, version as cuckoo_version + +log = logging.getLogger(__name__) + +def enumerate_plugins(dirpath, module_prefix, namespace, class_, + attributes={}, as_dict=False): + """Import plugins of type `class` located at `dirpath` into the + `namespace` that starts with `module_prefix`. If `dirpath` represents a + filepath then it is converted into its containing directory. The + `attributes` dictionary allows one to set extra fields for all imported + plugins. Using `as_dict` a dictionary based on the module name is + returned.""" + if os.path.isfile(dirpath): + dirpath = os.path.dirname(dirpath) + + for _, module_name, _ in pkgutil.iter_modules([dirpath], module_prefix+"."): + try: + importlib.import_module(module_name) + except ImportError as e: + raise CuckooOperationalError( + "Unable to load the Cuckoo plugin at %s: %s. Please " + "review its contents and/or validity!" % (module_name, e) + ) + + subclasses = class_.__subclasses__()[:] + + plugins = [] + while subclasses: + subclass = subclasses.pop(0) + + # Include subclasses of this subclass (there are some subclasses, e.g., + # LibVirtMachinery, that fail the fail the following module namespace + # check and as such we perform this logic here). + subclasses.extend(subclass.__subclasses__()) + + # Check whether this subclass belongs to the module namespace that + # we're currently importing. It should be noted that parent + # namespaces should fail the following if-statement. + if not subclass.__module__.startswith(module_prefix): + continue + + namespace[subclass.__name__] = subclass + for key, value in attributes.items(): + setattr(subclass, key, value) + + plugins.append(subclass) + + if as_dict: + ret = {} + for plugin in plugins: + plugin_module = plugin.__module__[len(module_prefix) + 1:] + ret[plugin_module] = plugin + return ret + + return sorted(plugins, key=lambda x: x.__name__.lower()) + +class RunAuxiliary(object): + """Auxiliary modules manager.""" + + def __init__(self, task, machine, guest_manager): + self.task = task + self.machine = machine + self.guest_manager = guest_manager + self.enabled = [] + + def start(self): + for module in cuckoo.auxiliary.plugins: + try: + current = module() + except: + log.exception( + "Failed to load the auxiliary module: %s", + module, extra={"task_id": self.task["id"]} + ) + return + + module_name = inspect.getmodule(current).__name__ + if "." in module_name: + module_name = module_name.rsplit(".", 1)[1] + + try: + options = config2("auxiliary", module_name) + except CuckooConfigurationError: + log.debug( + "Auxiliary module %s not found in configuration file", + module_name + ) + continue + + if not options.enabled: + continue + + current.set_task(self.task) + current.set_machine(self.machine) + current.set_guest_manager(self.guest_manager) + current.set_options(options) + + try: + current.start() + except NotImplementedError: + pass + except CuckooDisableModule: + continue + except: + log.exception( + "Unable to start auxiliary module %s", + module_name, extra={"task_id": self.task["id"]} + ) + else: + log.debug("Started auxiliary module: %s", + current.__class__.__name__) + self.enabled.append(current) + + def callback(self, name, *args, **kwargs): + def default(*args, **kwargs): + pass + + enabled = [] + for module in self.enabled: + try: + getattr(module, "cb_%s" % name, default)(*args, **kwargs) + except NotImplementedError: + pass + except CuckooDisableModule: + continue + except: + log.exception( + "Error performing callback %r on auxiliary module %r", + name, module.__class__.__name__, + extra={"task_id": self.task["id"]} + ) + + enabled.append(module) + self.enabled = enabled + + def stop(self): + stopped = [] + for module in self.enabled: + try: + module.stop() + except NotImplementedError: + pass + except: + log.exception( + "Unable to stop auxiliary module: %s", + module.__class__.__name__, + extra={"task_id": self.task["id"]} + ) + else: + log.debug("Stopped auxiliary module: %s", + module.__class__.__name__) + stopped.append(module) + + for s in stopped: + self.enabled.remove(s) + +class RunProcessing(object): + """Analysis Results Processing Engine. + + This class handles the loading and execution of the processing modules. + It executes the enabled ones sequentially and generates a dictionary which + is then passed over the reporting engine. + """ + + def __init__(self, task): + """@param task: task dictionary of the analysis to process.""" + self.task = task + self.machine = {} + self.analysis_path = cwd(analysis=task["id"]) + self.baseline_path = cwd("storage", "baseline") + + def process(self, module, results): + """Run a processing module. + @param module: processing module to run. + @param results: results dict. + @return: results generated by module. + """ + # Initialize the specified processing module. + try: + current = module() + except: + log.exception( + "Failed to load the processing module: %s", + module, extra={"task_id": self.task["id"]} + ) + return None, None + + # Extract the module name. + module_name = inspect.getmodule(current).__name__ + if "." in module_name: + module_name = module_name.rsplit(".", 1)[1] + + try: + options = config2("processing", module_name) + except CuckooConfigurationError: + log.debug( + "Processing module %s not found in configuration file", + module_name + ) + return None, None + + # If the processing module is disabled in the config, skip it. + if not options.enabled: + return None, None + + # Give it the path to the baseline directory. + current.set_baseline(self.baseline_path) + # Give it the path to the analysis results. + current.set_path(self.analysis_path) + # Give it the analysis task object. + current.set_task(self.task) + # Give it the configuration information on the machine. + current.set_machine(self.machine) + # Give it the options from the relevant processing.conf section. + current.set_options(options) + # Give it the results that we have obtained so far. + current.set_results(results) + + try: + # Run the processing module and retrieve the generated data to be + # appended to the general results container. + data = current.run() + + log.debug( + "Executed processing module \"%s\" for task #%d", + current.__class__.__name__, self.task["id"] + ) + + # If succeeded, return they module's key name and the data. + return current.key, data + except CuckooDependencyError as e: + log.warning( + "The processing module \"%s\" has missing dependencies: %s", + current.__class__.__name__, e + ) + except CuckooProcessingError as e: + log.warning( + "The processing module \"%s\" returned the following " + "error: %s", + current.__class__.__name__, e + ) + except: + log.exception( + "Failed to run the processing module \"%s\" for task #%d:", + current.__class__.__name__, self.task["id"], + extra={"task_id": self.task["id"]} + ) + + return None, None + + def populate_machine_info(self): + if not self.task.get("guest"): + return + + # TODO Actually fill out all of the fields as done for this analysis. + try: + self.machine["name"] = self.task["guest"]["name"] + self.machine.update(config2( + self.task["guest"]["manager"].lower(), + self.task["guest"]["name"] + )) + except CuckooConfigurationError: + pass + + def run(self): + """Run all processing modules and all signatures. + @return: processing results. + """ + # This is the results container. It's what will be used by all the + # reporting modules to make it consumable by humans and machines. + # It will contain all the results generated by every processing + # module available. Its structure can be observed through the JSON + # dump in the analysis' reports folder. (If jsondump is enabled.) + # We friendly call this "fat dict". + results = { + "_temp": {}, + } + + # Uses plain machine configuration as input. + self.populate_machine_info() + + # Order modules using the user-defined sequence number. + # If none is specified for the modules, they are selected in + # alphabetical order. + processing_list = cuckoo.processing.plugins + + # If no modules are loaded, return an empty dictionary. + if processing_list: + processing_list.sort(key=lambda module: module.order) + + # Run every loaded processing module. + for module in processing_list: + key, result = self.process(module, results) + + # If the module provided results, append it to the fat dict. + if key and result: + results[key] = result + else: + log.info("No processing modules loaded") + + results.pop("_temp", None) + + # Return the fat dict. + return results + +class RunSignatures(object): + """Run Signatures.""" + available_signatures = [] + version = cuckoo_version + ttp_descriptions = {} + + def __init__(self, results): + self.results = results + self.matched = [] + + # Initialize each applicable Signature. + self.signatures = [] + for signature in self.available_signatures: + if self.should_enable_signature(signature): + self.signatures.append(signature(self)) + + # Cache of signatures to call per API name. + self.api_sigs = {} + + # Prebuild a list of signatures that *may* be interested + self.call_always = set() + self.call_for_api = {} + self.call_for_cat = {} + for sig in self.signatures: + # Direct dispatch per API call + for n in dir(sig): + if n.startswith("on_call_"): + self.call_for_api.setdefault(n[8:], set()).add(sig) + if not self._on_call_defined(sig): + # Not implemented... + continue + if not sig.filter_apinames and not sig.filter_categories: + self.call_always.add(sig) + continue + for api in sig.filter_apinames: + self.call_for_api.setdefault(api, set()).add(sig) + for cat in sig.filter_categories: + self.call_for_cat.setdefault(cat, set()).add(sig) + + def _on_call_defined(self, sig): + """Test if on_call is defined. This is not pretty, but it allows + on_call to be defined in `abstracts` for documentation purposes. + """ + + # In Python 3, we can just use a simple check + if sys.version_info[0] >= 3: + return sig.on_call is not Signature.on_call + + # Check where the method was defined + return sig.on_call.__func__.__module__ != Signature.on_call.__func__.__module__ + + @classmethod + def init_once(cls): + cls.available_signatures = [] + + # Gather all enabled & up-to-date Signatures. + for signature in cuckoo.signatures: + if cls.should_load_signature(signature): + cls.available_signatures.append(signature) + + # Sort Signatures by their order. + cls.available_signatures.sort(key=lambda sig: sig.order) + + cwd_ttps = cwd("stuff", "ttp_descriptions.json") + if os.path.exists(cwd_ttps): + with open(cwd_ttps, "rb") as fp: + cls.ttp_descriptions = json.load(fp) + else: + log.warning( + "Missing TTP descriptions file. No TTP descriptions will be " + "added to matched Cuckoo signatures." + ) + + @classmethod + def should_load_signature(cls, signature): + """Should the given signature be enabled for this analysis?""" + if not signature.enabled or signature.name is None: + return False + + if not cls.check_signature_version(signature): + return False + + if hasattr(signature, "enable") and callable(signature.enable): + if not signature.enable(): + return False + + return True + + def should_enable_signature(self, signature): + # Network and/or cross-platform signatures. + if not signature.platform: + return True + + task_platform = self.results.get("info", {}).get("platform") + + # Windows is implied when a platform has not been specified during the + # submission of a sample, but for other platforms the platform has to + # be explicitly stated. + if not task_platform and signature.platform == "windows": + return True + + return task_platform == signature.platform + + @classmethod + def check_signature_version(cls, sig): + """Check signature version. + @param current: signature class/instance to check. + @return: check result. + """ + if not supported_version(cls.version, sig.minimum, sig.maximum): + log.debug( + "You are running a version of Cuckoo that's not compatible " + "with this Signature (either it's too old or too new): " + "cuckoo=%s signature=%s minversion=%s maxversion=%s", + cls.version, sig.name, sig.minimum, sig.maximum + ) + return False + + if hasattr(sig, "run"): + log.warning( + "This signatures features one or more deprecated functions " + "which indicates that it is very likely an old-style " + "signature. Please upgrade this signature: %s.", sig.name + ) + return False + + return True + + def call_signature(self, signature, handler, *args, **kwargs): + """Wrapper to call into 3rd party signatures. This wrapper yields the + event to the signature and handles matched signatures recursively.""" + try: + if not signature.matched and handler(*args, **kwargs): + signature.matched = True + for sig in self.signatures: + self.call_signature(sig, sig.on_signature, signature) + except: + task_id = self.results.get("info", {}).get("id") + log.exception( + "Failed to run '%s' of the %s signature", + handler.__name__, signature.name, + extra={"task_id": task_id} + ) + return True + + def yield_calls(self, proc): + """Yield calls of interest to each interested signature.""" + for idx, call in enumerate(proc.get("calls", [])): + api = call.get("api") + sigs = self.api_sigs.get(api) + if sigs is None: + # Build interested signatures + cat = call.get("category") + sigs = self.call_always.union( + self.call_for_api.get(api, set()), + self.call_for_cat.get(cat, set()) + ) + self.api_sigs[api] = sigs + name = "on_call_" + api + for sig in sigs: + sig.cid, sig.call = idx, call + func = getattr(sig, name, sig.on_call) + self.call_signature(sig, func, call, proc) + + def process_yara_matches(self): + """Yield any Yara matches to each signature.""" + def loop_yara(category, filepath, matches): + for match in matches: + match = YaraMatch(match, category) + for sig in self.signatures: + self.call_signature( + sig, sig.on_yara, category, filepath, match + ) + + target = self.results.get("target", {}) + if target.get("category") == "file" and target.get("file"): + loop_yara( + "sample", + self.results["target"]["file"]["path"], + self.results["target"]["file"]["yara"] + ) + + for procmem in self.results.get("procmemory", []): + # Yara matches on extracted PE files from process memory dumps. + for extr in procmem.get("extracted", []): + loop_yara("extracted", extr["path"], extr["yara"]) + + # Yara rules on the process memory dump itself. + loop_yara("procmem", procmem["file"], procmem["yara"]) + + for dropped in self.results.get("dropped", []): + loop_yara("dropped", dropped["path"], dropped["yara"]) + + for extr in self.results.get("extracted", []): + loop_yara("extracted", extr["raw"], extr["yara"]) + + def process_extracted(self): + task_id = self.results.get("info", {}).get("id") + if not task_id: + return + + for item in ExtractManager.for_task(task_id).results(): + for sig in self.signatures: + self.call_signature(sig, sig.on_extract, ExtractedMatch(item)) + + def run(self): + """Run signatures.""" + # Allow signatures to initialize themselves. + for signature in self.signatures: + signature.init() + + log.debug("Running %d signatures", len(self.signatures)) + + # Iterate calls and tell interested signatures about them. + for proc in self.results.get("behavior", {}).get("processes", []): + + # Yield the new process event. + for sig in self.signatures: + sig.pid = proc["pid"] + self.call_signature(sig, sig.on_process, proc) + + self.yield_calls(proc) + + # Iterate through all Yara matches. + self.process_yara_matches() + + # Iterate through all Extracted matches. + self.process_extracted() + + # TODO This logic should certainly be moved elsewhere. + self.c = Configuration() + for extracted in self.results.get("extracted", []): + if extracted["category"] == "config": + self.c.add(extracted["info"]) + + # Yield completion events to each signature. + for sig in self.signatures: + self.call_signature(sig, sig.on_complete) + + score, configs = 0, [] + for signature in self.signatures: + if not signature.matched: + continue + + log.debug( + "Analysis matched signature: %s", signature.name, extra={ + "action": "signature.match", "status": "success", + "signature": signature.name, + "severity": signature.severity, + } + ) + self.matched.append(signature.results()) + score += signature.severity + + for mark in signature.marks: + if mark["type"] == "config": + self.c.add(mark["config"]) + + # Sort the matched signatures by their severity level and put them + # into the results dictionary. + self.matched.sort(key=lambda key: key["severity"]) + self.results["signatures"] = self.matched + if "info" in self.results: + self.results["info"]["score"] = score / 5.0 + + # If malware configuration has been extracted, simplify its + # accessibility in the analysis report. + if self.c.results(): + # TODO Should this be included elsewhere? + if "metadata" in self.results: + self.results["metadata"]["cfgextr"] = self.c.results() + if "info" in self.results: + self.results["info"]["score"] = 10 + +class RunReporting(object): + """Reporting Engine. + + This class handles the loading and execution of the enabled reporting + modules. It receives the analysis results dictionary from the Processing + Engine and pass it over to the reporting modules before executing them. + """ + + def __init__(self, task, results): + """@param analysis_path: analysis folder path.""" + self.task = task + self.results = results + self.analysis_path = cwd("storage", "analyses", "%s" % task["id"]) + + def process(self, module): + """Run a single reporting module. + @param module: reporting module. + @param results: results results from analysis. + """ + # Initialize current reporting module. + try: + current = module() + except: + log.exception( + "Failed to load the reporting module: %s", module, + extra={"task_id": self.task["id"]} + ) + return + + # Extract the module name. + module_name = inspect.getmodule(current).__name__ + if "." in module_name: + module_name = module_name.rsplit(".", 1)[1] + + try: + options = config2("reporting", module_name) + except CuckooConfigurationError: + log.debug( + "Reporting module %s not found in configuration file", + module_name + ) + return + + # If the reporting module is disabled in the config, skip it. + if not options.enabled: + return + + # Give it the path to the analysis results folder. + current.set_path(self.analysis_path) + # Give it the analysis task object. + current.set_task(self.task) + # Give it the the relevant reporting.conf section. + current.set_options(options) + + try: + current.run(self.results) + log.debug("Executed reporting module \"%s\"", current.__class__.__name__) + except CuckooDependencyError as e: + log.warning( + "The reporting module \"%s\" has missing dependencies: %s", + current.__class__.__name__, e + ) + except CuckooReportError as e: + log.warning( + "The reporting module \"%s\" returned the following " + "error: %s", current.__class__.__name__, e + ) + except: + log.exception( + "Failed to run the reporting module: %s", + current.__class__.__name__, + extra={"task_id": self.task["id"]} + ) + + def run(self): + """Generate all reports. + @raise CuckooReportError: if a report module fails. + """ + # In every reporting module you can specify a numeric value that + # represents at which position that module should be executed among + # all the available ones. It can be used in the case where a + # module requires another one to be already executed beforehand. + reporting_list = cuckoo.reporting.plugins + + # Return if no reporting modules are loaded. + if reporting_list: + reporting_list.sort(key=lambda module: module.order) + + # Run every loaded reporting module. + for module in reporting_list: + self.process(module) + else: + log.info("No reporting modules loaded") diff --git a/cuckoo/core/report.py b/cuckoo/core/report.py new file mode 100644 index 0000000000..4b29b0512e --- /dev/null +++ b/cuckoo/core/report.py @@ -0,0 +1,35 @@ +# Copyright (C) 2016-2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +class Report(object): + def __init__(self, report): + self.report = report + + def get(self, *keys): + r = self.report + for key in keys: + if key not in r: + return + r = r[key] + return r + + @property + def info(self): + return self.get("info") or {} + + @property + def path(self): + return self.get("info", "analysis_path") + + @property + def feedback(self): + return self.get("feedback") or {} + + @property + def target(self): + return self.get("target") + + @property + def errors(self): + return self.get("debug", "errors") or [] diff --git a/cuckoo/core/resultserver.py b/cuckoo/core/resultserver.py new file mode 100644 index 0000000000..e307124394 --- /dev/null +++ b/cuckoo/core/resultserver.py @@ -0,0 +1,425 @@ +# Copyright (C) 2012-2013 Claudio Guarnieri. +# Copyright (C) 2014-2019 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +from __future__ import print_function + +import errno +import gevent.pool +import gevent.server +import gevent.socket +import json +import logging +import os +import socket +import threading + +from cuckoo.common.abstracts import ProtocolHandler +from cuckoo.common.config import config +from cuckoo.common.exceptions import CuckooCriticalError +from cuckoo.common.exceptions import CuckooOperationalError +from cuckoo.common.files import open_exclusive +from cuckoo.common.utils import Singleton +from cuckoo.core.log import task_log_start, task_log_stop +from cuckoo.misc import cwd + +log = logging.getLogger(__name__) + +# Maximum line length to read for netlog messages, to avoid memory exhaustion +MAX_NETLOG_LINE = 4 * 1024 + +# Maximum number of bytes to buffer for a single connection +BUFSIZE = 16 * 1024 + +# Directories in which analysis-related files will be stored; also acts as +# safelist +RESULT_UPLOADABLE = ("files", "shots", "buffer", "extracted", "memory") +RESULT_DIRECTORIES = RESULT_UPLOADABLE + ("reports", "logs") + +# Prevent malicious clients from using potentially dangerous filenames +# E.g. C API confusion by using null, or using the colon on NTFS (Alternate +# Data Streams); XXX: just replace illegal chars? +BANNED_PATH_CHARS = b'\x00:' + +def netlog_sanitize_fname(path): + """Validate agent-provided path for result files""" + path = path.replace("\\", "/") + dir_part, name = os.path.split(path) + if dir_part not in RESULT_UPLOADABLE: + raise CuckooOperationalError("Netlog client requested banned path: %r" + % path) + if any(c in BANNED_PATH_CHARS for c in name): + for c in BANNED_PATH_CHARS: + path = path.replace(c, "X") + + return path + +class HandlerContext(object): + """Holds context for protocol handlers. + + Can safely be cancelled from another thread, though in practice this will + not occur often -- usually the connection between VM and the ResultServer + will be reset during shutdown.""" + def __init__(self, task_id, storagepath, sock): + self.task_id = task_id + self.command = None + + # The path where artifacts will be stored + self.storagepath = storagepath + self.sock = sock + self.buf = "" + + def __repr__(self): + return "" % self.command + + def cancel(self): + """Cancel this context; gevent might complain about this with an + exception later on.""" + try: + self.sock.shutdown(socket.SHUT_RD) + except socket.error: + pass + + def read(self): + try: + return self.sock.recv(16384) + except socket.error as e: + if e.errno == errno.EBADF: + return "" + + if e.errno != errno.ECONNRESET: + raise + log.debug("Task #%s had connection reset for %r", self.task_id, + self) + return "" + + def drain_buffer(self): + """Drain buffer and end buffering""" + buf, self.buf = self.buf, None + return buf + + def read_newline(self): + """Read until the next newline character, but never more than + `MAX_NETLOG_LINE`.""" + while True: + pos = self.buf.find("\n") + if pos < 0: + if len(self.buf) >= MAX_NETLOG_LINE: + raise CuckooOperationalError("Received overly long line") + buf = self.read() + if buf == "": + raise EOFError + self.buf += buf + continue + line, self.buf = self.buf[:pos], self.buf[pos + 1:] + return line + + def copy_to_fd(self, fd, max_size=None): + if max_size: + fd = WriteLimiter(fd, max_size) + fd.write(self.drain_buffer()) + while True: + buf = self.read() + if buf == "": + break + fd.write(buf) + fd.flush() + +class WriteLimiter(object): + def __init__(self, fd, remain): + self.fd = fd + self.remain = remain + self.warned = False + + def write(self, buf): + size = len(buf) + write = min(size, self.remain) + if write: + self.fd.write(buf[:write]) + self.remain -= write + if size and size != write: + if not self.warned: + log.warning("Uploaded file length larger than upload_max_size, " + "stopping upload.") + self.fd.write("... (truncated)") + self.warned = True + + def flush(self): + self.fd.flush() + +class FileUpload(ProtocolHandler): + def init(self): + self.upload_max_size = config("cuckoo:resultserver:upload_max_size") + self.storagepath = self.handler.storagepath + self.fd = None + self.filelog = os.path.join(self.handler.storagepath, "files.json") + + def handle(self): + # Read until newline for file path, e.g., + # shots/0001.jpg or files/9498687557/libcurl-4.dll.bin + self.handler.sock.settimeout(30) + dump_path = netlog_sanitize_fname(self.handler.read_newline()) + + if self.version and self.version >= 2: + # NB: filepath is only used as metadata + filepath = self.handler.read_newline() + pids = map(int, self.handler.read_newline().split()) + else: + filepath, pids = None, [] + + log.debug("Task #%s: File upload for %r", self.task_id, dump_path) + file_path = os.path.join(self.storagepath, dump_path.decode("utf-8")) + + try: + self.fd = open_exclusive(file_path) + except OSError as e: + if e.errno == errno.EEXIST: + raise CuckooOperationalError("Analyzer for task #%s tried to " + "overwrite an existing file" % + self.task_id) + raise + + # Append-writes are atomic + with open(self.filelog, "a+b") as f: + print(json.dumps({ + "path": dump_path, + "filepath": filepath, + "pids": pids, + }), file=f) + + self.handler.sock.settimeout(None) + try: + return self.handler.copy_to_fd(self.fd, self.upload_max_size) + finally: + log.debug("Task #%s uploaded file length: %s", self.task_id, + self.fd.tell()) + +class LogHandler(ProtocolHandler): + """The live analysis log. Can only be opened once in a single session.""" + + def init(self): + self.logpath = os.path.join(self.handler.storagepath, "analysis.log") + try: + self.fd = open_exclusive(self.logpath, bufsize=1) + except OSError: + log.error("Task #%s: attempted to reopen live log analysis.log.", + self.task_id) + return + + log.debug("Task #%s: live log analysis.log initialized.", + self.task_id) + + def handle(self): + if self.fd: + return self.handler.copy_to_fd(self.fd) + +class BsonStore(ProtocolHandler): + def init(self): + # We cheat a little bit through the "version" variable, but that's + # acceptable and backwards compatible (for now). Backwards compatible + # in the sense that newer Cuckoo Monitor binaries work with older + # versions of Cuckoo, the other way around doesn't apply here. + if self.version is None: + log.warning("Agent is sending BSON files without PID parameter, " + "you should probably update it") + self.fd = None + return + + self.fd = open(os.path.join(self.handler.storagepath, + "logs", "%d.bson" % self.version), "wb") + + def handle(self): + """Read a BSON stream, attempting at least basic validation, and + log failures.""" + log.debug("Task #%s is sending a BSON stream", self.task_id) + if self.fd: + return self.handler.copy_to_fd(self.fd) + +class GeventResultServerWorker(gevent.server.StreamServer): + """The new ResultServer, providing a huge performance boost as well as + implementing a new dropped file storage format avoiding small fd limits. + + The old ResultServer would start a new thread per socket, greatly impacting + the overall performance of Cuckoo Sandbox. The new ResultServer uses + so-called Greenlets, low overhead green-threads by Gevent, imposing much + less kernel overhead. + + Furthermore, instead of writing each dropped file to its own location (in + $CWD/storage/analyses//files/_filename.ext) it's + capable of storing all dropped files in a streamable container format. This + is one of various steps to start being able to use less fd's in Cuckoo. + """ + commands = { + "BSON": BsonStore, + "FILE": FileUpload, + "LOG": LogHandler, + } + task_mgmt_lock = threading.Lock() + + def __init__(self, *args, **kwargs): + super(GeventResultServerWorker, self).__init__(*args, **kwargs) + + # Store IP address to task_id mapping + self.tasks = {} + + # Store running handlers for task_id + self.handlers = {} + + def do_run(self): + self.serve_forever() + + def add_task(self, task_id, ipaddr): + with self.task_mgmt_lock: + self.tasks[ipaddr] = task_id + log.debug("Now tracking machine %s for task #%s", ipaddr, task_id) + + def del_task(self, task_id, ipaddr): + """Delete ResultServer state and abort pending RequestHandlers. Since + we're about to shutdown the VM, any remaining open connections can + be considered a bug from the VM side, since all connections should + have been closed after the analyzer signalled completion.""" + with self.task_mgmt_lock: + if self.tasks.pop(ipaddr, None) is None: + log.warning( + "ResultServer did not have a task with ID %s and IP %s", + task_id, ipaddr + ) + else: + log.debug( + "Stopped tracking machine %s for task #%s", + ipaddr, task_id + ) + ctxs = self.handlers.pop(task_id, set()) + for ctx in ctxs: + log.debug("Cancel %s for task %r", ctx, task_id) + ctx.cancel() + + def handle(self, sock, addr): + """Handle the incoming connection. + Gevent will close the socket when the function returns.""" + ipaddr = addr[0] + + with self.task_mgmt_lock: + task_id = self.tasks.get(ipaddr) + if not task_id: + log.warning("ResultServer did not have a task for IP %s", + ipaddr) + return + + storagepath = cwd(analysis=task_id) + ctx = HandlerContext(task_id, storagepath, sock) + task_log_start(task_id) + try: + try: + protocol = self.negotiate_protocol(task_id, ctx) + except EOFError: + return + + # Registering the context allows us to abort the handler by + # shutting down its socket when the task is deleted; this should + # prevent lingering sockets + with self.task_mgmt_lock: + # NOTE: the task may have been cancelled during the negotation + # protocol and a different task for that IP address may have + # been registered + if self.tasks.get(ipaddr) != task_id: + log.warning("Task #%s for IP %s was cancelled during " + "negotiation", task_id, ipaddr) + return + s = self.handlers.setdefault(task_id, set()) + s.add(ctx) + + try: + with protocol: + protocol.handle() + except CuckooOperationalError as e: + log.error(e) + finally: + with self.task_mgmt_lock: + s.discard(ctx) + ctx.cancel() + if ctx.buf: + # This is usually not a good sign + log.warning("Task #%s with protocol %s has unprocessed " + "data before getting disconnected", + task_id, protocol) + + finally: + task_log_stop(task_id) + + def negotiate_protocol(self, task_id, ctx): + header = ctx.read_newline() + if " " in header: + command, version = header.split() + version = int(version) + else: + command, version = header, None + klass = self.commands.get(command) + if not klass: + log.warning("Task #%s: unknown netlog protocol requested (%r), " + "terminating connection.", task_id, command) + return + ctx.command = command + return klass(task_id, ctx, version) + +class ResultServer(object): + """Manager for the ResultServer worker and task state.""" + __metaclass__ = Singleton + + def __init__(self): + ip = config("cuckoo:resultserver:ip") + port = config("cuckoo:resultserver:port") + pool_size = config('cuckoo:resultserver:pool_size') + + sock = gevent.socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + + try: + sock.bind((ip, port)) + except (OSError, socket.error) as e: + if e.errno == errno.EADDRINUSE: + raise CuckooCriticalError( + "Cannot bind ResultServer on port %d " + "because it was in use, bailing." % port + ) + elif e.errno == errno.EADDRNOTAVAIL: + raise CuckooCriticalError( + "Unable to bind ResultServer on %s:%s %s. This " + "usually happens when you start Cuckoo without " + "bringing up the virtual interface associated with " + "the ResultServer IP address. Please refer to " + "https://cuckoo.sh/docs/faq/#troubles-problem " + "for more information." % (ip, port, e) + ) + else: + raise CuckooCriticalError( + "Unable to bind ResultServer on %s:%s: %s" % + (ip, port, e) + ) + + # We allow user to specify port 0 to get a random port, report it back + # here + _, self.port = sock.getsockname() + sock.listen(128) + + self.thread = threading.Thread(target=self.create_server, + args=(sock, pool_size)) + self.thread.daemon = True + self.thread.start() + + def add_task(self, task, machine): + """Register a task/machine with the ResultServer.""" + self.instance.add_task(task.id, machine.ip) + + def del_task(self, task, machine): + """Delete running task and cancel existing handlers.""" + self.instance.del_task(task.id, machine.ip) + + def create_server(self, sock, pool_size): + if pool_size: + pool = gevent.pool.Pool(pool_size) + else: + pool = 'default' + self.instance = GeventResultServerWorker(sock, spawn=pool) + self.instance.do_run() diff --git a/cuckoo/core/rooter.py b/cuckoo/core/rooter.py new file mode 100644 index 0000000000..ad143a68ac --- /dev/null +++ b/cuckoo/core/rooter.py @@ -0,0 +1,58 @@ +# Copyright (C) 2015-2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import json +import logging +import os.path +import socket +import tempfile +import threading + +from cuckoo.common.config import config + +log = logging.getLogger(__name__) +unixpath = tempfile.mktemp() +lock = threading.Lock() + +def rooter(command, *args, **kwargs): + if not os.path.exists(config("cuckoo:cuckoo:rooter")): + log.critical( + "Unable to passthrough root command (%s) as the rooter " + "unix socket doesn't exist.", command + ) + return + + lock.acquire() + + s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + + if os.path.exists(unixpath): + os.remove(unixpath) + + s.bind(unixpath) + + try: + s.connect(config("cuckoo:cuckoo:rooter")) + except socket.error as e: + log.critical( + "Unable to passthrough root command as we're unable to " + "connect to the rooter unix socket: %s.", e + ) + lock.release() + return + + s.send(json.dumps({ + "command": command, + "args": args, + "kwargs": kwargs, + })) + + ret = json.loads(s.recv(0x10000)) + + lock.release() + + if ret["exception"]: + log.warning("Rooter returned error: %s", ret["exception"]) + + return ret["output"] diff --git a/cuckoo/core/scheduler.py b/cuckoo/core/scheduler.py new file mode 100644 index 0000000000..edd6214e41 --- /dev/null +++ b/cuckoo/core/scheduler.py @@ -0,0 +1,1104 @@ +# Copyright (C) 2012-2013 Claudio Guarnieri. +# Copyright (C) 2014-2019 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import os +import time +import shutil +import logging +import threading +import Queue + +import cuckoo + +from cuckoo.common.config import Config, emit_options, config +from cuckoo.common.constants import faq +from cuckoo.common.exceptions import ( + CuckooMachineError, CuckooGuestError, CuckooOperationalError, + CuckooMachineSnapshotError, CuckooCriticalError, CuckooGuestCriticalTimeout +) +from cuckoo.common.objects import File +from cuckoo.common.files import Folders +from cuckoo.core.database import Database, TASK_COMPLETED, TASK_REPORTED +from cuckoo.core.guest import GuestManager +from cuckoo.core.plugins import RunAuxiliary, RunProcessing +from cuckoo.core.plugins import RunSignatures, RunReporting +from cuckoo.core.log import task_log_start, task_log_stop, logger +from cuckoo.core.resultserver import ResultServer, RESULT_DIRECTORIES +from cuckoo.core.rooter import rooter +from cuckoo.misc import cwd + +log = logging.getLogger(__name__) + +machinery = None +machine_lock = None +latest_symlink_lock = threading.Lock() + +active_analysis_count = 0 + + +class AnalysisManager(threading.Thread): + """Analysis Manager. + + This class handles the full analysis process for a given task. It takes + care of selecting the analysis machine, preparing the configuration and + interacting with the guest agent and analyzer components to launch and + complete the analysis and store, process and report its results. + """ + + def __init__(self, task_id, error_queue): + """@param task: task object containing the details for the analysis.""" + threading.Thread.__init__(self) + + self.errors = error_queue + self.cfg = Config() + self.storage = "" + self.binary = "" + self.storage_binary = "" + self.machine = None + self.db = Database() + self.task = self.db.view_task(task_id) + self.guest_manager = None + self.route = None + self.interface = None + self.rt_table = None + self.unrouted_network = False + self.stopped_aux = False + self.rs_port = config("cuckoo:resultserver:port") + + def init(self): + """Initialize the analysis.""" + self.storage = cwd(analysis=self.task.id) + + # If the analysis storage folder already exists, we need to abort the + # analysis or previous results will be overwritten and lost. + if os.path.exists(self.storage): + log.error("Analysis results folder already exists at path \"%s\", " + "analysis aborted", self.storage) + return False + + # If we're not able to create the analysis storage folder, we have to + # abort the analysis. + # Also create all directories that the ResultServer can use for file + # uploads. + try: + Folders.create(self.storage, RESULT_DIRECTORIES) + except CuckooOperationalError: + log.error("Unable to create analysis folder %s", self.storage) + return False + + self.store_task_info() + + if self.task.category == "file" or self.task.category == "archive": + # Check if we have permissions to access the file. + # And fail this analysis if we don't have access to the file. + if not os.access(self.task.target, os.R_OK): + log.error( + "Unable to access target file, please check if we have " + "permissions to access the file: \"%s\"", + self.task.target + ) + return False + + # Check whether the file has been changed for some unknown reason. + # And fail this analysis if it has been modified. + # TODO Absorb the file upon submission. + sample = self.db.view_sample(self.task.sample_id) + sha256 = File(self.task.target).get_sha256() + if sha256 != sample.sha256: + log.error( + "Target file has been modified after submission: \"%s\"", + self.task.target + ) + return False + + # Store a copy of the original file if does not exist already. + # TODO This should be done at submission time. + self.binary = cwd("storage", "binaries", sha256) + if not os.path.exists(self.binary): + try: + shutil.copy(self.task.target, self.binary) + except (IOError, shutil.Error): + log.error( + "Unable to store file from \"%s\" to \"%s\", " + "analysis aborted", self.task.target, self.binary + ) + return False + + # Each analysis directory contains a symlink/copy of the binary. + try: + self.storage_binary = os.path.join(self.storage, "binary") + + if hasattr(os, "symlink"): + os.symlink(self.binary, self.storage_binary) + else: + shutil.copy(self.binary, self.storage_binary) + except (AttributeError, OSError) as e: + log.error("Unable to create symlink/copy from \"%s\" to " + "\"%s\": %s", self.binary, self.storage, e) + return False + + # Initiates per-task logging. + task_log_start(self.task.id) + return True + + def store_task_info(self): + """Grab latest task from db (if available) and update self.task""" + dbtask = self.db.view_task(self.task.id) + self.task = dbtask.to_dict() + + task_info_path = os.path.join(self.storage, "task.json") + open(task_info_path, "w").write(dbtask.to_json()) + + def acquire_machine(self): + """Acquire an analysis machine from the pool of available ones.""" + machine = None + + # Start a loop to acquire the a machine to run the analysis on. + while True: + machine_lock.acquire() + + # In some cases it's possible that we enter this loop without + # having any available machines. We should make sure this is not + # such case, or the analysis task will fail completely. + if not machinery.availables(): + machine_lock.release() + time.sleep(1) + continue + + # If the user specified a specific machine ID, a platform to be + # used or machine tags acquire the machine accordingly. + machine = machinery.acquire(machine_id=self.task.machine, + platform=self.task.platform, + tags=self.task.tags) + + # If no machine is available at this moment, wait for one second + # and try again. + if not machine: + machine_lock.release() + log.debug("Task #%d: no machine available yet", self.task.id) + time.sleep(1) + else: + log.info( + "Task #%d: acquired machine %s (label=%s)", + self.task.id, machine.name, machine.label, extra={ + "action": "vm.acquire", + "status": "success", + "vmname": machine.name, + } + ) + break + + self.machine = machine + + def build_options(self): + """Generate analysis options. + @return: options dict. + """ + options = {} + + if self.task.category == "file": + options["file_name"] = File(self.task.target).get_name() + options["file_type"] = File(self.task.target).get_type() + options["pe_exports"] = \ + ",".join(File(self.task.target).get_exported_functions()) + + package, activity = File(self.task.target).get_apk_entry() + self.task.options["apk_entry"] = "%s:%s" % (package, activity) + elif self.task.category == "archive": + options["file_name"] = File(self.task.target).get_name() + + options["id"] = self.task.id + options["ip"] = self.machine.resultserver_ip + options["port"] = self.rs_port + options["category"] = self.task.category + options["target"] = self.task.target + options["package"] = self.task.package + options["options"] = emit_options(self.task.options) + options["enforce_timeout"] = self.task.enforce_timeout + options["clock"] = self.task.clock + options["terminate_processes"] = self.cfg.cuckoo.terminate_processes + + if not self.task.timeout: + options["timeout"] = self.cfg.timeouts.default + else: + options["timeout"] = self.task.timeout + + # copy in other analyzer specific options, TEMPORARY (most likely) + vm_options = getattr(machinery.options, self.machine.name) + for k in vm_options: + if k.startswith("analyzer_"): + options[k] = vm_options[k] + + return options + + def route_network(self): + """Enable network routing if desired.""" + # Determine the desired routing strategy (none, internet, VPN). + self.route = self.task.options.get( + "route", config("routing:routing:route") + ) + + if self.route == "none" or self.route == "drop": + self.interface = None + self.rt_table = None + elif self.route == "inetsim": + pass + elif self.route == "tor": + pass + elif self.route == "internet": + if config("routing:routing:internet") == "none": + log.warning( + "Internet network routing has been specified, but not " + "configured, ignoring routing for this analysis", extra={ + "action": "network.route", + "status": "error", + "route": self.route, + } + ) + self.route = "none" + self.task.options["route"] = "none" + self.interface = None + self.rt_table = None + else: + self.interface = config("routing:routing:internet") + self.rt_table = config("routing:routing:rt_table") + elif self.route in config("routing:vpn:vpns"): + self.interface = config("routing:%s:interface" % self.route) + self.rt_table = config("routing:%s:rt_table" % self.route) + else: + log.warning( + "Unknown network routing destination specified, ignoring " + "routing for this analysis: %r", self.route, extra={ + "action": "network.route", + "status": "error", + "route": self.route, + } + ) + self.route = "none" + self.task.options["route"] = "none" + self.interface = None + self.rt_table = None + + # Check if the network interface is still available. If a VPN dies for + # some reason, its tunX interface will no longer be available. + if self.interface and not rooter("nic_available", self.interface): + log.error( + "The network interface '%s' configured for this analysis is " + "not available at the moment, switching to route=none mode.", + self.interface, extra={ + "action": "network.route", + "status": "error", + "route": self.route, + } + ) + self.route = "none" + self.task.options["route"] = "none" + self.interface = None + self.rt_table = None + + # For now this doesn't work yet in combination with tor routing. + if self.route == "drop" or self.route == "internet": + rooter( + "drop_enable", self.machine.ip, + config("cuckoo:resultserver:ip"), + str(self.rs_port) + ) + + if self.route == "inetsim": + machinery = config("cuckoo:cuckoo:machinery") + rooter( + "inetsim_enable", self.machine.ip, + config("routing:inetsim:server"), + config("%s:%s:interface" % (machinery, machinery)), + str(self.rs_port), + config("routing:inetsim:ports") or "" + ) + + if self.route == "tor": + rooter( + "tor_enable", self.machine.ip, + str(config("cuckoo:resultserver:ip")), + str(config("routing:tor:dnsport")), + str(config("routing:tor:proxyport")) + ) + + if self.interface: + rooter( + "forward_enable", self.machine.interface, + self.interface, self.machine.ip + ) + + if self.rt_table: + rooter( + "srcroute_enable", self.rt_table, self.machine.ip + ) + + # Propagate the taken route to the database. + self.db.set_route(self.task.id, self.route) + + def unroute_network(self): + """Disable any enabled network routing.""" + if self.interface: + rooter( + "forward_disable", self.machine.interface, + self.interface, self.machine.ip + ) + + if self.rt_table: + rooter( + "srcroute_disable", self.rt_table, self.machine.ip + ) + + if self.route == "drop" or self.route == "internet": + rooter( + "drop_disable", self.machine.ip, + config("cuckoo:resultserver:ip"), + str(self.rs_port) + ) + + if self.route == "inetsim": + machinery = config("cuckoo:cuckoo:machinery") + rooter( + "inetsim_disable", self.machine.ip, + config("routing:inetsim:server"), + config("%s:%s:interface" % (machinery, machinery)), + str(self.rs_port), + config("routing:inetsim:ports") or "" + ) + + if self.route == "tor": + rooter( + "tor_disable", self.machine.ip, + str(config("cuckoo:resultserver:ip")), + str(config("routing:tor:dnsport")), + str(config("routing:tor:proxyport")) + ) + + self.unrouted_network = True + + def wait_finish(self): + """Some VMs don't have an actual agent. Mainly those that are used as + assistance for an analysis through the services auxiliary module. This + method just waits until the analysis is finished rather than actively + trying to engage with the Cuckoo Agent.""" + self.db.guest_set_status(self.task.id, "running") + while self.db.guest_get_status(self.task.id) == "running": + time.sleep(1) + + def guest_manage(self, options): + # Handle a special case where we're creating a baseline report of this + # particular virtual machine - a report containing all the results + # that are gathered if no additional samples are ran in the VM. These + # results, such as loaded drivers and opened sockets in volatility, or + # DNS requests to hostnames related to Microsoft Windows, etc may be + # omitted or at the very least given less priority when creating a + # report for an analysis that ran on this VM later on. + if self.task.category == "baseline": + time.sleep(options["timeout"]) + else: + # Start the analysis. + self.db.guest_set_status(self.task.id, "starting") + monitor = self.task.options.get("monitor", "latest") + self.guest_manager.start_analysis(options, monitor) + + # In case the Agent didn't respond and we force-quit the analysis + # at some point while it was still starting the analysis the state + # will be "stop" (or anything but "running", really). + if self.db.guest_get_status(self.task.id) == "starting": + self.db.guest_set_status(self.task.id, "running") + self.guest_manager.wait_for_completion() + + self.db.guest_set_status(self.task.id, "stopping") + + def launch_analysis(self): + """Start analysis.""" + succeeded = False + + if self.task.category == "file" or self.task.category == "archive": + target = os.path.basename(self.task.target) + else: + target = self.task.target + + log.info( + "Starting analysis of %s \"%s\" (task #%d, options \"%s\")", + self.task.category.upper(), target, self.task.id, + emit_options(self.task.options), extra={ + "action": "task.init", + "status": "starting", + "task_id": self.task.id, + "target": target, + "category": self.task.category, + "package": self.task.package, + "options": emit_options(self.task.options), + "custom": self.task.custom, + } + ) + + # Initialize the analysis. + if not self.init(): + logger("Failed to initialize", action="task.init", status="error") + return False + + # Acquire analysis machine. + try: + self.acquire_machine() + except CuckooOperationalError as e: + machine_lock.release() + log.error("Cannot acquire machine: %s", e, extra={ + "action": "vm.acquire", "status": "error", + }) + return False + + self.rs_port = self.machine.resultserver_port or ResultServer().port + + # At this point we can tell the ResultServer about it. + try: + ResultServer().add_task(self.task, self.machine) + except Exception as e: + machinery.release(self.machine.label) + self.errors.put(e) + + # Initialize the guest manager. + self.guest_manager = GuestManager( + self.machine.name, self.machine.ip, + self.machine.platform, self.task.id, self + ) + + self.aux = RunAuxiliary(self.task, self.machine, self.guest_manager) + self.aux.start() + + # Generate the analysis configuration file. + options = self.build_options() + + # Check if the current task has remotecontrol + # enabled before starting the machine. + control_enabled = ( + config("cuckoo:remotecontrol:enabled") and + "remotecontrol" in self.task.options + ) + if control_enabled: + try: + machinery.enable_remote_control(self.machine.label) + except NotImplementedError: + log.error( + "Remote control support has not been implemented for the " + "configured machinery module: %s", + config("cuckoo:cuckoo:machinery") + ) + + try: + unlocked = False + self.interface = None + + # Mark the selected analysis machine in the database as started. + guest_log = self.db.guest_start(self.task.id, + self.machine.name, + self.machine.label, + machinery.__class__.__name__) + logger( + "Starting VM", + action="vm.start", status="pending", + vmname=self.machine.name + ) + + # Start the machine. + machinery.start(self.machine.label, self.task) + + logger( + "Started VM", + action="vm.start", status="success", + vmname=self.machine.name + ) + + # retrieve the port used for remote control + if control_enabled: + try: + params = machinery.get_remote_control_params( + self.machine.label + ) + self.db.set_machine_rcparams(self.machine.label, params) + except NotImplementedError: + log.error( + "Remote control support has not been implemented for the " + "configured machinery module: %s", + config("cuckoo:cuckoo:machinery") + ) + + # Enable network routing. + self.route_network() + + # By the time start returns it will have fully started the Virtual + # Machine. We can now safely release the machine lock. + machine_lock.release() + unlocked = True + + # Run and manage the components inside the guest unless this + # machine has the "noagent" option specified (please refer to the + # wait_finish() function for more details on this function). + if "noagent" not in self.machine.options: + self.guest_manage(options) + else: + self.wait_finish() + + succeeded = True + except CuckooMachineSnapshotError as e: + log.error( + "Unable to restore to the snapshot for this Virtual Machine! " + "Does your VM have a proper Snapshot and can you revert to it " + "manually? VM: %s, error: %s", + self.machine.name, e, extra={ + "action": "vm.resume", + "status": "error", + "vmname": self.machine.name, + } + ) + except CuckooMachineError as e: + if not unlocked: + machine_lock.release() + log.error( + "Error starting Virtual Machine! VM: %s, error: %s", + self.machine.name, e, extra={ + "action": "vm.start", + "status": "error", + "vmname": self.machine.name, + } + ) + except CuckooGuestCriticalTimeout as e: + if not unlocked: + machine_lock.release() + log.error( + "Error from machine '%s': it appears that this Virtual " + "Machine hasn't been configured properly as the Cuckoo Host " + "wasn't able to connect to the Guest. There could be a few " + "reasons for this, please refer to our documentation on the " + "matter: %s", + self.machine.name, + faq("troubleshooting-vm-network-configuration"), + extra={ + "error_action": "vmrouting", + "action": "guest.handle", + "status": "error", + "task_id": self.task.id, + } + ) + except CuckooGuestError as e: + if not unlocked: + machine_lock.release() + log.error("Error from the Cuckoo Guest: %s", e, extra={ + "action": "guest.handle", + "status": "error", + "task_id": self.task.id, + }) + finally: + # Stop Auxiliary modules. + if not self.stopped_aux: + self.stopped_aux = True + self.aux.stop() + + # Take a memory dump of the machine before shutting it off. + if self.cfg.cuckoo.memory_dump or self.task.memory: + logger( + "Taking full memory dump", + action="vm.memdump", status="pending", + vmname=self.machine.name + ) + try: + dump_path = os.path.join(self.storage, "memory.dmp") + machinery.dump_memory(self.machine.label, dump_path) + + logger( + "Taken full memory dump", + action="vm.memdump", status="success", + vmname=self.machine.name + ) + except NotImplementedError: + log.error( + "The memory dump functionality is not available for " + "the current machine manager.", extra={ + "action": "vm.memdump", + "status": "error", + "vmname": self.machine.name, + } + ) + except CuckooMachineError as e: + log.error("Machinery error: %s", e, extra={ + "action": "vm.memdump", + "status": "error", + }) + + logger( + "Stopping VM", + action="vm.stop", status="pending", + vmname=self.machine.name + ) + + try: + # Stop the analysis machine. + machinery.stop(self.machine.label) + except CuckooMachineError as e: + log.warning( + "Unable to stop machine %s: %s", + self.machine.label, e, extra={ + "action": "vm.stop", + "status": "error", + "vmname": self.machine.name, + } + ) + + logger( + "Stopped VM", + action="vm.stop", status="success", + vmname=self.machine.name + ) + + # Disable remote control after stopping the machine + # if it was enabled for the task. + if control_enabled: + try: + machinery.disable_remote_control(self.machine.label) + except NotImplementedError: + log.error( + "Remote control support has not been implemented for the " + "configured machinery module: %s", + config("cuckoo:cuckoo:machinery") + ) + + # Mark the machine in the database as stopped. Unless this machine + # has been marked as dead, we just keep it as "started" in the + # database so it'll not be used later on in this session. + self.db.guest_stop(guest_log) + + # After all this, we can make the ResultServer forget about the + # internal state for this analysis task. + ResultServer().del_task(self.task, self.machine) + + # Drop the network routing rules if any. + if not self.unrouted_network: + self.unroute_network() + + try: + # Release the analysis machine. But only if the machine has + # not turned dead yet. + machinery.release(self.machine.label) + except CuckooMachineError as e: + log.error( + "Unable to release machine %s, reason %s. You might need " + "to restore it manually.", self.machine.label, e, extra={ + "action": "vm.release", + "status": "error", + "vmname": self.machine.name, + } + ) + + return succeeded + + def process_results(self): + """Process the analysis results and generate the enabled reports.""" + logger( + "Starting task reporting", + action="task.report", status="pending" + ) + + # TODO Refactor this function as currently "cuckoo process" has a 1:1 + # copy of its code. TODO Also remove "archive" files. + results = RunProcessing(task=self.task).run() + RunSignatures(results=results).run() + RunReporting(task=self.task, results=results).run() + + # If the target is a file and the user enabled the option, + # delete the original copy. + if self.task.category == "file" and self.cfg.cuckoo.delete_original: + if not os.path.exists(self.task.target): + log.warning("Original file does not exist anymore: \"%s\": " + "File not found.", self.task.target) + else: + try: + os.remove(self.task.target) + except OSError as e: + log.error("Unable to delete original file at path " + "\"%s\": %s", self.task.target, e) + + # If the target is a file and the user enabled the delete copy of + # the binary option, then delete the copy. + if self.task.category == "file" and self.cfg.cuckoo.delete_bin_copy: + if not os.path.exists(self.binary): + log.warning("Copy of the original file does not exist anymore: \"%s\": File not found", self.binary) + else: + try: + os.remove(self.binary) + except OSError as e: + log.error("Unable to delete the copy of the original file at path \"%s\": %s", self.binary, e) + # Check if the binary in the analysis directory is an invalid symlink. If it is, delete it. + if os.path.islink(self.storage_binary) and not os.path.exists(self.storage_binary): + try: + os.remove(self.storage_binary) + except OSError as e: + log.error("Unable to delete symlink to the binary copy at path \"%s\": %s", self.storage_binary, e) + + log.info( + "Task #%d: reports generation completed", + self.task.id, extra={ + "action": "task.report", + "status": "success", + } + ) + + return True + + def run(self): + """Run manager thread.""" + global active_analysis_count + active_analysis_count += 1 + try: + self.launch_analysis() + + log.debug("Released database task #%d", self.task.id) + + if self.cfg.cuckoo.process_results: + self.store_task_info() + self.db.set_status(self.task.id, TASK_COMPLETED) + # TODO If self.process_results() is unified with apps.py's + # process() method, then ensure that TASK_FAILED_PROCESSING is + # handled correctly and not overwritten by the db.set_status() + # at the end of this method. + self.process_results() + + # We make a symbolic link ("latest") which links to the latest + # analysis - this is useful for debugging purposes. This is only + # supported under systems that support symbolic links. + if hasattr(os, "symlink"): + latest = cwd("storage", "analyses", "latest") + + # First we have to remove the existing symbolic link, then we + # have to create the new one. + # Deal with race conditions using a lock. + latest_symlink_lock.acquire() + try: + # As per documentation, lexists() returns True for dead + # symbolic links. + if os.path.lexists(latest): + os.remove(latest) + + os.symlink(self.storage, latest) + except OSError as e: + log.warning("Error pointing latest analysis symlink: %s" % e) + finally: + latest_symlink_lock.release() + + # overwrite task.json so we have the latest data inside + self.store_task_info() + log.info( + "Task #%d: analysis procedure completed", + self.task.id, extra={ + "action": "task.stop", + "status": "success", + } + ) + except: + log.exception("Failure in AnalysisManager.run", extra={ + "action": "task.stop", + "status": "error", + }) + finally: + if self.cfg.cuckoo.process_results: + self.db.set_status(self.task.id, TASK_REPORTED) + else: + self.db.set_status(self.task.id, TASK_COMPLETED) + task_log_stop(self.task.id) + active_analysis_count -= 1 + + def cleanup(self): + # In case the analysis manager crashes, the network cleanup + # should still be performed. + if not self.unrouted_network: + self.unroute_network() + + if not self.stopped_aux: + self.stopped_aux = True + self.aux.stop() + + def force_stop(self): + # Make the guest manager stop the status checking loop and return + # to the main analysis manager routine. + if self.db.guest_get_status(self.task.id): + self.db.guest_set_status(self.task.id, "stopping") + + self.guest_manager.stop() + log.debug("Force stopping task #%s", self.task.id) + +class Scheduler(object): + """Tasks Scheduler. + + This class is responsible for the main execution loop of the tool. It + prepares the analysis machines and keep waiting and loading for new + analysis tasks. + Whenever a new task is available, it launches AnalysisManager which will + take care of running the full analysis process and operating with the + assigned analysis machine. + """ + def __init__(self, maxcount=None): + self.running = True + self.cfg = Config() + self.db = Database() + self.maxcount = maxcount + self.total_analysis_count = 0 + self.analysis_managers = set() + + def initialize(self): + """Initialize the machine manager.""" + global machinery, machine_lock + + machinery_name = self.cfg.cuckoo.machinery + + max_vmstartup_count = self.cfg.cuckoo.max_vmstartup_count + if max_vmstartup_count: + machine_lock = threading.Semaphore(max_vmstartup_count) + else: + machine_lock = threading.Lock() + + log.info("Using \"%s\" as machine manager", machinery_name, extra={ + "action": "init.machinery", + "status": "success", + "machinery": machinery_name, + }) + + # Initialize the machine manager. + machinery = cuckoo.machinery.plugins[machinery_name]() + + # Provide a dictionary with the configuration options to the + # machine manager instance. + machinery.set_options(Config(machinery_name)) + + # Initialize the machine manager. + try: + machinery.initialize(machinery_name) + except CuckooMachineError as e: + raise CuckooCriticalError("Error initializing machines: %s" % e) + + # At this point all the available machines should have been identified + # and added to the list. If none were found, Cuckoo aborts the + # execution. TODO In the future we'll probably want get rid of this. + if not machinery.machines(): + raise CuckooCriticalError("No machines available.") + + log.info("Loaded %s machine/s", len(machinery.machines()), extra={ + "action": "init.machines", + "status": "success", + "count": len(machinery.machines()), + }) + + if len(machinery.machines()) > 1 and self.db.engine.name == "sqlite": + log.warning("As you've configured Cuckoo to execute parallel " + "analyses, we recommend you to switch to a MySQL or " + "a PostgreSQL database as SQLite might cause some " + "issues.") + + if len(machinery.machines()) > 4 and self.cfg.cuckoo.process_results: + log.warning("When running many virtual machines it is recommended " + "to process the results in separate 'cuckoo process' " + "instances to increase throughput and stability. " + "Please read the documentation about the " + "`Processing Utility`.") + + # Drop all existing packet forwarding rules for each VM. Just in case + # Cuckoo was terminated for some reason and various forwarding rules + # have thus not been dropped yet. + for machine in machinery.machines(): + if not machine.interface: + log.info("Unable to determine the network interface for VM " + "with name %s, Cuckoo will not be able to give it " + "full internet access or route it through a VPN! " + "Please define a default network interface for the " + "machinery or define a network interface for each " + "VM.", machine.name) + continue + + # Drop forwarding rule to each VPN. + if config("routing:vpn:enabled"): + for vpn in config("routing:vpn:vpns"): + rooter( + "forward_disable", machine.interface, + config("routing:%s:interface" % vpn), machine.ip + ) + + # Drop forwarding rule to the internet / dirty line. + if config("routing:routing:internet") != "none": + rooter( + "forward_disable", machine.interface, + config("routing:routing:internet"), machine.ip + ) + + def stop(self): + """Stop scheduler.""" + self.running = False + + # Force stop all analysis managers. + for am in self.analysis_managers: + try: + am.force_stop() + except Exception as e: + log.exception("Error force stopping analysis manager: %s", e) + + # Shutdown machine manager (used to kill machines that still alive). + machinery.shutdown() + + # Remove network rules if any are present and stop auxiliary modules + for am in self.analysis_managers: + try: + am.cleanup() + except Exception as e: + log.exception( + "Error while cleaning up analysis manager: %s", e + ) + + def _cleanup_managers(self): + cleaned = set() + for am in self.analysis_managers: + if not am.isAlive(): + try: + am.cleanup() + except Exception as e: + log.exception("Error in analysis manager cleanup: %s", e) + + cleaned.add(am) + return cleaned + + def start(self): + """Start scheduler.""" + self.initialize() + + log.info("Waiting for analysis tasks.") + + # Message queue with threads to transmit exceptions (used as IPC). + errors = Queue.Queue() + + # Command-line overrides the configuration file. + if self.maxcount is None: + self.maxcount = self.cfg.cuckoo.max_analysis_count + + # This loop runs forever. + while self.running: + time.sleep(1) + + # Run cleanup on finished analysis managers and untrack them + for am in self._cleanup_managers(): + self.analysis_managers.discard(am) + + # Wait until the machine lock is not locked. This is only the case + # when all machines are fully running, rather that about to start + # or still busy starting. This way we won't have race conditions + # with finding out there are no available machines in the analysis + # manager or having two analyses pick the same machine. + if not machine_lock.acquire(False): + logger( + "Could not acquire machine lock", + action="scheduler.machine_lock", status="busy" + ) + continue + + machine_lock.release() + + # If not enough free disk space is available, then we print an + # error message and wait another round (this check is ignored + # when the freespace configuration variable is set to zero). + if self.cfg.cuckoo.freespace: + # Resolve the full base path to the analysis folder, just in + # case somebody decides to make a symbolic link out of it. + dir_path = cwd("storage", "analyses") + + # TODO: Windows support + if hasattr(os, "statvfs"): + dir_stats = os.statvfs(dir_path.encode("utf8")) + + # Calculate the free disk space in megabytes. + space_available = dir_stats.f_bavail * dir_stats.f_frsize + space_available /= 1024 * 1024 + + if space_available < self.cfg.cuckoo.freespace: + log.error( + "Not enough free disk space! (Only %d MB!)", + space_available, extra={ + "action": "scheduler.diskspace", + "status": "error", + "available": space_available, + } + ) + continue + + # If we have limited the number of concurrently executing machines, + # are we currently at the maximum? + maxvm = self.cfg.cuckoo.max_machines_count + if maxvm and len(machinery.running()) >= maxvm: + logger( + "Already maxed out on running machines", + action="scheduler.machines", status="maxed" + ) + continue + + # If no machines are available, it's pointless to fetch for + # pending tasks. Loop over. + if not machinery.availables(): + logger( + "No available machines", + action="scheduler.machines", status="none" + ) + continue + + # Exits if max_analysis_count is defined in the configuration + # file and has been reached. + if self.maxcount and self.total_analysis_count >= self.maxcount: + if active_analysis_count <= 0: + log.debug("Reached max analysis count, exiting.", extra={ + "action": "scheduler.max_analysis", + "status": "success", + "limit": self.total_analysis_count, + }) + self.stop() + else: + logger( + "Maximum analyses hit, awaiting active to finish off", + action="scheduler.max_analysis", status="busy", + active=active_analysis_count + ) + continue + + # Fetch a pending analysis task. + # TODO This fixes only submissions by --machine, need to add + # other attributes (tags etc). + # TODO We should probably move the entire "acquire machine" logic + # from the Analysis Manager to the Scheduler and then pass the + # selected machine onto the Analysis Manager instance. + task, available = None, False + for machine in self.db.get_available_machines(): + task = self.db.fetch(machine=machine.name) + if task: + break + + if machine.is_analysis(): + available = True + + # We only fetch a new task if at least one of the available + # machines is not a "service" machine (again, please refer to the + # services auxiliary module for more information on service VMs). + if not task and available: + task = self.db.fetch(service=False) + + if task: + log.debug("Processing task #%s", task.id) + self.total_analysis_count += 1 + + # Initialize and start the analysis manager. + analysis = AnalysisManager(task.id, errors) + analysis.daemon = True + analysis.start() + self.analysis_managers.add(analysis) + + # Deal with errors. + try: + raise errors.get(block=False) + except Queue.Empty: + pass + + log.debug("End of analyses.") diff --git a/cuckoo/core/startup.py b/cuckoo/core/startup.py new file mode 100644 index 0000000000..f4066128b7 --- /dev/null +++ b/cuckoo/core/startup.py @@ -0,0 +1,528 @@ +# Copyright (C) 2012-2013 Claudio Guarnieri. +# Copyright (C) 2014-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import errno +import logging +import logging.handlers +import os +import requests +import socket +import sys +import yara + +from distutils.version import StrictVersion, LooseVersion + +import cuckoo + +from cuckoo.common.colors import red, green, yellow, bold, color +from cuckoo.common.config import Config, config, config2 +from cuckoo.common.exceptions import CuckooStartupError, CuckooFeedbackError +from cuckoo.common.files import temppath +from cuckoo.common.objects import File +from cuckoo.common.utils import cmp_version +from cuckoo.core.database import ( + Database, TASK_RUNNING, TASK_FAILED_ANALYSIS, TASK_PENDING +) +from cuckoo.core.extract import ExtractManager +from cuckoo.core.feedback import CuckooFeedbackObject +from cuckoo.core.log import init_logger +from cuckoo.core.plugins import RunSignatures +from cuckoo.core.rooter import rooter +from cuckoo.misc import cwd, version, mkdir + +log = logging.getLogger(__name__) + +def check_specific_config(filename): + sections = Config.configuration[filename] + for section, entries in sections.items(): + if section == "*" or section == "__star__": + continue + + # If an enabled field is present, check it beforehand. + if config("%s:%s:enabled" % (filename, section)) is False: + continue + + for key, value in entries.items(): + config( + "%s:%s:%s" % (filename, section, key), + check=True, strict=True + ) + +def check_configs(): + """Check if config files exist. + @raise CuckooStartupError: if config files do not exist. + """ + configs = ( + "auxiliary", "cuckoo", "memory", "processing", "reporting", "routing", + ) + + for filename in configs: + if not os.path.exists(cwd("conf", "%s.conf" % filename)): + raise CuckooStartupError( + "Config file does not exist at path: %s" % + cwd("conf", "%s.conf" % filename) + ) + + check_specific_config(filename) + + # Also check the specific machinery handler for this instance. + machinery = config("cuckoo:cuckoo:machinery") + if machinery not in Config.configuration: + raise CuckooStartupError( + "An unknown machinery has been chosen (machinery=%s)!" % machinery + ) + + check_specific_config(machinery) + + # If Cuckoo Feedback is enabled, ensure its configuration is valid. + feedback_enabled = ( + config("cuckoo:feedback:enabled") or + config("reporting:feedback:enabled") + ) + if feedback_enabled: + try: + CuckooFeedbackObject( + name=config("cuckoo:feedback:name"), + email=config("cuckoo:feedback:email"), + company=config("cuckoo:feedback:company"), + message="startup" + ).validate() + except CuckooFeedbackError as e: + raise CuckooStartupError( + "You have filled out the Cuckoo Feedback configuration, but " + "there's an error in it: %s" % e + ) + return True + +def check_version(ignore_vuln=False): + """Check version of Cuckoo.""" + if not config("cuckoo:cuckoo:version_check"): + return + + ignore_vuln = ignore_vuln or config("cuckoo:cuckoo:ignore_vulnerabilities") + + import pkg_resources + + print(" Checking for updates...") + + try: + r = requests.get( + "https://cuckoosandbox.org/updates.json", + params={"version": version}, timeout=6 + ) + r.raise_for_status() + r = r.json() + except (requests.RequestException, ValueError) as e: + print(red(" Error checking for the latest Cuckoo version: %s!" % e)) + return + + try: + old = StrictVersion(version) < StrictVersion(r["version"]) + except ValueError: + old = True + + warnings = [] + for deptype, vulns in r.get("vulnerable", {}).iteritems(): + for dep in vulns: + compare = dep.get("highest") or dep.get("lowest") + + # Check if any of the mentioned Python dependencies are installed + if deptype == "pydep": + try: + v = pkg_resources.get_distribution( + dep["name"]).parsed_version + except (pkg_resources.DistributionNotFound, ValueError): + continue + + # See if the mentioned virtualization software is used + elif deptype == "machinery": + if config("cuckoo:cuckoo:machinery") != dep["name"]: + continue + + # If the version number cannot be determined, raise a warning + # to be sure. Virtualization vulnerabilities can potentially + # cause a lot of damage + v = cuckoo.machinery.plugins[dep["name"]].version() + if not v: + warnings.append( + bold(red( + "Potentially vulnerable %s version installed. " + "Failed to retrieve its version. Update if version" + " is: %s" % (dep["name"], compare)))) + continue + + else: + continue + + warn = False + # If a range is specified, check if the current version falls + # within the range. + if dep.get("highest") and dep.get("lowest"): + lv = LooseVersion(str(v)) + if (lv >= LooseVersion(dep["lowest"]) and + lv <= LooseVersion(dep["highest"])): + warn = True + + # If no range is specified, use the specified operator to see if + # the installed version is + # 'if highest/lowest specified' + elif cmp_version(str(v), compare, dep["op"]): + warn = True + + # Warn the user the dependency must be updated/ + if warn: + info = dep.get("info") + message = "Vulnerable version of %s installed (%s). It is " \ + "highly recommended to update. Please update and " \ + "restart Cuckoo." % (dep["name"], v) + + if deptype == "pydep": + message += " 'pip install %s%s'" % ( + dep["name"], dep["recommended"] + ) + + else: + message += " Recommended version: %s" % dep["recommended"] + + message = bold(red(message)) + + if info: + message += yellow("\nAdditional information: %s" % info) + + warnings.append(message) + + if warnings: + print(color(bold(red("Vulnerable dependencies found\n")), 5)) + for warning in warnings: + print("--> %s\n" % color(warning, 4)) + + if warnings and not ignore_vuln: + print( + "This check can be disabled by enabling " + "'ignore_vulnerabilities' in cuckoo.conf under the " + "[cuckoo] section" + ) + sys.exit(1) + + if old: + msg = "Cuckoo Sandbox version %s is available now." % r["version"] + print(red(" Outdated! ") + msg) + else: + print(green(" You're good to go!")) + + print("\n Our latest blogposts:") + for blogpost in r["blogposts"]: + print(" * %s, %s." % (yellow(blogpost["title"]), blogpost["date"])) + print(" %s" % red(blogpost["oneline"])) + print(" More at %s" % blogpost["url"]) + print("") + return r + +def init_logging(level): + """Initialize logging.""" + logging.getLogger().setLevel(logging.DEBUG) + init_logger("cuckoo.log", level) + init_logger("cuckoo.json") + init_logger("task") + +def init_console_logging(level=logging.INFO): + """Initialize logging only to console and database.""" + logging.getLogger().setLevel(logging.DEBUG) + init_logger("console", level) + init_logger("database") + +def init_logfile(logfile): + init_logger(logfile, logging.DEBUG) + +def init_tasks(): + """Check tasks and reschedule uncompleted ones.""" + db = Database() + + log.debug("Checking for locked tasks..") + for task in db.list_tasks(status=TASK_RUNNING): + if config("cuckoo:cuckoo:reschedule"): + task_id = db.reschedule(task.id) + log.info( + "Rescheduled task with ID %s and target %s: task #%s", + task.id, task.target, task_id + ) + else: + db.set_status(task.id, TASK_FAILED_ANALYSIS) + log.info( + "Updated running task ID %s status to failed_analysis", + task.id + ) + + log.debug("Checking for pending service tasks..") + for task in db.list_tasks(status=TASK_PENDING, category="service"): + db.set_status(task.id, TASK_FAILED_ANALYSIS) + +def init_modules(): + """Initialize plugins.""" + log.debug("Imported modules...") + + categories = ( + "auxiliary", "machinery", "processing", "signatures", "reporting", + ) + + # Call the init_once() static method of each plugin/module. If an exception + # is thrown in that initialization call, then a hard error is appropriate. + for category in categories: + for module in cuckoo.plugins[category]: + module.init_once() + + for category in categories: + log.debug("Imported \"%s\" modules:", category) + + entries = cuckoo.plugins[category] + for entry in entries: + if entry == entries[-1]: + log.debug("\t `-- %s", entry.__name__) + else: + log.debug("\t |-- %s", entry.__name__) + + # Initialize the RunSignatures module with all available Signatures and + # the ExtractManager with all available Extractors. + RunSignatures.init_once() + ExtractManager.init_once() + +def init_yara(): + """Initialize & load/compile Yara rules.""" + categories = ( + "binaries", "urls", "memory", "scripts", "shellcode", + "dumpmem", "office", + ) + log.debug("Initializing Yara...") + for category in categories: + dirpath = cwd("yara", category) + if not os.path.exists(dirpath): + log.warning("Missing Yara directory: %s?", dirpath) + + rules, indexed = {}, [] + for dirpath, dirnames, filenames in os.walk(dirpath, followlinks=True): + for filename in filenames: + if not filename.endswith((".yar", ".yara")): + continue + + filepath = os.path.join(dirpath, filename) + + try: + # TODO Once Yara obtains proper Unicode filepath support we + # can remove this check. See also this Github issue: + # https://github.com/VirusTotal/yara-python/issues/48 + assert len(str(filepath)) == len(filepath) + except (UnicodeEncodeError, AssertionError): + log.warning( + "Can't load Yara rules at %r as Unicode filepaths are " + "currently not supported in combination with Yara!", + filepath + ) + continue + + rules["rule_%s_%d" % (category, len(rules))] = filepath + indexed.append(filename) + + # Need to define each external variable that will be used in the + # future. Otherwise Yara will complain. + externals = { + "filename": "", + } + + try: + File.yara_rules[category] = yara.compile( + filepaths=rules, externals=externals + ) + except yara.Error as e: + raise CuckooStartupError( + "There was a syntax error in one or more Yara rules: %s" % e + ) + + # The memory.py processing module requires a yara file with all of its + # rules embedded in it, so create this file to remain compatible. + if category == "memory": + f = open(cwd("stuff", "index_memory.yar"), "wb") + for filename in sorted(indexed): + f.write('include "%s"\n' % cwd("yara", "memory", filename)) + + indexed = sorted(indexed) + for entry in indexed: + if (category, entry) == indexed[-1]: + log.debug("\t `-- %s %s", category, entry) + else: + log.debug("\t |-- %s %s", category, entry) + + # Store the compiled Yara rules for the "dumpmem" category in + # $CWD/stuff/ so that we may pass it along to zer0m0n during analysis. + File.yara_rules["dumpmem"].save(cwd("stuff", "dumpmem.yarac")) + +def init_binaries(): + """Inform the user about the need to periodically look for new analyzer + binaries. These include the Windows monitor etc.""" + def throw(): + raise CuckooStartupError( + "The binaries used for Windows analysis are updated regularly, " + "independently from the release line. It appears that you're " + "not up-to-date. This may happen when you've just installed the " + "latest development version of Cuckoo or when you've updated " + "to the latest Cuckoo. In order to get up-to-date, please run " + "the following command: `cuckoo community`." + ) + + dirpath = cwd("monitor", "latest") + + # If "latest" is a symbolic link, check that it exists. + if os.path.islink(dirpath): + if not os.path.exists(dirpath): + throw() + # If "latest" is a file, check that it contains a legitimate hash. + elif os.path.isfile(dirpath): + monitor = os.path.basename(open(dirpath, "rb").read().strip()) + if not monitor or not os.path.isdir(cwd("monitor", monitor)): + throw() + else: + throw() + +def init_rooter(): + """If required, check if the rooter is running and if we can connect + to it. The default configuration doesn't require the rooter to be ran.""" + required = ( + config("routing:routing:route") != "none" or + config("routing:routing:internet") != "none" or + config("routing:routing:drop") or + config("routing:inetsim:enabled") or + config("routing:tor:enabled") or + config("routing:vpn:enabled") + ) + if not required: + return + + s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + + try: + s.connect(config("cuckoo:cuckoo:rooter")) + except socket.error as e: + if e.strerror == "No such file or directory": + raise CuckooStartupError( + "The rooter is required but it is either not running or it " + "has been configured to a different Unix socket path. Please " + "refer to the documentation on working with the rooter." + ) + + if e.strerror == "Connection refused": + raise CuckooStartupError( + "The rooter is required but we can't connect to it as the " + "rooter is not actually running. Please refer to the " + "documentation on working with the rooter." + ) + + if e.strerror == "Permission denied": + raise CuckooStartupError( + "The rooter is required but we can't connect to it due to " + "incorrect permissions. Did you assign it the correct group? " + "Please refer to the documentation on working with the " + "rooter." + ) + + raise CuckooStartupError("Unknown rooter error: %s" % e) + + # Do not forward any packets unless we have explicitly stated so. + rooter("forward_drop") + + # Enable stateful connection tracking (but only once). + rooter("state_disable") + rooter("state_enable") + +def init_routing(): + """Initialize and check whether the routing information is correct.""" + interfaces = set() + + # Check if all configured VPNs exist and are up and enable NAT on + # each VPN interface. + if config("routing:vpn:enabled"): + for name in config("routing:vpn:vpns"): + entry = config2("routing", name) + if not rooter("nic_available", entry.interface): + raise CuckooStartupError( + "The network interface that has been configured for " + "VPN %s is not available." % entry.name + ) + + if not rooter("rt_available", entry.rt_table): + raise CuckooStartupError( + "The routing table that has been configured for " + "VPN %s is not available." % entry.name + ) + + interfaces.add((entry.rt_table, entry.interface)) + + standard_routes = "none", "drop", "internet", "inetsim", "tor" + + # Check whether the default VPN exists if specified. + if config("routing:routing:route") not in standard_routes: + if config("routing:routing:route") not in config("routing:vpn:vpns"): + raise CuckooStartupError( + "The default routing target (%s) has not been configured in " + "routing.conf, is it supposed to be a VPN?" % + config("routing:routing:route") + ) + + if not config("routing:vpn:enabled"): + raise CuckooStartupError( + "The default route configured is a VPN, but VPNs have " + "not been enabled in routing.conf." + ) + + # Check whether the dirty line exists if it has been defined. + if config("routing:routing:internet") != "none": + if not rooter("nic_available", config("routing:routing:internet")): + raise CuckooStartupError( + "The network interface that has been configured as dirty " + "line is not available." + ) + + if not rooter("rt_available", config("routing:routing:rt_table")): + raise CuckooStartupError( + "The routing table that has been configured for dirty " + "line interface is not available." + ) + + interfaces.add(( + config("routing:routing:rt_table"), + config("routing:routing:internet") + )) + + for rt_table, interface in interfaces: + # Disable & enable NAT on this network interface. Disable it just + # in case we still had the same rule from a previous run. + rooter("disable_nat", interface) + rooter("enable_nat", interface) + + # Populate routing table with entries from main routing table. + if config("routing:routing:auto_rt"): + rooter("flush_rttable", rt_table) + rooter("init_rttable", rt_table, interface) + +def ensure_tmpdir(): + """Verify if the current user can read and create files in the + cuckoo temporary directory (and creates it, if needed).""" + try: + if not os.path.isdir(temppath()): + mkdir(temppath()) + except OSError as e: + # Currently we only handle EACCES. + if e.errno != errno.EACCES: + raise + + if os.path.isdir(temppath()) and os.access(temppath(), os.R_OK | os.W_OK): + return True + + print red( + "Cuckoo cannot read or write files into the temporary directory '%s'," + " please make sure the user running Cuckoo has the ability to do so. " + "If the directory does not yet exist and the parent directory is " + "owned by root, then please create and chown the directory with root." + % temppath() + ) + return False diff --git a/cuckoo/core/submit.py b/cuckoo/core/submit.py new file mode 100644 index 0000000000..589563b76e --- /dev/null +++ b/cuckoo/core/submit.py @@ -0,0 +1,434 @@ +# Copyright (C) 2016-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import copy +import json +import logging +import os +import sflock +import zipfile + +from cuckoo.common.exceptions import CuckooOperationalError +from cuckoo.common.files import Folders, Files, Storage +from cuckoo.common.utils import validate_url, validate_hash +from cuckoo.common.virustotal import VirusTotalAPI +from cuckoo.core.database import Database, TASK_COMPLETED +from cuckoo.misc import cwd, mkdir + +log = logging.getLogger(__name__) +db = Database() + +class SubmitManager(object): + known_web_options = [ + "enable-injection", "enforce-timeout", "full-memory-dump", + "process-memory-dump", "remote-control", "simulated-human-interaction", + ] + + def _handle_string(self, submit, tmppath, line): + if not line: + return + + if validate_hash(line): + try: + filedata = VirusTotalAPI().hash_fetch(line) + except CuckooOperationalError as e: + submit["errors"].append( + "Error retrieving file hash: %s" % e + ) + return + + filepath = Files.create(tmppath, line, filedata) + + submit["data"].append({ + "type": "file", + "data": filepath + }) + return + + if validate_url(line): + submit["data"].append({ + "type": "url", + "data": validate_url(line), + }) + return + + submit["errors"].append( + "'%s' was neither a valid hash or url" % line + ) + + def translate_options_from(self, entry, options): + """Translate from Web Interface options to Cuckoo database options.""" + ret = {} + + if not options.get("simulated-human-interaction", True): + ret["human"] = int(options.get("simulated-human-interaction", True)) + + if not options.get("enable-injection", True): + ret["free"] = "yes" + + if options.get("process-memory-dump"): + ret["procmemdump"] = "yes" + + if options.get("remote-control"): + ret["remotecontrol"] = "yes" + + # VPN takes precedence over the network-routing option (this should + # actually be resolved in the frontend, though). + if entry.get("vpn"): + ret["route"] = entry["vpn"] + elif entry.get("network-routing"): + ret["route"] = entry["network-routing"] + + # Propagate any additional manually set key/value pairs. + for key, value in options.items(): + if key not in self.known_web_options: + ret[key] = value + + return ret + + def translate_options_to(self, options): + """Translate from Cuckoo database options to Web Interface options.""" + ret = {} + + if not int(options.get("human", "1")): + ret["simulated-human-interaction"] = False + options.pop("human") + + if options.get("free") == "yes": + ret["enable-injection"] = False + options.pop("free") + + if options.get("procmemdump") == "yes": + ret["process-memory-dump"] = True + options.pop("procmemdump") + + if options.get("remotecontrol") == "yes": + ret["remote-control"] = True + options.pop("remotecontrol") + + if options.get("route"): + ret["network-routing"] = options["route"] + options.pop("route") + + # Propagate any additional manually set key/value pairs. + for key, value in options.items(): + if key not in self.known_web_options: + ret[key] = value + + return ret + + def pre(self, submit_type, data, options=None): + """ + The first step to submitting new analysis. + @param submit_type: "files" or "strings" + @param data: a list of dicts containing "name" (file name) + and "data" (file data) or a list of strings (urls or hashes) + @return: submit id + """ + if submit_type not in ("strings", "files"): + log.error("Bad parameter '%s' for submit_type", submit_type) + return False + + path_tmp = Folders.create_temp() + submit_data = { + "data": [], + "errors": [], + "options": options or {}, + } + + if submit_type == "strings": + for line in data: + self._handle_string(submit_data, path_tmp, line.strip()) + + if submit_type == "files": + for entry in data: + filename = Storage.get_filename_from_path(entry["name"]) + filepath = Files.create(path_tmp, filename, entry["data"]) + submit_data["data"].append({ + "type": "file", + "data": filepath, + "options": self.translate_options_to( + entry.get("options", {}) + ), + }) + + return db.add_submit(path_tmp, submit_type, submit_data) + + def get_files(self, submit_id, password=None, astree=False): + """ + Return files or URLs from a submitted analysis. + @param password: The password to unlock container archives with + @param astree: sflock option; determines the format in which the files are returned + @return: A tree of files + """ + submit = db.view_submit(submit_id) + files, duplicates = [], [] + + for data in submit.data["data"]: + if data["type"] == "file": + filename = Storage.get_filename_from_path(data["data"]) + filepath = os.path.join(submit.tmp_path, filename) + + unpacked = sflock.unpack( + filepath=filepath, password=password, + duplicates=duplicates + ) + + if astree: + unpacked = unpacked.astree(sanitize=True) + + files.append(unpacked) + elif data["type"] == "url": + files.append({ + "filename": data["data"], + "filepath": "", + "relapath": "", + "selected": True, + "size": 0, + "type": "url", + "package": "ie", + "extrpath": [], + "duplicate": False, + "children": [], + "mime": "text/html", + "finger": { + "magic_human": "url", + "magic": "url" + } + }) + else: + raise RuntimeError( + "Unknown data entry type: %s" % data["type"] + ) + + return files, submit.data["errors"], submit.data["options"] + + def submit(self, submit_id, config): + """Read, interpret, and convert the JSON configuration provided by + the Web Interface into something we insert into the database.""" + ret = [] + submit = db.view_submit(submit_id) + + machines = {} + + for entry in config["file_selection"]: + # Merge the global & per-file analysis options. + info = copy.deepcopy(config["global"]) + info.update(entry) + info.update(entry.get("options", {})) + options = copy.deepcopy(config["global"]["options"]) + options.update(entry.get("options", {}).get("options", {})) + + machine = info.get("machine") + if machine: + if machine not in machines: + m = db.view_machine(machine) + # TODO Add error handling for missing machine entry. + machines[machine] = m.label if m else None + + machine = machines[machine] + else: + machine = None + + kw = { + "package": info.get("package") or "", + "timeout": info.get("timeout", 120), + "priority": info.get("priority"), + "custom": info.get("custom"), + "owner": info.get("owner"), + "tags": info.get("tags"), + "memory": options.get("full-memory-dump"), + "enforce_timeout": options.get("enforce-timeout"), + "machine": machine, + "platform": info.get("platform"), + "options": self.translate_options_from(info, options), + "submit_id": submit_id, + } + + if entry["type"] == "url": + ret.append(db.add_url( + url=info["filename"], **kw + )) + continue + + # for each selected file entry, create a new temp. folder + path_dest = Folders.create_temp() + + if not info["extrpath"]: + path = os.path.join( + submit.tmp_path, os.path.basename(info["filename"]) + ) + + filepath = Files.copy(path, path_dest=path_dest) + + ret.append(db.add_path( + file_path=filepath, **kw + )) + elif len(info["extrpath"]) == 1: + arcpath = os.path.join( + submit.tmp_path, os.path.basename(info["arcname"]) + ) + if not os.path.exists(arcpath): + submit.data["errors"].append( + "Unable to find parent archive file: %s" % + os.path.basename(info["arcname"]) + ) + continue + + arc = sflock.zipify(sflock.unpack( + contents=open(arcpath, "rb").read(), + filename=info["arcname"] + )) + + # Create a .zip archive out of this container. + arcpath = Files.temp_named_put( + arc, os.path.basename(info["arcname"]) + ) + + ret.append(db.add_archive( + file_path=arcpath, filename=info["relaname"], **kw + )) + else: + arcpath = os.path.join( + submit.tmp_path, os.path.basename(info["arcname"]) + ) + if not os.path.exists(arcpath): + submit.data["errors"].append( + "Unable to find parent archive file: %s" % + os.path.basename(info["arcname"]) + ) + continue + + content = sflock.unpack(arcpath).read(info["extrpath"][:-1]) + subarc = sflock.unpack( + contents=content, filename=info["extrpath"][-2] + ) + + # Write intermediate .zip archive file. + arcpath = Files.temp_named_put( + sflock.zipify(subarc), + os.path.basename(info["extrpath"][-2]) + ) + + ret.append(db.add_archive( + file_path=arcpath, filename=info["relaname"], **kw + )) + + return ret + + def import_(self, f, submit_id): + """Import an analysis identified by the file(-like) object f.""" + try: + z = zipfile.ZipFile(f) + except zipfile.BadZipfile: + raise CuckooOperationalError( + "Imported analysis is not a proper .zip file." + ) + + # Ensure there are no files with illegal or potentially insecure names. + # TODO Keep in mind that if we start to support other archive formats + # (e.g., .tar) that those may also support symbolic links. In that case + # we should probably start using sflock here. + for filename in z.namelist(): + if filename.startswith("/") or ".." in filename or ":" in filename: + raise CuckooOperationalError( + "The .zip file contains a file with a potentially " + "incorrect filename: %s" % filename + ) + + if "task.json" not in z.namelist(): + raise CuckooOperationalError( + "The task.json file is required in order to be able to import " + "an analysis! This file contains metadata about the analysis." + ) + + required_fields = { + "options": dict, "route": basestring, "package": basestring, + "target": basestring, "category": basestring, "memory": bool, + "timeout": (int, long), "priority": (int, long), + "custom": basestring, "tags": (tuple, list), + } + + try: + info = json.loads(z.read("task.json")) + for key, type_ in required_fields.items(): + if key not in info: + raise ValueError("missing %s" % key) + if info[key] is not None and not isinstance(info[key], type_): + raise ValueError("%s => %s" % (key, info[key])) + except ValueError as e: + raise CuckooOperationalError( + "The provided task.json file, required for properly importing " + "the analysis, is incorrect or incomplete (%s)." % e + ) + + if info["category"] == "url": + task_id = db.add_url( + url=info["target"], package=info["package"], + timeout=info["timeout"], options=info["options"], + priority=info["priority"], custom=info["custom"], + memory=info["memory"], tags=info["tags"], submit_id=submit_id + ) + else: + # Users may have the "delete_bin_copy" enabled and in such cases + # the binary file won't be included in the .zip file. + if "binary" in z.namelist(): + filepath = Files.temp_named_put( + z.read("binary"), os.path.basename(info["target"]) + ) + else: + # Generate a temp file as a target if no target is present + filepath = Files.temp_put("") + + # We'll be updating the target shortly. + task_id = db.add_path( + file_path=filepath, package=info["package"], + timeout=info["timeout"], options=info["options"], + priority=info["priority"], custom=info["custom"], + memory=info["memory"], tags=info["tags"], submit_id=submit_id + ) + + if not task_id: + raise CuckooOperationalError( + "There was an error creating a task for the to-be imported " + "analysis in our database.. Can't proceed." + ) + + # The constructors currently don't accept this argument. + db.set_route(task_id, info["route"]) + + mkdir(cwd(analysis=task_id)) + z.extractall(cwd(analysis=task_id)) + + # If there's an analysis.json file, load it up to figure out additional + # metdata regarding this analysis. + if os.path.exists(cwd("analysis.json", analysis=task_id)): + try: + obj = json.load( + open(cwd("analysis.json", analysis=task_id), "rb") + ) + if not isinstance(obj, dict): + raise ValueError + if "errors" in obj and not isinstance(obj["errors"], list): + raise ValueError + if "action" in obj and not isinstance(obj["action"], list): + raise ValueError + except ValueError: + log.warning( + "An analysis.json file was provided, but wasn't a valid " + "JSON object/structure that we can to enhance the " + "analysis information." + ) + else: + for error in set(obj.get("errors", [])): + if isinstance(error, basestring): + db.add_error(error, task_id) + for action in set(obj.get("action", [])): + if isinstance(action, basestring): + db.add_error("", task_id, action) + + # We set this analysis as completed so that it will be processed + # automatically (assuming 'cuckoo process' is running). + db.set_status(task_id, TASK_COMPLETED) + return task_id diff --git a/cuckoo/data/__init__.py b/cuckoo/data/__init__.py new file mode 100644 index 0000000000..a717326af0 --- /dev/null +++ b/cuckoo/data/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. diff --git a/cuckoo/data/agent/agent.py b/cuckoo/data/agent/agent.py new file mode 100644 index 0000000000..1be1c170db --- /dev/null +++ b/cuckoo/data/agent/agent.py @@ -0,0 +1,427 @@ +#!/usr/bin/env python +# Copyright (C) 2015-2019 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import argparse +import cgi +import io +import json +import os +import platform +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import traceback +import zipfile + +import SimpleHTTPServer +import SocketServer + +AGENT_VERSION = "0.10" +AGENT_FEATURES = [ + "execpy", "pinning", "logs", "largefile", "unicodepath", +] + +sys.stdout = io.BytesIO() +sys.stderr = io.BytesIO() + +class MiniHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): + server_version = "Cuckoo Agent" + + def do_GET(self): + request.client_ip, request.client_port = self.client_address + request.form = {} + request.files = {} + request.method = "GET" + + self.httpd.handle(self) + + def do_POST(self): + environ = { + "REQUEST_METHOD": "POST", + "CONTENT_TYPE": self.headers.get("Content-Type"), + } + + form = cgi.FieldStorage(fp=self.rfile, + headers=self.headers, + environ=environ) + + request.client_ip, request.client_port = self.client_address + request.form = {} + request.files = {} + request.method = "POST" + + # Another pretty fancy workaround. Since we provide backwards + # compatibility with the Old Agent we will get an xmlrpc request + # from the analyzer when the analysis has finished. Now xmlrpc being + # xmlrpc we're getting text/xml as content-type which cgi does not + # handle. This check detects when there is no available data rather + # than getting a hard exception trying to do so. + if form.list: + for key in form.keys(): + value = form[key] + if value.filename: + request.files[key] = value.file + else: + request.form[key] = value.value.decode("utf8") + + self.httpd.handle(self) + +class MiniHTTPServer(object): + def __init__(self): + self.handler = MiniHTTPRequestHandler + + # Reference back to the server. + self.handler.httpd = self + + self.routes = { + "GET": [], + "POST": [], + } + + def run(self, host="0.0.0.0", port=8000): + self.s = SocketServer.TCPServer((host, port), self.handler) + self.s.allow_reuse_address = True + self.s.serve_forever() + + def route(self, path, methods=["GET"]): + def register(fn): + for method in methods: + self.routes[method].append((re.compile(path + "$"), fn)) + return fn + return register + + def handle(self, obj): + if "client_ip" in state and request.client_ip != state["client_ip"]: + if request.client_ip != "127.0.0.1": + return + if obj.path != "/status" or request.method != "POST": + return + + for route, fn in self.routes[obj.command]: + if route.match(obj.path): + ret = fn() + break + else: + ret = json_error(404, message="Route not found") + + ret.init() + obj.send_response(ret.status_code) + ret.headers(obj) + obj.end_headers() + + if isinstance(ret, jsonify): + obj.wfile.write(ret.json()) + elif isinstance(ret, send_file): + ret.write(obj.wfile) + + def shutdown(self): + # BaseServer also features a .shutdown() method, but you can't use + # that from the same thread as that will deadlock the whole thing. + self.s._BaseServer__shutdown_request = True + +class jsonify(object): + """Wrapper that represents Flask.jsonify functionality.""" + def __init__(self, **kwargs): + self.status_code = 200 + self.values = kwargs + + def init(self): + pass + + def json(self): + return json.dumps(self.values) + + def headers(self, obj): + pass + +class send_file(object): + """Wrapper that represents Flask.send_file functionality.""" + def __init__(self, path): + self.path = path + self.status_code = 200 + + def init(self): + if not os.path.isfile(self.path): + self.status_code = 404 + self.length = 0 + else: + self.length = os.path.getsize(self.path) + + def write(self, sock): + if not self.length: + return + + with open(self.path, "rb") as f: + while True: + buf = f.read(1024 * 1024) + if not buf: + break + + sock.write(buf) + + def headers(self, obj): + obj.send_header("Content-Length", self.length) + +class request(object): + form = {} + files = {} + client_ip = None + client_port = None + method = None + environ = { + "werkzeug.server.shutdown": lambda: app.shutdown(), + } + +app = MiniHTTPServer() +state = {} + +def json_error(error_code, message): + r = jsonify(message=message, error_code=error_code) + r.status_code = error_code + return r + +def json_exception(message): + r = jsonify(message=message, error_code=500, + traceback=traceback.format_exc()) + r.status_code = 500 + return r + +def json_success(message, **kwargs): + return jsonify(message=message, **kwargs) + +@app.route("/") +def get_index(): + return json_success( + "Cuckoo Agent!", version=AGENT_VERSION, features=AGENT_FEATURES + ) + +@app.route("/status") +def get_status(): + return json_success("Analysis status", + status=state.get("status"), + description=state.get("description")) + +@app.route("/status", methods=["POST"]) +def put_status(): + if "status" not in request.form: + return json_error(400, "No status has been provided") + + state["status"] = request.form["status"] + state["description"] = request.form.get("description") + return json_success("Analysis status updated") + +@app.route("/logs") +def get_logs(): + return json_success( + "Agent logs", + stdout=sys.stdout.getvalue(), + stderr=sys.stderr.getvalue() + ) + +@app.route("/system") +def get_system(): + return json_success("System", system=platform.system()) + +@app.route("/environ") +def get_environ(): + return json_success("Environment variables", environ=dict(os.environ)) + +@app.route("/path") +def get_path(): + return json_success("Agent path", filepath=os.path.abspath(__file__)) + +@app.route("/mkdir", methods=["POST"]) +def do_mkdir(): + if "dirpath" not in request.form: + return json_error(400, "No dirpath has been provided") + + mode = int(request.form.get("mode", 0777)) + + try: + os.makedirs(request.form["dirpath"], mode=mode) + except: + return json_exception("Error creating directory") + + return json_success("Successfully created directory") + +@app.route("/mktemp", methods=["GET", "POST"]) +def do_mktemp(): + suffix = request.form.get("suffix", "") + prefix = request.form.get("prefix", "tmp") + dirpath = request.form.get("dirpath") + + try: + fd, filepath = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dirpath) + except: + return json_exception("Error creating temporary file") + + os.close(fd) + + return json_success("Successfully created temporary file", + filepath=filepath) + +@app.route("/mkdtemp", methods=["GET", "POST"]) +def do_mkdtemp(): + suffix = request.form.get("suffix", "") + prefix = request.form.get("prefix", "tmp") + dirpath = request.form.get("dirpath") + + try: + dirpath = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=dirpath) + except: + return json_exception("Error creating temporary directory") + + return json_success("Successfully created temporary directory", + dirpath=dirpath) + +@app.route("/store", methods=["POST"]) +def do_store(): + if "filepath" not in request.form: + return json_error(400, "No filepath has been provided") + + if "file" not in request.files: + return json_error(400, "No file has been provided") + + try: + with open(request.form["filepath"], "wb") as f: + shutil.copyfileobj(request.files["file"], f, 10*1024*1024) + except: + return json_exception("Error storing file") + + return json_success("Successfully stored file") + +@app.route("/retrieve", methods=["POST"]) +def do_retrieve(): + if "filepath" not in request.form: + return json_error(400, "No filepath has been provided") + + return send_file(request.form["filepath"]) + +@app.route("/extract", methods=["POST"]) +def do_extract(): + if "dirpath" not in request.form: + return json_error(400, "No dirpath has been provided") + + if "zipfile" not in request.files: + return json_error(400, "No zip file has been provided") + + try: + with zipfile.ZipFile(request.files["zipfile"], "r") as archive: + archive.extractall(request.form["dirpath"]) + except: + return json_exception("Error extracting zip file") + + return json_success("Successfully extracted zip file") + +@app.route("/remove", methods=["POST"]) +def do_remove(): + if "path" not in request.form: + return json_error(400, "No path has been provided") + + try: + if os.path.isdir(request.form["path"]): + # Mark all files as readable so they can be deleted. + for dirpath, _, filenames in os.walk(request.form["path"]): + for filename in filenames: + os.chmod(os.path.join(dirpath, filename), stat.S_IWRITE) + + shutil.rmtree(request.form["path"]) + message = "Successfully deleted directory" + elif os.path.isfile(request.form["path"]): + os.chmod(request.form["path"], stat.S_IWRITE) + os.remove(request.form["path"]) + message = "Successfully deleted file" + else: + return json_error(404, "Path provided does not exist") + except: + return json_exception("Error removing file or directory") + + return json_success(message) + +@app.route("/execute", methods=["POST"]) +def do_execute(): + if "command" not in request.form: + return json_error(400, "No command has been provided") + + # Execute the command asynchronously? As a shell command? + async = "async" in request.form + shell = "shell" in request.form + + cwd = request.form.get("cwd") + stdout = stderr = None + + try: + if async: + subprocess.Popen(request.form["command"], shell=shell, cwd=cwd) + else: + p = subprocess.Popen( + request.form["command"], shell=shell, cwd=cwd, + stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + stdout, stderr = p.communicate() + except: + return json_exception("Error executing command") + + return json_success("Successfully executed command", + stdout=stdout, stderr=stderr) + +@app.route("/execpy", methods=["POST"]) +def do_execpy(): + if "filepath" not in request.form: + return json_error(400, "No Python file has been provided") + + # Execute the command asynchronously? As a shell command? + async = "async" in request.form + + cwd = request.form.get("cwd") + stdout = stderr = None + + args = [ + sys.executable, + request.form["filepath"], + ] + + try: + if async: + subprocess.Popen(args, cwd=cwd) + else: + p = subprocess.Popen(args, cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + except: + return json_exception("Error executing command") + + return json_success("Successfully executed command", + stdout=stdout, stderr=stderr) + +@app.route("/pinning") +def do_pinning(): + if "client_ip" in state: + return json_error(500, "Agent has already been pinned to an IP!") + + state["client_ip"] = request.client_ip + return json_success("Successfully pinned Agent", + client_ip=request.client_ip) + +@app.route("/kill") +def do_kill(): + shutdown = request.environ.get("werkzeug.server.shutdown") + if shutdown is None: + return json_error(500, "Not running with the Werkzeug server") + + shutdown() + return json_success("Quit the Cuckoo Agent") + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("host", nargs="?", default="0.0.0.0") + parser.add_argument("port", nargs="?", default="8000") + args = parser.parse_args() + + app.run(host=args.host, port=int(args.port)) diff --git a/agent/agent.sh b/cuckoo/data/agent/agent.sh similarity index 76% rename from agent/agent.sh rename to cuckoo/data/agent/agent.sh index 748ac228fa..35f0428e2f 100755 --- a/agent/agent.sh +++ b/cuckoo/data/agent/agent.sh @@ -1,5 +1,6 @@ #!/bin/bash -# Copyright (C) 2010-2015 Cuckoo Foundation. +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. diff --git a/cuckoo/data/analyzer/android/analyzer.py b/cuckoo/data/analyzer/android/analyzer.py new file mode 100644 index 0000000000..b31b54ec9c --- /dev/null +++ b/cuckoo/data/analyzer/android/analyzer.py @@ -0,0 +1,261 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. +# Originally contributed by Check Point Software Technologies, Ltd. + +import os +import logging +import pkgutil +import shutil +import sys +import xmlrpclib +import time + +from lib.core.packages import choose_package +from lib.common.exceptions import CuckooError, CuckooPackageError +from lib.common.abstracts import Package, Auxiliary +from lib.common.constants import PATHS +from lib.core.config import Config +from lib.core.startup import init_logging +from modules import auxiliary + +logging.disable(level=logging.DEBUG) +log = logging.getLogger() + +class Analyzer(object): + def __init__(self): + self.config = None + self.target = None + + def complete(self): + """End analysis.""" + log.info("Analysis completed") + + def get_options(self): + """Get analysis options. + @return: options dict. + """ + # The analysis package can be provided with some options in the + # following format: + # option1=value1,option2=value2,option3=value3 + # + # Here we parse such options and provide a dictionary that will be made + # accessible to the analysis package. + options = {} + if self.config.options: + try: + # Split the options by comma. + fields = self.config.options.strip().split(",") + except ValueError as e: + log.warning("Failed parsing the options: %s", e) + else: + for field in fields: + # Split the name and the value of the option. + try: + key, value = field.strip().split("=") + except ValueError as e: + log.warning("Failed parsing option (%s): %s", field, e) + else: + # If the parsing went good, we add the option to the + # dictionary. + options[key.strip()] = value.strip() + + return options + + def prepare(self): + # Initialize logging. + init_logging() + + # Parse the analysis configuration file generated by the agent. + self.config = Config(cfg="analysis.conf") + + # We update the target according to its category. If it's a file, then + # we store the path. + if self.config.category == "file": + self.target = os.path.join("/data/local/tmp", str(self.config.file_name)) + shutil.copyfile("config/hooks.json", "/data/local/tmp/hooks.json") + # If it's a URL, well.. we store the URL. + else: + self.target = self.config.target + + def run(self): + self.prepare() + + log.info("Starting analyzer from: {0}".format(os.getcwd())) + log.info("Storing results at: {0}".format(PATHS["root"])) + log.info("Target is: {0}".format(self.target)) + + # If no analysis package was specified at submission, we try to select + # one automatically. + if not self.config.package: + log.info("No analysis package specified, trying to detect it automagically") + # If the analysis target is a file, we choose the package according + # to the file format. + if self.config.category == "file": + package = choose_package(self.config.file_type, self.config.file_name) + # If it's an URL, we'll just use the default Internet Explorer + # package. + else: + package = "default_browser" + + # If we weren't able to automatically determine the proper package, + # we need to abort the analysis. + if not package: + raise CuckooError("No valid package available for file type: {0}".format(self.config.file_type)) + + log.info("Automatically selected analysis package \"%s\"", package) + # Otherwise just select the specified package. + else: + package = self.config.package + + # Generate the package path. + package_name = "modules.packages.%s" % package + + # Try to import the analysis package. + try: + __import__(package_name, globals(), locals(), ["dummy"], -1) + # If it fails, we need to abort the analysis. + except ImportError: + raise CuckooError("Unable to import package \"{0}\", does not exist.".format(package_name)) + + # Initialize the package parent abstract. + Package() + + # Enumerate the abstract's subclasses. + try: + package_class = Package.__subclasses__()[0] + except IndexError as e: + raise CuckooError("Unable to select package class (package={0}): {1}".format(package_name, e)) + + # Initialize the analysis package. + pack = package_class(self.get_options()) + + # Initialize Auxiliary modules + Auxiliary() + prefix = auxiliary.__name__ + "." + for loader, name, ispkg in pkgutil.iter_modules(auxiliary.__path__, prefix): + if ispkg: + continue + + # Import the auxiliary module. + try: + __import__(name, globals(), locals(), ["dummy"], -1) + except ImportError as e: + log.warning("Unable to import the auxiliary module " + "\"%s\": %s", name, e) + + # Walk through the available auxiliary modules. + aux_enabled = [] + for module in Auxiliary.__subclasses__(): + # Try to start the auxiliary module. + try: + aux = module() + aux.start() + except (NotImplementedError, AttributeError): + log.warning("Auxiliary module %s was not implemented", + aux.__class__.__name__) + continue + except Exception as e: + log.warning("Cannot execute auxiliary module %s: %s", + aux.__class__.__name__, e) + continue + finally: + log.info("Started auxiliary module %s", + aux.__class__.__name__) + aux_enabled.append(aux) + + # Start analysis package. If for any reason, the execution of the + # analysis package fails, we have to abort the analysis. + try: + pack.start(self.target) + except NotImplementedError: + raise CuckooError("The package \"{0}\" doesn't contain a run " + "function.".format(package_name)) + except CuckooPackageError as e: + raise CuckooError("The package \"{0}\" start function raised an " + "error: {1}".format(package_name, e)) + except Exception as e: + raise CuckooError("The package \"{0}\" start function encountered " + "an unhandled exception: " + "{1}".format(package_name, e)) + + time_counter = 0 + while True: + time_counter += 1 + if time_counter == int(self.config.timeout): + log.info("Analysis timeout hit, terminating analysis") + break + + try: + # The analysis packages are provided with a function that + # is executed at every loop's iteration. If such function + # returns False, it means that it requested the analysis + # to be terminate. + if not pack.check(): + log.info("The analysis package requested the " + "termination of the analysis...") + break + + # If the check() function of the package raised some exception + # we don't care, we can still proceed with the analysis but we + # throw a warning. + except Exception as e: + log.warning("The package \"%s\" check function raised " + "an exception: %s", package_name, e) + finally: + # Zzz. + time.sleep(1) + + try: + # Before shutting down the analysis, the package can perform some + # final operations through the finish() function. + pack.finish() + except Exception as e: + log.warning("The package \"%s\" finish function raised an " + "exception: %s", package_name, e) + + # Terminate the Auxiliary modules. + for aux in aux_enabled: + try: + aux.stop() + except (NotImplementedError, AttributeError): + continue + except Exception as e: + log.warning("Cannot terminate auxiliary module %s: %s", + aux.__class__.__name__, e) + + # Let's invoke the completion procedure. + self.complete() + return True + +if __name__ == "__main__": + success = False + error = "" + + try: + # Initialize the main analyzer class. + analyzer = Analyzer() + # Run it and wait for the response. + success = analyzer.run() + # This is not likely to happen. + except KeyboardInterrupt: + error = "Keyboard Interrupt" + # If the analysis process encountered a critical error, it will raise a + # CuckooError exception, which will force the termination of the analysis + # weill notify the agent of the failure. Also catched unexpected + # exceptions. + except Exception as e: + # Store the error. + error = str(e) + + # Just to be paranoid. + if len(log.handlers) > 0: + log.critical(error) + else: + sys.stderr.write("{0}\n".format(e)) + # Once the analysis is completed or terminated for any reason, we report + # back to the agent, notifying that it can report back to the host. + finally: + # Establish connection with the agent XMLRPC server. + server = xmlrpclib.Server("http://127.0.0.1:8000") + server.complete(success, error, PATHS["root"]) diff --git a/analyzer/android/config/hooks.json b/cuckoo/data/analyzer/android/config/hooks.json similarity index 100% rename from analyzer/android/config/hooks.json rename to cuckoo/data/analyzer/android/config/hooks.json diff --git a/cuckoo/data/analyzer/android/lib/__init__.py b/cuckoo/data/analyzer/android/lib/__init__.py new file mode 100644 index 0000000000..c364169724 --- /dev/null +++ b/cuckoo/data/analyzer/android/lib/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. +# Originally contributed by Check Point Software Technologies, Ltd. diff --git a/cuckoo/data/analyzer/android/lib/api/__init__.py b/cuckoo/data/analyzer/android/lib/api/__init__.py new file mode 100644 index 0000000000..c364169724 --- /dev/null +++ b/cuckoo/data/analyzer/android/lib/api/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. +# Originally contributed by Check Point Software Technologies, Ltd. diff --git a/analyzer/android/lib/api/adb.py b/cuckoo/data/analyzer/android/lib/api/adb.py similarity index 98% rename from analyzer/android/lib/api/adb.py rename to cuckoo/data/analyzer/android/lib/api/adb.py index 84d61fa256..8ec3d9d5a1 100644 --- a/analyzer/android/lib/api/adb.py +++ b/cuckoo/data/analyzer/android/lib/api/adb.py @@ -1,4 +1,4 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. +# Copyright (C) 2014-2016 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. # Originally contributed by Check Point Software Technologies, Ltd. diff --git a/cuckoo/data/analyzer/android/lib/api/screenshot.py b/cuckoo/data/analyzer/android/lib/api/screenshot.py new file mode 100644 index 0000000000..d8733d405c --- /dev/null +++ b/cuckoo/data/analyzer/android/lib/api/screenshot.py @@ -0,0 +1,53 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. +# Originally contributed by Check Point Software Technologies, Ltd. + +import math +import filecmp + +try: + import ImageChops + from PIL import Image + HAVE_PIL = True +except: + try: + from PIL import ImageChops + from PIL import Image + HAVE_PIL = True + except: + HAVE_PIL = False + +class Screenshot: + """Get screenshots.""" + + def have_pil(self): + """Is Python Image Library installed? + @return: installed status. + """ + return HAVE_PIL + + def equal_old(self, img1, img2): + """Compares two screenshots using Root-Mean-Square Difference (RMS). + @param img1: screenshot to compare. + @param img2: screenshot to compare. + @return: equal status. + """ + if not HAVE_PIL: + return None + image1 = Image.open(img1) + image2 = Image.open(img2) + # To get a measure of how similar two images are, we use + # root-mean-square (RMS). If the images are exactly identical, + # this value is zero. + diff = ImageChops.difference(image1, image2) + h = diff.histogram() + sq = (value*((idx % 256)**2) for idx, value in enumerate(h)) + sum_of_squares = sum(sq) + rms = math.sqrt(sum_of_squares/float(image1.size[0] * image2.size[1])) + + # Might need to tweak the threshold. + return rms < 8 + + def equal(self, img1, img2): + return filecmp.cmp(img1, img2) diff --git a/cuckoo/data/analyzer/android/lib/common/__init__.py b/cuckoo/data/analyzer/android/lib/common/__init__.py new file mode 100644 index 0000000000..c364169724 --- /dev/null +++ b/cuckoo/data/analyzer/android/lib/common/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. +# Originally contributed by Check Point Software Technologies, Ltd. diff --git a/cuckoo/data/analyzer/android/lib/common/abstracts.py b/cuckoo/data/analyzer/android/lib/common/abstracts.py new file mode 100644 index 0000000000..8255ad942f --- /dev/null +++ b/cuckoo/data/analyzer/android/lib/common/abstracts.py @@ -0,0 +1,40 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. +# Originally contributed by Check Point Software Technologies, Ltd. + +class Package(object): + """Base abstact analysis package.""" + + def __init__(self, options={}): + """@param options: options dict.""" + self.options = options + self.pids = [] + + def set_pids(self, pids): + """Update list of monitored PIDs in the package context. + @param pids: list of pids. + """ + self.pids = pids + + def start(self): + """Run analysis package. + @param path: sample path. + @raise NotImplementedError: this method is abstract. + """ + raise NotImplementedError + + def check(self): + """Check. + @raise NotImplementedError: this method is abstract. + """ + raise NotImplementedError + + def finish(self): + """Finish run. + @raise NotImplementedError: this method is abstract. + """ + raise NotImplementedError + +class Auxiliary(object): + pass diff --git a/cuckoo/data/analyzer/android/lib/common/constants.py b/cuckoo/data/analyzer/android/lib/common/constants.py new file mode 100644 index 0000000000..7cd235c9e1 --- /dev/null +++ b/cuckoo/data/analyzer/android/lib/common/constants.py @@ -0,0 +1,22 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. +# Originally contributed by Check Point Software Technologies, Ltd. + +import os +import string +import random + +def _rand_string(a, b): + return "".join(random.choice(string.ascii_lowercase) for x in xrange(random.randint(a, b))) + +ROOT = os.path.join("/data/local/tmp", _rand_string(6, 10)) + +PATHS = { + "root" : ROOT, + "logs" : os.path.join(ROOT, "logs"), + "files" : os.path.join(ROOT, "files"), + "shots" : os.path.join(ROOT, "shots"), + "memory" : os.path.join(ROOT, "memory"), + "drop" : os.path.join(ROOT, "drop") +} diff --git a/cuckoo/data/analyzer/android/lib/common/exceptions.py b/cuckoo/data/analyzer/android/lib/common/exceptions.py new file mode 100644 index 0000000000..447af1d782 --- /dev/null +++ b/cuckoo/data/analyzer/android/lib/common/exceptions.py @@ -0,0 +1,10 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. +# Originally contributed by Check Point Software Technologies, Ltd. + +class CuckooError(Exception): + pass + +class CuckooPackageError(Exception): + pass diff --git a/cuckoo/data/analyzer/android/lib/common/results.py b/cuckoo/data/analyzer/android/lib/common/results.py new file mode 100644 index 0000000000..7cf0276d84 --- /dev/null +++ b/cuckoo/data/analyzer/android/lib/common/results.py @@ -0,0 +1,93 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. +# Originally contributed by Check Point Software Technologies, Ltd. + +import logging +import socket +import time + +from lib.core.config import Config + +log = logging.getLogger(__name__) + +BUFSIZE = 1024*1024 + +def upload_to_host(file_path, dump_path): + nc = infd = None + try: + nc = NetlogFile(dump_path) + + infd = open(file_path, "rb") + buf = infd.read(BUFSIZE) + while buf: + nc.send(buf, retry=False) + buf = infd.read(BUFSIZE) + except Exception as e: + log.error("Exception uploading file %s to host: %s", file_path, e) + finally: + if infd: + infd.close() + if nc: + nc.close() + +class NetlogConnection(object): + def __init__(self, proto=""): + config = Config(cfg="analysis.conf") + self.hostip, self.hostport = config.ip, config.port + self.sock = None + self.proto = proto + + def connect(self): + # Try to connect as quickly as possible. Just sort of force it to + # connect with a short timeout. + while not self.sock: + try: + s = socket.create_connection((self.hostip, self.hostport), 0.1) + s.sendall(self.proto) + except socket.error: + time.sleep(0.1) + continue + + self.sock = s + + def send(self, data, retry=True): + if not self.sock: + self.connect() + + try: + self.sock.sendall(data) + except socket.error as e: + if retry: + self.connect() + self.send(data, retry=False) + else: + raise + except Exception as e: + log.error("Unhandled exception in NetlogConnection: %s", str(e)) + # We really have nowhere to log this, if the netlog connection + # does not work, we can assume that any logging won't work either. + # So we just fail silently. + self.close() + + def close(self): + try: + self.sock.close() + except Exception: + pass + +class NetlogFile(NetlogConnection): + def __init__(self, filepath): + self.filepath = filepath + NetlogConnection.__init__(self, proto="FILE\n{0}\n".format(self.filepath)) + self.connect() + +class NetlogHandler(logging.Handler, NetlogConnection): + def __init__(self): + logging.Handler.__init__(self) + NetlogConnection.__init__(self, proto="LOG\n") + self.connect() + + def emit(self, record): + msg = self.format(record) + self.send("{0}\n".format(msg)) diff --git a/cuckoo/data/analyzer/android/lib/common/utils.py b/cuckoo/data/analyzer/android/lib/common/utils.py new file mode 100644 index 0000000000..303ce3d85d --- /dev/null +++ b/cuckoo/data/analyzer/android/lib/common/utils.py @@ -0,0 +1,12 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. +# Originally contributed by Check Point Software Technologies, Ltd. + +from lib.common.results import NetlogFile + +def send_file(name, data): + """Send file to result server""" + nf = NetlogFile(name) + nf.sock.sendall(data) + nf.close() diff --git a/cuckoo/data/analyzer/android/lib/core/__init__.py b/cuckoo/data/analyzer/android/lib/core/__init__.py new file mode 100644 index 0000000000..c364169724 --- /dev/null +++ b/cuckoo/data/analyzer/android/lib/core/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. +# Originally contributed by Check Point Software Technologies, Ltd. diff --git a/cuckoo/data/analyzer/android/lib/core/config.py b/cuckoo/data/analyzer/android/lib/core/config.py new file mode 100644 index 0000000000..b7d3031cbb --- /dev/null +++ b/cuckoo/data/analyzer/android/lib/core/config.py @@ -0,0 +1,24 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. +# Originally contributed by Check Point Software Technologies, Ltd. + +import ConfigParser + +class Config: + def __init__(self, cfg): + """@param cfg: configuration file.""" + config = ConfigParser.ConfigParser(allow_no_value=True) + config.read(cfg) + + for section in config.sections(): + for name, raw_value in config.items(section): + try: + value = config.getboolean(section, name) + except ValueError: + try: + value = config.getint(section, name) + except ValueError: + value = config.get(section, name) + + setattr(self, name, value) diff --git a/cuckoo/data/analyzer/android/lib/core/packages.py b/cuckoo/data/analyzer/android/lib/core/packages.py new file mode 100644 index 0000000000..9f872b5da6 --- /dev/null +++ b/cuckoo/data/analyzer/android/lib/core/packages.py @@ -0,0 +1,24 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. +# Originally contributed by Check Point Software Technologies, Ltd. + +def choose_package(file_type, file_name): + """Choose analysis package due to file type and file extension. + @param file_type: file type. + @return: package or None. + """ + if not file_type: + return None + + file_type = file_type.lower() + file_name = file_name.lower() + + if "apk" in file_name: + return "apk" + elif "zip" in file_type: + return "apk" + # elif "DEX" in file_type: + # return "dex" + else: + return "apk" diff --git a/cuckoo/data/analyzer/android/lib/core/startup.py b/cuckoo/data/analyzer/android/lib/core/startup.py new file mode 100644 index 0000000000..ab8f7ecfdd --- /dev/null +++ b/cuckoo/data/analyzer/android/lib/core/startup.py @@ -0,0 +1,36 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. +# Originally contributed by Check Point Software Technologies, Ltd. + +import os +import logging + +from lib.common.constants import PATHS +from lib.common.results import NetlogHandler + +log = logging.getLogger() + +def create_folders(): + """Create folders in PATHS.""" + for name, folder in PATHS.items(): + if os.path.exists(folder): + continue + + try: + os.makedirs(folder) + except OSError: + pass + +def init_logging(): + """Initialize logger.""" + formatter = logging.Formatter("%(asctime)s [%(name)s] %(levelname)s: %(message)s") + sh = logging.StreamHandler() + sh.setFormatter(formatter) + log.addHandler(sh) + + nh = NetlogHandler() + nh.setFormatter(formatter) + log.addHandler(nh) + + log.setLevel(logging.DEBUG) diff --git a/cuckoo/data/analyzer/android/modules/__init__.py b/cuckoo/data/analyzer/android/modules/__init__.py new file mode 100644 index 0000000000..c364169724 --- /dev/null +++ b/cuckoo/data/analyzer/android/modules/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. +# Originally contributed by Check Point Software Technologies, Ltd. diff --git a/cuckoo/data/analyzer/android/modules/auxiliary/__init__.py b/cuckoo/data/analyzer/android/modules/auxiliary/__init__.py new file mode 100644 index 0000000000..c364169724 --- /dev/null +++ b/cuckoo/data/analyzer/android/modules/auxiliary/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. +# Originally contributed by Check Point Software Technologies, Ltd. diff --git a/cuckoo/data/analyzer/android/modules/auxiliary/screenshots.py b/cuckoo/data/analyzer/android/modules/auxiliary/screenshots.py new file mode 100644 index 0000000000..7c1bae450c --- /dev/null +++ b/cuckoo/data/analyzer/android/modules/auxiliary/screenshots.py @@ -0,0 +1,63 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. +# Originally contributed by Check Point Software Technologies, Ltd. + +import time +import logging +import StringIO +from threading import Thread +from lib.common.abstracts import Auxiliary +from lib.common.results import NetlogFile +from lib.api.adb import take_screenshot +from lib.api.screenshot import Screenshot + +log = logging.getLogger(__name__) +SHOT_DELAY = 2 + +class Screenshots(Auxiliary, Thread): + """Take screenshots.""" + + def __init__(self): + Thread.__init__(self) + self.do_run = True + + def stop(self): + """Stop screenshotting.""" + self.do_run = False + + def run(self): + """Run screenshotting. + @return: operation status. + """ + img_counter = 0 + img_last = None + + while self.do_run: + time.sleep(SHOT_DELAY) + + try: + filename = "screenshot%s.jpg" % str(img_counter) + img_current = take_screenshot(filename) + if img_last: + if Screenshot().equal(img_last, img_current): + continue + + file = open(img_current, 'r') + tmpio = StringIO.StringIO(file.read()) + # now upload to host from the StringIO + nf = NetlogFile("shots/%s.jpg" % str(img_counter).rjust(4, "0")) + + for chunk in tmpio: + nf.sock.sendall(chunk) + + nf.close() + file.close() + img_counter += 1 + img_last = img_current + + except IOError as e: + log.error("Cannot take screenshot: %s", e) + continue + + return True diff --git a/cuckoo/data/analyzer/android/modules/packages/__init__.py b/cuckoo/data/analyzer/android/modules/packages/__init__.py new file mode 100644 index 0000000000..c364169724 --- /dev/null +++ b/cuckoo/data/analyzer/android/modules/packages/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. +# Originally contributed by Check Point Software Technologies, Ltd. diff --git a/analyzer/android/modules/packages/apk.py b/cuckoo/data/analyzer/android/modules/packages/apk.py similarity index 94% rename from analyzer/android/modules/packages/apk.py rename to cuckoo/data/analyzer/android/modules/packages/apk.py index c84b0fc4ef..5a3da9da04 100644 --- a/analyzer/android/modules/packages/apk.py +++ b/cuckoo/data/analyzer/android/modules/packages/apk.py @@ -1,4 +1,4 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. +# Copyright (C) 2014-2016 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. # Originally contributed by Check Point Software Technologies, Ltd. diff --git a/analyzer/android/modules/packages/default_browser.py b/cuckoo/data/analyzer/android/modules/packages/default_browser.py similarity index 92% rename from analyzer/android/modules/packages/default_browser.py rename to cuckoo/data/analyzer/android/modules/packages/default_browser.py index 9f69c8d3af..b9d6875a20 100644 --- a/analyzer/android/modules/packages/default_browser.py +++ b/cuckoo/data/analyzer/android/modules/packages/default_browser.py @@ -1,4 +1,4 @@ -# Copyright (C) 2010-2015 Cuckoo Foundation. +# Copyright (C) 2014-2016 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. # Originally contributed by Check Point Software Technologies, Ltd. diff --git a/cuckoo/data/analyzer/darwin/analyzer.py b/cuckoo/data/analyzer/darwin/analyzer.py new file mode 100644 index 0000000000..80fc6c1562 --- /dev/null +++ b/cuckoo/data/analyzer/darwin/analyzer.py @@ -0,0 +1,220 @@ +# Copyright (C) 2015-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import hashlib +import logging +import os +import pkgutil +import sys +import traceback +import urllib +import urllib2 +import xmlrpclib + +from lib.common.config import Config +from lib.common.hashing import hash_file +from lib.common.results import NetlogHandler, upload_to_host +from lib.core.constants import PATHS +from lib.core.packages import choose_package_class +from lib.core.osx import set_wallclock +from lib.core.host import CuckooHost +from lib.common.abstracts import Auxiliary +from lib.common.exceptions import CuckooDisableModule +from modules import auxiliary + +log = logging.getLogger("analyzer") + +class Macalyzer(object): + """Cuckoo OS X analyser. + """ + + log = logging.getLogger() + target = None + + files_to_upload = [] + uploaded_hashes = [] + + def __init__(self, host, configuration=None): + self.config = configuration + self.host = host + + def bootstrap(self): + _create_result_folders() + _setup_logging() + self._detect_target() + + def run(self): + """Run analysis. + """ + self.bootstrap() + + self.log.debug("Starting analyzer from %s", os.getcwd()) + self.log.debug("Storing results at: %s", PATHS["root"]) + + package = self._setup_analysis_package() + + if self.config.clock: + set_wallclock(self.config.clock) + + # Initialize Auxiliary modules + Auxiliary() + prefix = auxiliary.__name__ + "." + for loader, name, ispkg in pkgutil.iter_modules(auxiliary.__path__, prefix): + if ispkg: + continue + + # Import the auxiliary module. + try: + __import__(name, globals(), locals(), ["dummy"], -1) + except ImportError as e: + log.warning("Unable to import the auxiliary module " + "\"%s\": %s", name, e) + + # Walk through the available auxiliary modules. + aux_enabled, aux_avail = [], [] + for module in Auxiliary.__subclasses__(): + # Try to start the auxiliary module. + try: + aux = module(options=self.config.options, analyzer=self) + aux_avail.append(aux) + aux.start() + except (NotImplementedError, AttributeError): + log.warning("Auxiliary module %s was not implemented", + module.__name__) + except CuckooDisableModule: + continue + except Exception as e: + log.warning("Cannot execute auxiliary module %s: %s", + module.__name__, e) + else: + log.debug("Started auxiliary module %s", + module.__name__) + aux_enabled.append(aux) + + self._analysis(package) + + return self._complete() + + def _complete(self): + for f in self.files_to_upload: + self._upload_file(f) + return True + + # + # Implementation details + # + + def _detect_target(self): + if self.config.category == "file": + self.target = os.path.join("/tmp/", str(self.config.file_name)) + else: # It's not a file, but a URL + self.target = self.config.target + + def _setup_analysis_package(self): + # Do we have a suggestion about an analysis package? + if self.config.package: + suggestion = self.config.package + elif self.config.category != "file": + suggestion = "url" + else: + suggestion = None + # Try to figure out what analysis package to use with this target + kwargs = {"suggestion" : suggestion} + package_class = choose_package_class(self.config.file_type, + self.config.file_name, **kwargs) + if not package_class: + raise Exception("Could not find an appropriate analysis package") + # Package initialization + kwargs = { + "options" : self.config.get_options(), + "timeout" : self.config.timeout + } + return package_class(self.target, self.host, **kwargs) + + def _analysis(self, package): + package.start() + self.files_to_upload = package.touched_files + + def _upload_file(self, filepath): + if not os.path.isfile(filepath): + return + # Check whether we've already dumped this file - in that case skip it + try: + hashsum = hash_file(hashlib.sha256, filepath) + if sha256 in self.uploaded_hashes: + return + except IOError as e: + self.log.info("Error dumping file from path \"%s\": %s", filepath, e) + return + filename = "%s_%s" % (hashsum[:16], os.path.basename(filepath)) + upload_path = os.path.join("files", filename) + + try: + upload_to_host(filepath, upload_path) + self.uploaded_hashes.append(hashsum) + except IOError as e: + self.log.error("Unable to upload dropped file at path \"%s\": %s", filepath, e) + +def _create_result_folders(): + for _, folder in PATHS.items(): + if os.path.exists(folder): + continue + try: + os.makedirs(folder) + except OSError: + pass + + +def _setup_logging(): + """ Initialize logger. """ + logger = logging.getLogger() + formatter = logging.Formatter("%(asctime)s [%(name)s] %(levelname)s: %(message)s") + + stream = logging.StreamHandler() + stream.setFormatter(formatter) + logger.addHandler(stream) + + netlog = NetlogHandler() + netlog.setFormatter(formatter) + logger.addHandler(netlog) + logger.setLevel(logging.DEBUG) + + + +if __name__ == "__main__": + success = False + error = "" + + try: + config = Config(cfg="analysis.conf") + cuckoo = CuckooHost(config.ip, config.port) + analyzer = Macalyzer(cuckoo, config) + success = analyzer.run() + + except KeyboardInterrupt: + error = "Keyboard Interrupt" + + except Exception as err: + error_exc = traceback.format_exc() + error = str(err) + if len(analyzer.log.handlers): + analyzer.log.exception(error_exc) + else: + sys.stderr.write("{0}\n".format(error_exc)) + # Once the analysis is completed or terminated for any reason, we report + # back to the agent, notifying that it can report back to the host. + finally: + try: + # Establish connection with the agent XMLRPC server. + server = xmlrpclib.Server("http://127.0.0.1:8000") + server.complete(success, error, PATHS["root"]) + except Exception as e: + # new agent + data = { + "status": "complete", + "description": success + } + urllib2.urlopen( + "http://127.0.0.1:8000/status", urllib.urlencode(data) + ) diff --git a/analyzer/__init__.py b/cuckoo/data/analyzer/darwin/lib/__init__.py similarity index 100% rename from analyzer/__init__.py rename to cuckoo/data/analyzer/darwin/lib/__init__.py diff --git a/cuckoo/data/analyzer/darwin/lib/api/__init__.py b/cuckoo/data/analyzer/darwin/lib/api/__init__.py new file mode 100644 index 0000000000..6b3e0da70a --- /dev/null +++ b/cuckoo/data/analyzer/darwin/lib/api/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2017-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. diff --git a/cuckoo/data/analyzer/darwin/lib/api/screenshot.py b/cuckoo/data/analyzer/darwin/lib/api/screenshot.py new file mode 100644 index 0000000000..613551fd07 --- /dev/null +++ b/cuckoo/data/analyzer/darwin/lib/api/screenshot.py @@ -0,0 +1,81 @@ +# Copyright (C) 2015-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import math + +try: + import ImageChops + import ImageGrab + import ImageDraw + HAVE_PIL = True +except: + try: + from PIL import ImageChops + from PIL import ImageGrab + from PIL import ImageDraw + HAVE_PIL = True + except: + HAVE_PIL = False + +class Screenshot: + """Get screenshots.""" + + def _draw_rectangle(self, img, xy): + """Draw a black rectangle. + @param img: PIL Image object + @param xy: Coordinates as refined in PIL rectangle() doc + @return: Image with black rectangle + """ + dr = ImageDraw.Draw(img) + dr.rectangle(xy, fill="black", outline="black") + return img + + def have_pil(self): + """Is Python Image Library installed? + @return: installed status. + """ + return HAVE_PIL + + def equal(self, img1, img2, skip_area=None): + """Compare two screenshots using Root-Mean-Square Difference (RMS). + @param img1: screenshot to compare. + @param img2: screenshot to compare. + @return: equal status. + """ + if not HAVE_PIL: + return None + + # Trick to avoid getting a lot of screen shots only because the time in the windows + # clock is changed. + # We draw a black rectangle on the coordinates where the clock is locates, and then + # run the comparison. + # NOTE: the coordinates are changing with VM screen resolution. + if skip_area: + # Copying objects to draw in another object. + img1 = img1.copy() + img2 = img2.copy() + # Draw a rectangle to cover windows clock. + for img in (img1, img2): + self._draw_rectangle(img, skip_area) + + # To get a measure of how similar two images are, we use + # root-mean-square (RMS). If the images are exactly identical, + # this value is zero. + diff = ImageChops.difference(img1, img2) + h = diff.histogram() + sq = (value * ((idx % 256)**2) for idx, value in enumerate(h)) + sum_of_squares = sum(sq) + rms = math.sqrt(sum_of_squares/float(img1.size[0] * img1.size[1])) + + # Might need to tweak the threshold. + return rms < 8 + + def take(self): + """Take a screenshot. + @return: screenshot or None. + """ + if not HAVE_PIL: + return None + + return ImageGrab.grab() diff --git a/cuckoo/data/analyzer/darwin/lib/common/__init__.py b/cuckoo/data/analyzer/darwin/lib/common/__init__.py new file mode 100644 index 0000000000..7b600d0859 --- /dev/null +++ b/cuckoo/data/analyzer/darwin/lib/common/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. diff --git a/cuckoo/data/analyzer/darwin/lib/common/abstracts.py b/cuckoo/data/analyzer/darwin/lib/common/abstracts.py new file mode 100644 index 0000000000..a38f2cd208 --- /dev/null +++ b/cuckoo/data/analyzer/darwin/lib/common/abstracts.py @@ -0,0 +1,9 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +class Auxiliary(object): + def __init__(self, options={}, analyzer=None): + self.options = options + self.analyzer = analyzer diff --git a/cuckoo/data/analyzer/darwin/lib/common/config.py b/cuckoo/data/analyzer/darwin/lib/common/config.py new file mode 100644 index 0000000000..841e7ad979 --- /dev/null +++ b/cuckoo/data/analyzer/darwin/lib/common/config.py @@ -0,0 +1,68 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import ConfigParser + +class Config: + def __init__(self, cfg): + """@param cfg: configuration file.""" + config = ConfigParser.ConfigParser(allow_no_value=True) + config.read(cfg) + + for section in config.sections(): + for name, raw_value in config.items(section): + if name == "file_name": + value = config.get(section, name) + else: + try: + value = config.getboolean(section, name) + except ValueError: + try: + value = config.getint(section, name) + except ValueError: + value = config.get(section, name) + setattr(self, name, value) + + def get_options(self): + """Get analysis options. + @return: options dict. + """ + # The analysis package can be provided with some options in the + # following format: + # option1=value1,option2=value2,option3=value3 + # + # Here we parse such options and provide a dictionary that will be made + # accessible to the analysis package. + options = {} + if hasattr(self, "options") and len(self.options) > 0: + try: + # Split the options by comma. + fields = self.options.split(",") + except ValueError: + pass + else: + for field in fields: + # Split the name and the value of the option. + try: + # Sometimes, we have a key without a value (i.e. it's a + # command line argument), so we can't use the + # `key, value = field.split("=", 1)` style here + parts = field.split("=", 1) + except ValueError: + pass + else: + key = parts[0].strip() + arg_prefix = "arg-" + if not key.startswith(arg_prefix): + # If the parsing went good, we add the option to the + # dictionary. + value = parts[1].strip() + options[key] = value + elif len(key) > len(arg_prefix): + # Remove "arg-" prefix from the key + key = key[4:]; parts[0] = key + # Add this key (with a value maybe) to the args + if "args" not in options: options["args"] = [] + options["args"] += parts + return options diff --git a/cuckoo/data/analyzer/darwin/lib/common/exceptions.py b/cuckoo/data/analyzer/darwin/lib/common/exceptions.py new file mode 100644 index 0000000000..c3e74767a9 --- /dev/null +++ b/cuckoo/data/analyzer/darwin/lib/common/exceptions.py @@ -0,0 +1,13 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +class CuckooError(Exception): + pass + +class CuckooPackageError(Exception): + pass + +class CuckooDisableModule(CuckooError): + """Exception for disabling a module dynamically.""" diff --git a/cuckoo/data/analyzer/darwin/lib/common/hashing.py b/cuckoo/data/analyzer/darwin/lib/common/hashing.py new file mode 100644 index 0000000000..010d4fa11c --- /dev/null +++ b/cuckoo/data/analyzer/darwin/lib/common/hashing.py @@ -0,0 +1,21 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +BUFSIZE = 1024*1024 + + +def hash_file(method, path): + """Calculate a hash on a file by path. + @param method: callable hashing method + @param path: file path + @return: computed hash string + """ + f = open(path, "rb") + h = method() + while True: + buf = f.read(BUFSIZE) + if not buf: + break + h.update(buf) + return h.hexdigest() diff --git a/analyzer/darwin/lib/common/rand.py b/cuckoo/data/analyzer/darwin/lib/common/rand.py similarity index 100% rename from analyzer/darwin/lib/common/rand.py rename to cuckoo/data/analyzer/darwin/lib/common/rand.py diff --git a/cuckoo/data/analyzer/darwin/lib/common/results.py b/cuckoo/data/analyzer/darwin/lib/common/results.py new file mode 100644 index 0000000000..53f9fb05cc --- /dev/null +++ b/cuckoo/data/analyzer/darwin/lib/common/results.py @@ -0,0 +1,94 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import time +import socket +import logging +from config import Config + +log = logging.getLogger(__name__) + +BUFSIZE = 1024*1024 + +def upload_to_host(file_path, dump_path): + nc = infd = None + try: + nc = NetlogFile(dump_path) + + infd = open(file_path, "rb") + buf = infd.read(BUFSIZE) + while buf: + nc.send(buf, retry=False) + buf = infd.read(BUFSIZE) + except Exception as e: + log.error("Exception uploading file %s to host: %s", file_path, e) + finally: + if infd: + infd.close() + if nc: + nc.close() + +class NetlogConnection(object): + def __init__(self, proto=""): + config = Config(cfg="analysis.conf") + self.hostip, self.hostport = config.ip, config.port + self.sock, self.file = None, None + self.proto = proto + + def connect(self): + i = 1 + # this can loop forever, if we can't connect the whole analysis is useless anyways + while True: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + s.connect((self.hostip, self.hostport)) + s.sendall(self.proto) + except: + time.sleep(i) + i = min(i + 1, 60) + else: + self.sock = s + self.file = s.makefile() + break + + def send(self, data, retry=True): + if not self.sock: self.connect() + + try: + self.sock.sendall(data) + except socket.error as e: + if retry: + self.connect() + self.send(data, retry=False) + else: + raise + except Exception as e: + log.error("Unhandled exception in NetlogConnection: %s", str(e)) + # We really have nowhere to log this, if the netlog connection + # does not work, we can assume that any logging won't work either. + # So we just fail silently. + self.close() + + def close(self): + try: + self.file.close() + self.sock.close() + except Exception: + pass + +class NetlogFile(NetlogConnection): + def __init__(self, filepath): + self.filepath = filepath + NetlogConnection.__init__(self, proto="FILE\n{0}\n".format(self.filepath)) + self.connect() + +class NetlogHandler(logging.Handler, NetlogConnection): + def __init__(self): + logging.Handler.__init__(self) + NetlogConnection.__init__(self, proto="LOG\n") + self.connect() + + def emit(self, record): + msg = self.format(record) + self.send("{0}\n".format(msg)) diff --git a/analyzer/darwin/__init__.py b/cuckoo/data/analyzer/darwin/lib/core/__init__.py similarity index 100% rename from analyzer/darwin/__init__.py rename to cuckoo/data/analyzer/darwin/lib/core/__init__.py diff --git a/cuckoo/data/analyzer/darwin/lib/core/constants.py b/cuckoo/data/analyzer/darwin/lib/core/constants.py new file mode 100644 index 0000000000..824a1a65ae --- /dev/null +++ b/cuckoo/data/analyzer/darwin/lib/core/constants.py @@ -0,0 +1,18 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import os +from tempfile import gettempdir +from ..common.rand import random_string + +ROOT = os.path.join(gettempdir() + os.sep, random_string(6, 10)) + +PATHS = { + "root" : ROOT, + "logs" : os.path.join(ROOT, "logs"), + "files" : os.path.join(ROOT, "files"), + "shots" : os.path.join(ROOT, "shots"), + "memory" : os.path.join(ROOT, "memory"), + "drop" : os.path.join(ROOT, "drop") +} diff --git a/analyzer/darwin/lib/core/data/signatures.yml b/cuckoo/data/analyzer/darwin/lib/core/data/signatures.yml similarity index 100% rename from analyzer/darwin/lib/core/data/signatures.yml rename to cuckoo/data/analyzer/darwin/lib/core/data/signatures.yml diff --git a/analyzer/darwin/lib/core/data/types.yml b/cuckoo/data/analyzer/darwin/lib/core/data/types.yml similarity index 100% rename from analyzer/darwin/lib/core/data/types.yml rename to cuckoo/data/analyzer/darwin/lib/core/data/types.yml diff --git a/analyzer/darwin/lib/core/filetimes.py b/cuckoo/data/analyzer/darwin/lib/core/filetimes.py similarity index 100% rename from analyzer/darwin/lib/core/filetimes.py rename to cuckoo/data/analyzer/darwin/lib/core/filetimes.py diff --git a/analyzer/darwin/lib/core/host.py b/cuckoo/data/analyzer/darwin/lib/core/host.py similarity index 100% rename from analyzer/darwin/lib/core/host.py rename to cuckoo/data/analyzer/darwin/lib/core/host.py diff --git a/analyzer/darwin/lib/core/osx.py b/cuckoo/data/analyzer/darwin/lib/core/osx.py similarity index 100% rename from analyzer/darwin/lib/core/osx.py rename to cuckoo/data/analyzer/darwin/lib/core/osx.py diff --git a/analyzer/darwin/lib/core/packages.py b/cuckoo/data/analyzer/darwin/lib/core/packages.py similarity index 100% rename from analyzer/darwin/lib/core/packages.py rename to cuckoo/data/analyzer/darwin/lib/core/packages.py diff --git a/analyzer/darwin/lib/__init__.py b/cuckoo/data/analyzer/darwin/lib/dtrace/__init__.py similarity index 100% rename from analyzer/darwin/lib/__init__.py rename to cuckoo/data/analyzer/darwin/lib/dtrace/__init__.py diff --git a/analyzer/darwin/lib/dtrace/apicalls.d b/cuckoo/data/analyzer/darwin/lib/dtrace/apicalls.d similarity index 100% rename from analyzer/darwin/lib/dtrace/apicalls.d rename to cuckoo/data/analyzer/darwin/lib/dtrace/apicalls.d diff --git a/analyzer/darwin/lib/dtrace/apicalls.py b/cuckoo/data/analyzer/darwin/lib/dtrace/apicalls.py similarity index 100% rename from analyzer/darwin/lib/dtrace/apicalls.py rename to cuckoo/data/analyzer/darwin/lib/dtrace/apicalls.py diff --git a/analyzer/darwin/lib/dtrace/autoprobes.py b/cuckoo/data/analyzer/darwin/lib/dtrace/autoprobes.py similarity index 100% rename from analyzer/darwin/lib/dtrace/autoprobes.py rename to cuckoo/data/analyzer/darwin/lib/dtrace/autoprobes.py diff --git a/analyzer/darwin/lib/dtrace/common.py b/cuckoo/data/analyzer/darwin/lib/dtrace/common.py similarity index 100% rename from analyzer/darwin/lib/dtrace/common.py rename to cuckoo/data/analyzer/darwin/lib/dtrace/common.py diff --git a/analyzer/darwin/lib/dtrace/dtruss.py b/cuckoo/data/analyzer/darwin/lib/dtrace/dtruss.py similarity index 98% rename from analyzer/darwin/lib/dtrace/dtruss.py rename to cuckoo/data/analyzer/darwin/lib/dtrace/dtruss.py index 73f3f484c2..e829dc5e7c 100644 --- a/analyzer/darwin/lib/dtrace/dtruss.py +++ b/cuckoo/data/analyzer/darwin/lib/dtrace/dtruss.py @@ -6,8 +6,6 @@ import os import json from getpass import getuser -from time import sleep -from sys import argv from collections import namedtuple from subprocess import Popen from tempfile import NamedTemporaryFile diff --git a/analyzer/darwin/lib/dtrace/dtruss.sh b/cuckoo/data/analyzer/darwin/lib/dtrace/dtruss.sh similarity index 100% rename from analyzer/darwin/lib/dtrace/dtruss.sh rename to cuckoo/data/analyzer/darwin/lib/dtrace/dtruss.sh diff --git a/analyzer/darwin/lib/dtrace/follow_children.d b/cuckoo/data/analyzer/darwin/lib/dtrace/follow_children.d similarity index 100% rename from analyzer/darwin/lib/dtrace/follow_children.d rename to cuckoo/data/analyzer/darwin/lib/dtrace/follow_children.d diff --git a/analyzer/darwin/lib/dtrace/ipconnections.d b/cuckoo/data/analyzer/darwin/lib/dtrace/ipconnections.d similarity index 100% rename from analyzer/darwin/lib/dtrace/ipconnections.d rename to cuckoo/data/analyzer/darwin/lib/dtrace/ipconnections.d diff --git a/analyzer/darwin/lib/dtrace/ipconnections.py b/cuckoo/data/analyzer/darwin/lib/dtrace/ipconnections.py similarity index 100% rename from analyzer/darwin/lib/dtrace/ipconnections.py rename to cuckoo/data/analyzer/darwin/lib/dtrace/ipconnections.py diff --git a/analyzer/darwin/lib/core/__init__.py b/cuckoo/data/analyzer/darwin/modules/__init__.py similarity index 100% rename from analyzer/darwin/lib/core/__init__.py rename to cuckoo/data/analyzer/darwin/modules/__init__.py diff --git a/analyzer/darwin/lib/dtrace/__init__.py b/cuckoo/data/analyzer/darwin/modules/auxiliary/__init__.py similarity index 100% rename from analyzer/darwin/lib/dtrace/__init__.py rename to cuckoo/data/analyzer/darwin/modules/auxiliary/__init__.py diff --git a/cuckoo/data/analyzer/darwin/modules/auxiliary/screenshots.py b/cuckoo/data/analyzer/darwin/modules/auxiliary/screenshots.py new file mode 100644 index 0000000000..9370faee07 --- /dev/null +++ b/cuckoo/data/analyzer/darwin/modules/auxiliary/screenshots.py @@ -0,0 +1,79 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import time +import logging +import StringIO +from threading import Thread + +from lib.common.abstracts import Auxiliary +from lib.common.results import NetlogFile +from lib.api.screenshot import Screenshot + +log = logging.getLogger(__name__) +SHOT_DELAY = 1 +# Skip the following area when comparing screen shots. +# Example for 800x600 screen resolution. +# SKIP_AREA = ((735, 575), (790, 595)) +SKIP_AREA = None + +class Screenshots(Auxiliary, Thread): + """Take screenshots.""" + + def __init__(self, options={}, analyzer=None): + Thread.__init__(self) + Auxiliary.__init__(self, options, analyzer) + self.do_run = True + + def stop(self): + """Stop screenshotting.""" + self.do_run = False + + def run(self): + """Run screenshotting. + @return: operation status. + """ + if "screenshots" in self.options: + self.do_run = int(self.options["screenshots"]) + + if not Screenshot().have_pil(): + log.warning("Python Image Library is not installed, " + "screenshots are disabled") + return False + + img_counter = 0 + img_last = None + + while self.do_run: + time.sleep(SHOT_DELAY) + + try: + img_current = Screenshot().take() + except IOError as e: + log.error("Cannot take screenshot: %s", e) + continue + + if img_last: + if Screenshot().equal(img_last, img_current, SKIP_AREA): + continue + + img_counter += 1 + + # workaround as PIL can't write to the socket file object :( + tmpio = StringIO.StringIO() + img_current.save(tmpio, format="PNG") + tmpio.seek(0) + + # now upload to host from the StringIO + nf = NetlogFile("shots/%s.png" % str(img_counter).rjust(4, "0")) + + for chunk in tmpio: + nf.sock.sendall(chunk) + + nf.close() + + img_last = img_current + + return True diff --git a/analyzer/darwin/modules/__init__.py b/cuckoo/data/analyzer/darwin/modules/packages/__init__.py similarity index 100% rename from analyzer/darwin/modules/__init__.py rename to cuckoo/data/analyzer/darwin/modules/packages/__init__.py diff --git a/analyzer/darwin/modules/packages/app.py b/cuckoo/data/analyzer/darwin/modules/packages/app.py similarity index 100% rename from analyzer/darwin/modules/packages/app.py rename to cuckoo/data/analyzer/darwin/modules/packages/app.py diff --git a/analyzer/darwin/modules/packages/bash.py b/cuckoo/data/analyzer/darwin/modules/packages/bash.py similarity index 100% rename from analyzer/darwin/modules/packages/bash.py rename to cuckoo/data/analyzer/darwin/modules/packages/bash.py diff --git a/analyzer/darwin/modules/packages/macho.py b/cuckoo/data/analyzer/darwin/modules/packages/macho.py similarity index 100% rename from analyzer/darwin/modules/packages/macho.py rename to cuckoo/data/analyzer/darwin/modules/packages/macho.py diff --git a/cuckoo/data/analyzer/darwin/modules/packages/zip.py b/cuckoo/data/analyzer/darwin/modules/packages/zip.py new file mode 100644 index 0000000000..18388c8a03 --- /dev/null +++ b/cuckoo/data/analyzer/darwin/modules/packages/zip.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python +# Copyright (C) 2015 Dmitry Rodionov +# This software may be modified and distributed under the terms +# of the MIT license. See the LICENSE file for details. + +import logging +from shutil import move +from os import path, environ +from random import SystemRandom +from string import ascii_letters +from subprocess import check_output +from zipfile import ZipFile, BadZipfile +from lib.core.packages import Package, choose_package_class + +log = logging.getLogger(__name__) + +class Zip(Package): + + real_package = None + + def prepare(self): + password = self.options.get("password") + files = self._extract(self.target, password) + if not files or len(files) == 0: + raise Exception("Invalid (or empty) zip archive: %s" % self.target) + # Look for a file to analyse + target_name = self.options.get("file") + if not target_name: + # If no file name is provided via option, take the first file + target_name = files[0] + log.debug("Missing file option, auto executing: %s", target_name) + + filepath = path.join(environ.get("TEMP", "/tmp"), target_name) + # Remove the trailing slash (if any) + if filepath.endswith("/"): + self.target = filepath[:-1] + else: + self.target = filepath + + # Since we don't know what kind of file we're going to analyse, let's + # detect it automatically and create an appropriate analysis package + # for this file + file_info = _fileinfo(self.target) + pkg_class = choose_package_class(file_info, target_name) + + if not pkg_class: + raise Exception("Unable to detect analysis package for the file %s" % target_name) + else: + log.debug("Analysing file \"%s\" using package \"%s\"", target_name, pkg_class.__name__) + + kwargs = { + "options" : self.options, + "timeout" : self.timeout + } + # We'll forward start() method invocation to the proper package later + self.real_package = pkg_class(self.target, self.host, **kwargs) + + def start(self): + # We have nothing to do here; let the proper package do it's job + self.prepare() + if not self.real_package: + raise Exception("Invalid analysis package, aborting") + self.real_package.start() + + def _extract(self, filename, password): + archive_path = _prepare_archive_at_path(filename) + if not archive_path: + return None + # Extraction. + extract_path = environ.get("TEMP", "/tmp") + with ZipFile(archive_path, "r") as archive: + try: + archive.extractall(path=extract_path, pwd=password) + except BadZipfile: + raise Exception("Invalid Zip file") + # Try to extract it again, but with a default password + except RuntimeError: + try: + archive.extractall(path=extract_path, pwd="infected") + except RuntimeError as err: + raise Exception("Unable to extract Zip file: %s" % err) + finally: + self._extract_nested_archives(archive, extract_path, password) + return archive.namelist() + + def _extract_nested_archives(self, archive, where, password): + for name in archive.namelist(): + if name.endswith(".zip"): + self._extract(path.join(where, name), password) + + +def _prepare_archive_at_path(filename): + """ Verify that there's a readable zip archive at the given path. + + This function returns a new name for the archive (for most cases it's + the same as the original one; but if an archive named "foo.zip" contains + a file named "foo" this archive will be renamed to avoid being overwrite. + """ + # Verify that the archive is actually readable + try: + with ZipFile(filename, "r") as archive: + archive.close() + except BadZipfile: + return None + # Test if zip file contains a file named as itself + if _is_overwritten(filename): + log.debug("ZIP file contains a file with the same name, original is \ + going to be overwrite") + # In this case we just change the file name + new_zip_path = filename + _random_extension() + move(filename, new_zip_path) + filename = new_zip_path + return filename + + +def _is_overwritten(zip_path): + archive = ZipFile(zip_path, "r") + try: + # Test if zip file contains a file named as itself + return any(n == path.basename(zip_path) for n in archive.namelist()) + except BadZipfile: + raise Exception("Invalid Zip file") + + +def _random_extension(length=5): + return '.' + ''.join(SystemRandom().choice(ascii_letters) for _ in range(length)) + + +def _fileinfo(target): + raw = check_output(["file", target]) + # The utility has the following output format: "%filename%: %description%", + # so we just skip everything before the actual description + return raw[raw.index(":")+2:] diff --git a/cuckoo/data/analyzer/linux/analyzer.py b/cuckoo/data/analyzer/linux/analyzer.py new file mode 100644 index 0000000000..9e92066c1b --- /dev/null +++ b/cuckoo/data/analyzer/linux/analyzer.py @@ -0,0 +1,382 @@ +# Copyright (C) 2015-2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import os +import sys +import pkgutil +import logging +import tempfile +import xmlrpclib +import traceback +import urllib +import urllib2 +import time +import datetime + +from lib.api.process import Process +from lib.common.abstracts import Package, Auxiliary +from lib.common.constants import PATHS +from lib.common.exceptions import CuckooError, CuckooPackageError +from lib.common.results import upload_to_host +from lib.core.config import Config +from lib.core.startup import create_folders, init_logging +from modules import auxiliary + +log = logging.getLogger() + +PID = os.getpid() +FILES_LIST = set() +DUMPED_LIST = set() +PROCESS_LIST = set() +SEEN_LIST = set() +PPID = Process(pid=PID).get_parent_pid() + +def add_pids(pids): + """Add PID.""" + if not isinstance(pids, (tuple, list, set)): + pids = [pids] + + for pid in pids: + log.info("Added new process to list with pid: %s", pid) + pid = int(pid) + if pid not in SEEN_LIST: + PROCESS_LIST.add(pid) + SEEN_LIST.add(pid) + +def dump_files(): + """Dump all the dropped files.""" + for file_path in FILES_LIST: + log.info("PLS IMPLEMENT DUMP, want to dump %s", file_path) + +class Analyzer: + """Cuckoo Linux Analyzer. + + This class handles the initialization and execution of the analysis + procedure, including the auxiliary modules and the analysis packages. + """ + + def __init__(self): + self.config = None + self.target = None + + def prepare(self): + """Prepare env for analysis.""" + + # Create the folders used for storing the results. + create_folders() + + # Initialize logging. + init_logging() + + # Parse the analysis configuration file generated by the agent. + self.config = Config(cfg="analysis.conf") + + if self.config.get("clock", None): + # Set virtual machine clock. + clock = datetime.datetime.strptime(self.config.clock, "%Y%m%dT%H:%M:%S") + # Setting date and time. + os.system("date -s \"{0}\"".format(clock.strftime("%y-%m-%d %H:%M:%S"))) + + # We update the target according to its category. If it's a file, then + # we store the path. + if self.config.category == "file": + self.target = os.path.join(tempfile.gettempdir(), self.config.file_name) + # If it's a URL, well.. we store the URL. + else: + self.target = self.config.target + + def complete(self): + """End analysis.""" + # Dump all the notified files. + dump_files() + + # Hell yeah. + log.info("Analysis completed.") + + def run(self): + """Run analysis. + @return: operation status. + """ + self.prepare() + + log.debug("Starting analyzer from: %s", os.getcwd()) + log.debug("Storing results at: %s", PATHS["root"]) + + # If no analysis package was specified at submission, we try to select + # one automatically. + if not self.config.package: + log.debug("No analysis package specified, trying to detect " + "it automagically.") + + if self.config.category == "file": + package = "generic" + else: + package = "wget" + + # If we weren't able to automatically determine the proper package, + # we need to abort the analysis. + if not package: + raise CuckooError("No valid package available for file " + "type: {0}".format(self.config.file_type)) + + log.info("Automatically selected analysis package \"%s\"", package) + # Otherwise just select the specified package. + else: + package = self.config.package + + # Generate the package path. + package_name = "modules.packages.%s" % package + + # Try to import the analysis package. + try: + __import__(package_name, globals(), locals(), ["dummy"], -1) + # If it fails, we need to abort the analysis. + except ImportError: + raise CuckooError("Unable to import package \"{0}\", does " + "not exist.".format(package_name)) + + # Initialize the package parent abstract. + Package() + + # Enumerate the abstract subclasses. + try: + package_class = Package.__subclasses__()[0] + except IndexError as e: + raise CuckooError("Unable to select package class " + "(package={0}): {1}".format(package_name, e)) + + # Initialize the analysis package. + pack = package_class(self.config.get_options()) + + # Initialize Auxiliary modules + Auxiliary() + prefix = auxiliary.__name__ + "." + for loader, name, ispkg in pkgutil.iter_modules(auxiliary.__path__, prefix): + if ispkg: + continue + + # Import the auxiliary module. + try: + __import__(name, globals(), locals(), ["dummy"], -1) + except ImportError as e: + log.warning("Unable to import the auxiliary module " + "\"%s\": %s", name, e) + + # Walk through the available auxiliary modules. + aux_enabled, aux_avail = [], [] + for module in sorted(Auxiliary.__subclasses__(), key=lambda x: x.priority, reverse=True): + # Try to start the auxiliary module. + try: + aux = module() + aux_avail.append(aux) + aux.start() + except (NotImplementedError, AttributeError): + log.warning("Auxiliary module %s was not implemented", + aux.__class__.__name__) + continue + except Exception as e: + log.warning("Cannot execute auxiliary module %s: %s", + aux.__class__.__name__, e) + continue + finally: + log.debug("Started auxiliary module %s", + aux.__class__.__name__) + aux_enabled.append(aux) + + # Start analysis package. If for any reason, the execution of the + # analysis package fails, we have to abort the analysis. + try: + pids = pack.start(self.target) + except NotImplementedError: + raise CuckooError("The package \"{0}\" doesn't contain a run " + "function.".format(package_name)) + except CuckooPackageError as e: + raise CuckooError("The package \"{0}\" start function raised an " + "error: {1}".format(package_name, e)) + except Exception as e: + raise CuckooError("The package \"{0}\" start function encountered " + "an unhandled exception: " + "{1}".format(package_name, e)) + + # If the analysis package returned a list of process IDs, we add them + # to the list of monitored processes and enable the process monitor. + if pids: + add_pids(pids) + pid_check = True + + # If the package didn't return any process ID (for example in the case + # where the package isn't enabling any behavioral analysis), we don't + # enable the process monitor. + else: + log.info("No process IDs returned by the package, running " + "for the full timeout.") + pid_check = False + + # Check in the options if the user toggled the timeout enforce. If so, + # we need to override pid_check and disable process monitor. + if self.config.enforce_timeout: + log.info("Enabled timeout enforce, running for the full timeout.") + pid_check = False + + time_counter = 0 + + while True: + time_counter += 1 + if time_counter == int(self.config.timeout): + log.info("Analysis timeout hit, terminating analysis.") + break + + try: + # If the process monitor is enabled we start checking whether + # the monitored processes are still alive. + if pid_check: + for pid in list(PROCESS_LIST): + if not Process(pid=pid).is_alive(): + log.info("Process with pid %s has terminated", pid) + PROCESS_LIST.remove(pid) + + # ask the package if it knows any new pids + add_pids(pack.get_pids()) + + # also ask the auxiliaries + for aux in aux_avail: + add_pids(aux.get_pids()) + + # If none of the monitored processes are still alive, we + # can terminate the analysis. + if not PROCESS_LIST: + log.info("Process list is empty, " + "terminating analysis.") + break + + # Update the list of monitored processes available to the + # analysis package. It could be used for internal + # operations within the module. + pack.set_pids(PROCESS_LIST) + + try: + # The analysis packages are provided with a function that + # is executed at every loop's iteration. If such function + # returns False, it means that it requested the analysis + # to be terminate. + if not pack.check(): + log.info("The analysis package requested the " + "termination of the analysis.") + break + + # If the check() function of the package raised some exception + # we don't care, we can still proceed with the analysis but we + # throw a warning. + except Exception as e: + log.warning("The package \"%s\" check function raised " + "an exception: %s", package_name, e) + except Exception as e: + log.exception("The PID watching loop raised an exception: %s", e) + finally: + # Zzz. + time.sleep(1) + + try: + # Before shutting down the analysis, the package can perform some + # final operations through the finish() function. + pack.finish() + except Exception as e: + log.warning("The package \"%s\" finish function raised an " + "exception: %s", package_name, e) + + try: + # Upload files the package created to package_files in the results folder + package_files = pack.package_files() + if package_files is not None: + for package in package_files: + upload_to_host( + package[0], os.path.join("package_files", package[1]) + ) + except Exception as e: + log.warning("The package \"%s\" package_files function raised an " + "exception: %s", package_name, e) + + # Terminate the Auxiliary modules. + for aux in sorted(aux_enabled, key=lambda x: x.priority): + try: + aux.stop() + except (NotImplementedError, AttributeError): + continue + except Exception as e: + log.warning("Cannot terminate auxiliary module %s: %s", + aux.__class__.__name__, e) + + if self.config.terminate_processes: + # Try to terminate remaining active processes. We do this to make sure + # that we clean up remaining open handles (sockets, files, etc.). + log.info("Terminating remaining processes before shutdown.") + + for pid in PROCESS_LIST: + proc = Process(pid=pid) + if proc.is_alive(): + try: + proc.terminate() + except: + continue + + # Run the finish callback of every available Auxiliary module. + for aux in aux_avail: + try: + aux.finish() + except (NotImplementedError, AttributeError): + continue + except Exception as e: + log.warning("Exception running finish callback of auxiliary " + "module %s: %s", aux.__class__.__name__, e) + + # Let's invoke the completion procedure. + self.complete() + + return True + +if __name__ == "__main__": + success = False + error = "" + + try: + # Initialize the main analyzer class. + analyzer = Analyzer() + + # Run it and wait for the response. + success = analyzer.run() + + # This is not likely to happen. + except KeyboardInterrupt: + error = "Keyboard Interrupt" + + # If the analysis process encountered a critical error, it will raise a + # CuckooError exception, which will force the termination of the analysis. + # Notify the agent of the failure. Also catch unexpected exceptions. + except Exception as e: + # Store the error. + error_exc = traceback.format_exc() + error = str(e) + + # Just to be paranoid. + if len(log.handlers): + log.exception(error_exc) + else: + sys.stderr.write("{0}\n".format(error_exc)) + + # Once the analysis is completed or terminated for any reason, we report + # back to the agent, notifying that it can report back to the host. + finally: + try: + # old agent + server = xmlrpclib.Server("http://127.0.0.1:8000") + server.complete(success, error, PATHS["root"]) + except xmlrpclib.ProtocolError: + # new agent + data = { + "status": "complete", + "description": success + } + urllib2.urlopen( + "http://127.0.0.1:8000/status", urllib.urlencode(data) + ) diff --git a/analyzer/darwin/modules/packages/__init__.py b/cuckoo/data/analyzer/linux/lib/__init__.py similarity index 100% rename from analyzer/darwin/modules/packages/__init__.py rename to cuckoo/data/analyzer/linux/lib/__init__.py diff --git a/analyzer/linux/lib/__init__.py b/cuckoo/data/analyzer/linux/lib/api/__init__.py similarity index 100% rename from analyzer/linux/lib/__init__.py rename to cuckoo/data/analyzer/linux/lib/api/__init__.py diff --git a/cuckoo/data/analyzer/linux/lib/api/process.py b/cuckoo/data/analyzer/linux/lib/api/process.py new file mode 100644 index 0000000000..7aaae353ba --- /dev/null +++ b/cuckoo/data/analyzer/linux/lib/api/process.py @@ -0,0 +1,47 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import os +import subprocess +import logging + +log = logging.getLogger(__name__) + +class Process: + """Linux process.""" + first_process = True + first_process_pid = None + + def __init__(self, pid=0): + """@param pid: PID. + """ + self.pid = pid + + def is_alive(self): + if not os.path.exists("/proc/%u" % self.pid): return False + status = self.get_proc_status() + if not status: return False + if "zombie" in status.get("State:", ""): return False + return True + + def get_parent_pid(self): + return self.get_proc_status().get("PPid", None) + + def get_proc_status(self): + try: + status = open("/proc/%u/status" % self.pid).readlines() + status_values = dict((i[0], i[1]) for i in [j.strip().split(None, 1) for j in status]) + return status_values + except: + log.critical("could not get process status for pid %u", self.pid) + return {} + + def execute(self, cmd): + """Start a subprocess. + @param cmd: process path + @return: subprocess status + """ + self.proc = proc = subprocess.Popen(cmd) + self.pid = proc.pid + return True diff --git a/analyzer/linux/lib/api/__init__.py b/cuckoo/data/analyzer/linux/lib/common/__init__.py similarity index 100% rename from analyzer/linux/lib/api/__init__.py rename to cuckoo/data/analyzer/linux/lib/common/__init__.py diff --git a/cuckoo/data/analyzer/linux/lib/common/abstracts.py b/cuckoo/data/analyzer/linux/lib/common/abstracts.py new file mode 100644 index 0000000000..9507ff816a --- /dev/null +++ b/cuckoo/data/analyzer/linux/lib/common/abstracts.py @@ -0,0 +1,71 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +from lib.api.process import Process +from lib.common.exceptions import CuckooPackageError + +class Package(object): + """Base abstract analysis package.""" + PATHS = [] + + def __init__(self, options={}): + """@param options: options dict.""" + self.options = options + self.pids = [] + + def set_pids(self, pids): + """Update list of monitored PIDs in the package context. + @param pids: list of pids. + """ + self.pids = pids + + def start(self): + """Run analysis package. + @raise NotImplementedError: this method is abstract. + """ + raise NotImplementedError + + def check(self): + """Check.""" + return True + + def execute(self, cmd): + """Start an executable for analysis. + @param cmd: executable path + @return: process pid + """ + p = Process() + if not p.execute(cmd): + raise CuckooPackageError("Unable to execute the initial process, " + "analysis aborted.") + + return p.pid + + def package_files(self): + """A list of files to upload to host. + The list should be a list of tuples (, ). + (package_files is a folder that will be created in analysis folder). + """ + return None + + def finish(self): + """Finish run. + If specified to do so, this method dumps the memory of + all running processes. + """ + if self.options.get("procmemdump"): + for pid in self.pids: + p = Process(pid=pid) + p.dump_memory() + + return True + + def get_pids(self): + return [] + +class Auxiliary(object): + priority = 0 + + def get_pids(self): + return [] diff --git a/cuckoo/data/analyzer/linux/lib/common/constants.py b/cuckoo/data/analyzer/linux/lib/common/constants.py new file mode 100644 index 0000000000..b748949e01 --- /dev/null +++ b/cuckoo/data/analyzer/linux/lib/common/constants.py @@ -0,0 +1,16 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import os +import tempfile +from lib.common.utils import random_string + +ROOT = os.path.join(tempfile.gettempdir(), random_string(6, 10)) + +PATHS = {"root" : ROOT, + "logs" : os.path.join(ROOT, "logs"), + "files" : os.path.join(ROOT, "files"), + "shots" : os.path.join(ROOT, "shots"), + "memory" : os.path.join(ROOT, "memory"), + "drop" : os.path.join(ROOT, "drop")} diff --git a/cuckoo/data/analyzer/linux/lib/common/exceptions.py b/cuckoo/data/analyzer/linux/lib/common/exceptions.py new file mode 100644 index 0000000000..48439fdf3e --- /dev/null +++ b/cuckoo/data/analyzer/linux/lib/common/exceptions.py @@ -0,0 +1,9 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +class CuckooError(Exception): + pass + +class CuckooPackageError(Exception): + pass \ No newline at end of file diff --git a/cuckoo/data/analyzer/linux/lib/common/hashing.py b/cuckoo/data/analyzer/linux/lib/common/hashing.py new file mode 100644 index 0000000000..1ffa54190d --- /dev/null +++ b/cuckoo/data/analyzer/linux/lib/common/hashing.py @@ -0,0 +1,25 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import hashlib + +BUFSIZE = 1024*1024 + +def sha256_file(path): + return hash_file(hashlib.sha256, path) + +def hash_file(method, path): + """Calculate a hash on a file by path. + @param method: callable hashing method + @param path: file path + @return: computed hash string + """ + f = open(path, "rb") + h = method() + while True: + buf = f.read(BUFSIZE) + if not buf: + break + h.update(buf) + return h.hexdigest() diff --git a/cuckoo/data/analyzer/linux/lib/common/results.py b/cuckoo/data/analyzer/linux/lib/common/results.py new file mode 100644 index 0000000000..1590bc8956 --- /dev/null +++ b/cuckoo/data/analyzer/linux/lib/common/results.py @@ -0,0 +1,95 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import logging +import socket +import time + +from lib.core.config import Config + +log = logging.getLogger(__name__) + +BUFSIZE = 1024*1024 + +def upload_to_host(file_path, dump_path): + nc = infd = None + try: + nc = NetlogFile(dump_path) + + infd = open(file_path, "rb") + buf = infd.read(BUFSIZE) + while buf: + nc.send(buf, retry=False) + buf = infd.read(BUFSIZE) + except Exception as e: + log.error("Exception uploading file %s to host: %s", file_path, e) + finally: + if infd: + infd.close() + if nc: + nc.close() + +class NetlogConnection(object): + def __init__(self, proto=""): + config = Config(cfg="analysis.conf") + self.hostip, self.hostport = config.ip, config.port + self.sock, self.file = None, None + self.proto = proto + + def connect(self): + i = 1 + # this can loop forever, if we can't connect the whole analysis is useless anyways + while True: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + s.connect((self.hostip, self.hostport)) + s.sendall(self.proto) + except: + time.sleep(i) + i = min(i + 1, 60) + else: + self.sock = s + self.file = s.makefile() + break + + def send(self, data, retry=True): + if not self.sock: self.connect() + + try: + self.sock.sendall(data) + except socket.error as e: + if retry: + self.connect() + self.send(data, retry=False) + else: + raise + except Exception as e: + log.error("Unhandled exception in NetlogConnection: %s", str(e)) + # We really have nowhere to log this, if the netlog connection + # does not work, we can assume that any logging won't work either. + # So we just fail silently. + self.close() + + def close(self): + try: + self.file.close() + self.sock.close() + except Exception: + pass + +class NetlogFile(NetlogConnection): + def __init__(self, filepath): + self.filepath = filepath + NetlogConnection.__init__(self, proto="FILE\n{0}\n".format(self.filepath)) + self.connect() + +class NetlogHandler(logging.Handler, NetlogConnection): + def __init__(self): + logging.Handler.__init__(self) + NetlogConnection.__init__(self, proto="LOG\n") + self.connect() + + def emit(self, record): + msg = self.format(record) + self.send("{0}\n".format(msg)) diff --git a/cuckoo/data/analyzer/linux/lib/common/utils.py b/cuckoo/data/analyzer/linux/lib/common/utils.py new file mode 100644 index 0000000000..60ff19d981 --- /dev/null +++ b/cuckoo/data/analyzer/linux/lib/common/utils.py @@ -0,0 +1,18 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import random +import string + +def random_string(minimum, maximum=None): + if maximum is None: + maximum = minimum + + count = random.randint(minimum, maximum) + return "".join(random.choice(string.ascii_letters) for x in xrange(count)) + +def random_integer(digits): + start = 10 ** (digits - 1) + end = (10 ** digits) - 1 + return random.randint(start, end) diff --git a/analyzer/linux/lib/common/__init__.py b/cuckoo/data/analyzer/linux/lib/core/__init__.py similarity index 100% rename from analyzer/linux/lib/common/__init__.py rename to cuckoo/data/analyzer/linux/lib/core/__init__.py diff --git a/cuckoo/data/analyzer/linux/lib/core/config.py b/cuckoo/data/analyzer/linux/lib/core/config.py new file mode 100644 index 0000000000..4a70b227c3 --- /dev/null +++ b/cuckoo/data/analyzer/linux/lib/core/config.py @@ -0,0 +1,62 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import ConfigParser + +class Config: + def __init__(self, cfg): + """@param cfg: configuration file.""" + config = ConfigParser.ConfigParser(allow_no_value=True) + config.read(cfg) + + for section in config.sections(): + for name, raw_value in config.items(section): + if name == "file_name": + value = config.get(section, name) + else: + try: + value = config.getboolean(section, name) + except ValueError: + try: + value = config.getint(section, name) + except ValueError: + value = config.get(section, name) + setattr(self, name, value) + + def get(self, name, default=None): + """Get attribute. + @return: attribute. + """ + return getattr(self, name, default) + + def get_options(self): + """Get analysis options. + @return: options dict. + """ + # The analysis package can be provided with some options in the + # following format: + # option1=value1,option2=value2,option3=value3 + # + # Here we parse such options and provide a dictionary that will be made + # accessible to the analysis package. + options = {} + if hasattr(self, "options"): + try: + # Split the options by comma. + fields = self.options.split(",") + except ValueError as e: + pass + else: + for field in fields: + # Split the name and the value of the option. + try: + key, value = field.split("=", 1) + except ValueError: + pass + else: + # If the parsing went good, we add the option to the + # dictionary. + options[key.strip()] = value.strip() + + return options diff --git a/cuckoo/data/analyzer/linux/lib/core/startup.py b/cuckoo/data/analyzer/linux/lib/core/startup.py new file mode 100644 index 0000000000..1d1c73c195 --- /dev/null +++ b/cuckoo/data/analyzer/linux/lib/core/startup.py @@ -0,0 +1,35 @@ +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import os +import logging + +from lib.common.constants import PATHS +from lib.common.results import NetlogHandler + +log = logging.getLogger() + +def create_folders(): + """Create folders in PATHS.""" + for name, folder in PATHS.items(): + if os.path.exists(folder): + continue + + try: + os.makedirs(folder) + except OSError: + pass + +def init_logging(): + """Initialize logger.""" + formatter = logging.Formatter("%(asctime)s [%(name)s] %(levelname)s: %(message)s") + sh = logging.StreamHandler() + sh.setFormatter(formatter) + log.addHandler(sh) + + nh = NetlogHandler() + nh.setFormatter(formatter) + log.addHandler(nh) + + log.setLevel(logging.DEBUG) diff --git a/analyzer/linux/lib/core/__init__.py b/cuckoo/data/analyzer/linux/modules/__init__.py similarity index 100% rename from analyzer/linux/lib/core/__init__.py rename to cuckoo/data/analyzer/linux/modules/__init__.py diff --git a/analyzer/linux/modules/__init__.py b/cuckoo/data/analyzer/linux/modules/auxiliary/__init__.py similarity index 100% rename from analyzer/linux/modules/__init__.py rename to cuckoo/data/analyzer/linux/modules/auxiliary/__init__.py diff --git a/cuckoo/data/analyzer/linux/modules/auxiliary/stap.py b/cuckoo/data/analyzer/linux/modules/auxiliary/stap.py new file mode 100644 index 0000000000..3ddbdfeec0 --- /dev/null +++ b/cuckoo/data/analyzer/linux/modules/auxiliary/stap.py @@ -0,0 +1,72 @@ +# Copyright (C) 2015-2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import logging +import os +import subprocess +import time + +from lib.common.abstracts import Auxiliary +from lib.common.results import NetlogFile +from lib.core.config import Config + +log = logging.getLogger(__name__) + +class STAP(Auxiliary): + """System-wide syscall trace with stap.""" + priority = -10 # low prio to wrap tightly around the analysis + + def __init__(self): + self.config = Config(cfg="analysis.conf") + self.proc = None + + def start(self): + # helper function locating the stap module + def has_stap(p): + only_stap = [fn for fn in os.listdir(p) if fn.startswith("stap_") and fn.endswith(".ko")] + if only_stap: return os.path.join(p, only_stap[0]) + return False + + path_cfg = self.config.get("analyzer_stap_path", None) + if path_cfg and os.path.exists(path_cfg): + path = path_cfg + elif os.path.exists("/root/.cuckoo") and has_stap("/root/.cuckoo"): + path = has_stap("/root/.cuckoo") + else: + log.warning("Could not find STAP LKM, aborting systemtap analysis.") + return False + + stap_start = time.time() + self.proc = subprocess.Popen([ + "staprun", "-vv", + "-x", str(os.getpid()), + "-o", "stap.log", + path, + ], stderr=subprocess.PIPE) + + while "systemtap_module_init() returned 0" not in self.proc.stderr.readline(): + pass + + stap_stop = time.time() + log.info("STAP aux module startup took %.2f seconds" % (stap_stop - stap_start)) + return True + + @staticmethod + def _upload_file(local, remote): + if os.path.exists(local): + nf = NetlogFile(remote) + with open(local, "rb") as f: + for chunk in f: + nf.sock.sendall(chunk) # dirty direct send, no reconnecting + nf.close() + + def stop(self): + try: + r = self.proc.poll() + log.debug("stap subprocess retval %r", r) + self.proc.kill() + except Exception as e: + log.warning("Exception killing stap: %s", e) + + self._upload_file("stap.log", "logs/all.stap") diff --git a/analyzer/linux/modules/auxiliary/__init__.py b/cuckoo/data/analyzer/linux/modules/packages/__init__.py similarity index 100% rename from analyzer/linux/modules/auxiliary/__init__.py rename to cuckoo/data/analyzer/linux/modules/packages/__init__.py diff --git a/cuckoo/data/analyzer/linux/modules/packages/generic.py b/cuckoo/data/analyzer/linux/modules/packages/generic.py new file mode 100644 index 0000000000..bcdb3bff00 --- /dev/null +++ b/cuckoo/data/analyzer/linux/modules/packages/generic.py @@ -0,0 +1,18 @@ +# Copyright (C) 2015-2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import os + +from lib.common.abstracts import Package + +class Generic(Package): + """Generic analysis package. Uses shell based execution.""" + + def __init__(self, *args, **kwargs): + Package.__init__(self, *args, **kwargs) + self.seen_pids = set() + + def start(self, path): + os.chmod(path, 0o755) + return self.execute(["sh", "-c", path]) diff --git a/cuckoo/data/analyzer/windows/analyzer.py b/cuckoo/data/analyzer/windows/analyzer.py new file mode 100644 index 0000000000..de1be36162 --- /dev/null +++ b/cuckoo/data/analyzer/windows/analyzer.py @@ -0,0 +1,857 @@ +# Copyright (C) 2011-2013 Claudio Guarnieri. +# Copyright (C) 2014-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import datetime +import hashlib +import logging +import os +import pkgutil +import socket +import struct +import sys +import threading +import traceback +import urllib +import urllib2 +import xmlrpclib +import zipfile + +from lib.api.process import Process +from lib.common.abstracts import Package, Auxiliary +from lib.common.constants import SHUTDOWN_MUTEX +from lib.common.decide import dump_memory +from lib.common.defines import KERNEL32 +from lib.common.exceptions import CuckooError, CuckooDisableModule +from lib.common.hashing import hash_file +from lib.common.rand import random_string +from lib.common.results import upload_to_host +from lib.core.config import Config +from lib.core.ioctl import zer0m0n +from lib.core.packages import choose_package +from lib.core.pipe import PipeServer, PipeForwarder, PipeDispatcher +from lib.core.pipe import disconnect_pipes +from lib.core.privileges import grant_privilege +from lib.core.startup import init_logging, disconnect_logger, set_clock +from modules import auxiliary + +log = logging.getLogger("analyzer") + +class Files(object): + PROTECTED_NAMES = () + + def __init__(self): + self.files = {} + self.files_orig = {} + self.dumped = [] + + def is_protected_filename(self, file_name): + """Return whether or not to inject into a process with this name.""" + return file_name.lower() in self.PROTECTED_NAMES + + def add_pid(self, filepath, pid, verbose=True): + """Track a process identifier for this file.""" + if not pid or filepath.lower() not in self.files: + return + + if pid not in self.files[filepath.lower()]: + self.files[filepath.lower()].append(pid) + verbose and log.info("Added pid %s for %r", pid, filepath) + + def add_file(self, filepath, pid=None): + """Add filepath to the list of files and track the pid.""" + if filepath.lower() not in self.files: + log.info( + "Added new file to list with pid %s and path %s", + pid, filepath.encode("utf8") + ) + self.files[filepath.lower()] = [] + self.files_orig[filepath.lower()] = filepath + + self.add_pid(filepath, pid, verbose=False) + + def dump_file(self, filepath): + """Dump a file to the host.""" + if not os.path.isfile(filepath): + log.warning("File at path %r does not exist, skip.", filepath) + return False + + # Check whether we've already dumped this file - in that case skip it. + try: + sha256 = hash_file(hashlib.sha256, filepath) + if sha256 in self.dumped: + return + except IOError as e: + log.info("Error dumping file from path \"%s\": %s", filepath, e) + return + + filename = "%s_%s" % (sha256[:16], os.path.basename(filepath)) + upload_path = os.path.join("files", filename) + + try: + upload_to_host( + # If available use the original filepath, the one that is + # not lowercased. + self.files_orig.get(filepath.lower(), filepath), + upload_path, self.files.get(filepath.lower(), []) + ) + self.dumped.append(sha256) + except (IOError, socket.error) as e: + log.error( + "Unable to upload dropped file at path \"%s\": %s", + filepath, e + ) + + def delete_file(self, filepath, pid=None): + """A file is about to removed and thus should be dumped right away.""" + self.add_pid(filepath, pid) + self.dump_file(filepath) + + # Remove the filepath from the files list. + self.files.pop(filepath.lower(), None) + self.files_orig.pop(filepath.lower(), None) + + def move_file(self, oldfilepath, newfilepath, pid=None): + """A file will be moved - track this change.""" + self.add_pid(oldfilepath, pid) + if oldfilepath.lower() in self.files: + # Replace the entry with the new filepath. + self.files[newfilepath.lower()] = \ + self.files.pop(oldfilepath.lower(), []) + + def dump_files(self): + """Dump all pending files.""" + while self.files: + self.delete_file(self.files.keys()[0]) + +class ProcessList(object): + def __init__(self): + self.pids = [] + self.pids_notrack = [] + + def add_pid(self, pid, track=True): + """Add a process identifier to the process list. + + Track determines whether the analyzer should be monitoring this + process, i.e., whether Cuckoo should wait for this process to finish. + """ + if int(pid) not in self.pids and int(pid) not in self.pids_notrack: + if track: + self.pids.append(int(pid)) + else: + self.pids_notrack.append(int(pid)) + + def add_pids(self, pids): + """Add one or more process identifiers to the process list.""" + if isinstance(pids, (tuple, list)): + for pid in pids: + self.add_pid(pid) + else: + self.add_pid(pids) + + def has_pid(self, pid, notrack=True): + """Return whether or not this process identifier being tracked.""" + if int(pid) in self.pids: + return True + + if notrack and int(pid) in self.pids_notrack: + return True + + return False + + def remove_pid(self, pid): + """Remove a process identifier from being tracked.""" + if pid in self.pids: + self.pids.remove(pid) + + if pid in self.pids_notrack: + self.pids_notrack.remove(pid) + +class CommandPipeHandler(object): + """Pipe Handler. + + This class handles the notifications received through the Pipe Server and + decides what to do with them. + """ + ignore_list = dict(pid=[]) + + def __init__(self, analyzer): + self.analyzer = analyzer + self.tracked = {} + + def _handle_debug(self, data): + """Debug message from the monitor.""" + log.debug(data) + + def _handle_info(self, data): + """Regular message from the monitor.""" + log.info(data) + + def _handle_warning(self, data): + """Warning message from the monitor.""" + log.warning(data) + + def _handle_critical(self, data): + """Critical message from the monitor.""" + log.critical(data) + + def _handle_loaded(self, data): + """The monitor has loaded into a particular process.""" + if not data or data.count(",") != 1: + log.warning("Received loaded command with incorrect parameters, " + "skipping it.") + return + + pid, track = data.split(",") + if not pid.isdigit() or not track.isdigit(): + log.warning("Received loaded command with incorrect parameters, " + "skipping it.") + return + + self.analyzer.process_lock.acquire() + self.analyzer.process_list.add_pid(int(pid), track=int(track)) + self.analyzer.process_lock.release() + + log.debug("Loaded monitor into process with pid %s", pid) + + def _handle_getpids(self, data): + """Return the process identifiers of the agent and its parent + process.""" + return struct.pack("II", self.analyzer.pid, self.analyzer.ppid) + + def _inject_process(self, process_id, thread_id, mode): + """Helper function for injecting the monitor into a process.""" + # We acquire the process lock in order to prevent the analyzer to + # terminate the analysis while we are operating on the new process. + self.analyzer.process_lock.acquire() + + # Set the current DLL to the default one provided at submission. + dll = self.analyzer.default_dll + + if process_id in (self.analyzer.pid, self.analyzer.ppid): + if process_id not in self.ignore_list["pid"]: + log.warning("Received request to inject Cuckoo processes, " + "skipping it.") + self.ignore_list["pid"].append(process_id) + self.analyzer.process_lock.release() + return + + # We inject the process only if it's not being monitored already, + # otherwise we would generated polluted logs (if it wouldn't crash + # horribly to start with). + if self.analyzer.process_list.has_pid(process_id): + # This pid is already on the notrack list, move it to the + # list of tracked pids. + if not self.analyzer.process_list.has_pid(process_id, notrack=False): + log.debug("Received request to inject pid=%d. It was already " + "on our notrack list, moving it to the track list.") + + self.analyzer.process_list.remove_pid(process_id) + self.analyzer.process_list.add_pid(process_id) + self.ignore_list["pid"].append(process_id) + # Spit out an error once and just ignore it further on. + elif process_id not in self.ignore_list["pid"]: + self.ignore_list["pid"].append(process_id) + + # We're done operating on the processes list, release the lock. + self.analyzer.process_lock.release() + return + + # Open the process and inject the DLL. Hope it enjoys it. + proc = Process(pid=process_id, tid=thread_id) + + filename = os.path.basename(proc.get_filepath()) + + if not self.analyzer.files.is_protected_filename(filename): + # Add the new process ID to the list of monitored processes. + self.analyzer.process_list.add_pid(process_id) + + # We're done operating on the processes list, + # release the lock. Let the injection do its thing. + self.analyzer.process_lock.release() + + # If we have both pid and tid, then we can use APC to inject. + if process_id and thread_id: + proc.inject(dll, apc=True, mode="%s" % mode) + else: + proc.inject(dll, apc=False, mode="%s" % mode) + + log.info("Injected into process with pid %s and name %r", + proc.pid, filename) + + def _handle_process(self, data): + """Request for injection into a process.""" + # Parse the process identifier. + if not data or not data.isdigit(): + log.warning("Received PROCESS command from monitor with an " + "incorrect argument.") + return + + return self._inject_process(int(data), None, 0) + + def _handle_process2(self, data): + """Request for injection into a process using APC.""" + # Parse the process and thread identifier. + if not data or data.count(",") != 2: + log.warning("Received PROCESS2 command from monitor with an " + "incorrect argument.") + return + + pid, tid, mode = data.split(",") + if not pid.isdigit() or not tid.isdigit() or not mode.isdigit(): + log.warning("Received PROCESS2 command from monitor with an " + "incorrect argument.") + return + + return self._inject_process(int(pid), int(tid), int(mode)) + + def _handle_file_new(self, data): + """Notification of a new dropped file.""" + self.analyzer.files.add_file(data.decode("utf8"), self.pid) + + def _handle_file_del(self, data): + """Notification of a file being removed (if it exists) - we have to + dump it before it's being removed.""" + filepath = data.decode("utf8") + if os.path.exists(filepath): + self.analyzer.files.delete_file(filepath, self.pid) + + def _handle_file_move(self, data): + """A file is being moved - track these changes.""" + if "::" not in data: + log.warning("Received FILE_MOVE command from monitor with an " + "incorrect argument.") + return + + old_filepath, new_filepath = data.split("::", 1) + self.analyzer.files.move_file( + old_filepath.decode("utf8"), new_filepath.decode("utf8"), self.pid + ) + + def _handle_kill(self, data): + """A process is being killed.""" + if not data.isdigit(): + log.warning("Received KILL command with an incorrect argument.") + return + + if self.analyzer.config.options.get("procmemdump"): + dump_memory(int(data)) + + def _handle_dumpmem(self, data): + """Dump the memory of a process as it is right now.""" + if not data.isdigit(): + log.warning("Received DUMPMEM command with an incorrect argument.") + return + + dump_memory(int(data)) + + def _handle_dumpreqs(self, data): + if not data.isdigit(): + log.warning("Received DUMPREQS command with an incorrect argument %r.", data) + return + + pid = int(data) + + if pid not in self.tracked: + log.warning("Received DUMPREQS command but there are no reqs for pid %d.", pid) + return + + dumpreqs = self.tracked[pid].get("dumpreq", []) + for addr, length in dumpreqs: + log.debug("tracked dump req (%r, %r, %r)", pid, addr, length) + + if not addr or not length: + continue + + Process(pid=pid).dump_memory_block(int(addr), int(length)) + + def _handle_track(self, data): + if not data.count(":") == 2: + log.warning("Received TRACK command with an incorrect argument %r.", data) + return + + pid, scope, params = data.split(":", 2) + pid = int(pid) + + paramtuple = params.split(",") + if pid not in self.tracked: + self.tracked[pid] = {} + if scope not in self.tracked[pid]: + self.tracked[pid][scope] = [] + self.tracked[pid][scope].append(paramtuple) + + def dispatch(self, data): + response = "NOPE" + + if not data or ":" not in data: + log.critical("Unknown command received from the monitor: %r", + data.strip()) + else: + # Backwards compatibility (old syntax is, e.g., "FILE_NEW:" vs the + # new syntax, e.g., "1234:FILE_NEW:"). + if data[0].isupper(): + command, arguments = data.strip().split(":", 1) + self.pid = None + else: + self.pid, command, arguments = data.strip().split(":", 2) + + fn = getattr(self, "_handle_%s" % command.lower(), None) + if not fn: + log.critical("Unknown command received from the monitor: %r", + data.strip()) + else: + try: + response = fn(arguments) + except: + log.exception( + "Pipe command handler exception occurred (command " + "%s args %r).", command, arguments + ) + + return response + +class Analyzer(object): + """Cuckoo Windows Analyzer. + + This class handles the initialization and execution of the analysis + procedure, including handling of the pipe server, the auxiliary modules and + the analysis packages. + """ + + def __init__(self): + self.config = None + self.target = None + self.do_run = True + self.time_counter = 0 + + self.process_lock = threading.Lock() + self.default_dll = None + self.pid = os.getpid() + self.ppid = Process(pid=self.pid).get_parent_pid() + self.files = Files() + self.process_list = ProcessList() + self.package = None + + self.reboot = [] + + def get_pipe_path(self, name): + """Return \\\\.\\PIPE on Windows XP and \\??\\PIPE elsewhere.""" + version = sys.getwindowsversion() + if version.major == 5 and version.minor == 1: + return "\\\\.\\PIPE\\%s" % name + return "\\??\\PIPE\\%s" % name + + def prepare(self): + """Prepare env for analysis.""" + # Get SeDebugPrivilege for the Python process. It will be needed in + # order to perform the injections. + grant_privilege("SeDebugPrivilege") + grant_privilege("SeLoadDriverPrivilege") + + # Initialize logging. + init_logging() + + # Parse the analysis configuration file generated by the agent. + self.config = Config(cfg="analysis.conf") + + # Pass the configuration through to the Process class. + Process.set_config(self.config) + + # Set virtual machine clock. + set_clock(datetime.datetime.strptime( + self.config.clock, "%Y%m%dT%H:%M:%S" + )) + + # Set the default DLL to be used for this analysis. + self.default_dll = self.config.options.get("dll") + + # If a pipe name has not set, then generate a random one. + self.config.pipe = self.get_pipe_path( + self.config.options.get("pipe", random_string(16, 32)) + ) + + # Generate a random name for the logging pipe server. + self.config.logpipe = self.get_pipe_path(random_string(16, 32)) + + # Initialize and start the Command Handler pipe server. This is going + # to be used for communicating with the monitored processes. + self.command_pipe = PipeServer( + PipeDispatcher, self.config.pipe, message=True, + dispatcher=CommandPipeHandler(self) + ) + self.command_pipe.daemon = True + self.command_pipe.start() + + # Initialize and start the Log Pipe Server - the log pipe server will + # open up a pipe that monitored processes will use to send logs to + # before they head off to the host machine. + destination = self.config.ip, self.config.port + self.log_pipe_server = PipeServer( + PipeForwarder, self.config.logpipe, destination=destination + ) + self.log_pipe_server.daemon = True + self.log_pipe_server.start() + + # We update the target according to its category. If it's a file, then + # we store the target path. + if self.config.category == "file": + self.target = os.path.join( + os.environ["TEMP"], self.config.file_name + ) + elif self.config.category == "archive": + zip_path = os.path.join(os.environ["TEMP"], self.config.file_name) + zipfile.ZipFile(zip_path).extractall(os.environ["TEMP"]) + self.target = os.path.join( + os.environ["TEMP"], self.config.options["filename"] + ) + # If it's a URL, well.. we store the URL. + else: + self.target = self.config.target + + def stop(self): + """Allow an auxiliary module to stop the analysis.""" + self.do_run = False + + def complete(self): + """End analysis.""" + # Stop the Pipe Servers. + self.command_pipe.stop() + self.log_pipe_server.stop() + + # Cleanly close remaining connections + disconnect_pipes() + disconnect_logger() + + def run(self): + """Run analysis. + @return: operation status. + """ + self.prepare() + self.path = os.getcwd() + + log.debug("Starting analyzer from: %s", self.path) + log.debug("Pipe server name: %s", self.config.pipe) + log.debug("Log pipe server name: %s", self.config.logpipe) + + # If no analysis package was specified at submission, we try to select + # one automatically. + if not self.config.package: + log.debug( + "No analysis package specified, trying to detect " + "it automagically." + ) + + # If the analysis target is a file, we choose the package according + # to the file format. + if self.config.category == "file": + package = choose_package( + self.config.file_type, self.config.file_name, + self.config.pe_exports.split(",") + ) + # If it's an URL, we'll just use the default Internet Explorer + # package. + else: + package = "ie" + + # If we weren't able to automatically determine the proper package, + # we need to abort the analysis. + if not package: + raise CuckooError("No valid package available for file " + "type: {0}".format(self.config.file_type)) + + log.info("Automatically selected analysis package \"%s\"", package) + # Otherwise just select the specified package. + else: + package = self.config.package + + # Generate the package path. + package_name = "modules.packages.%s" % package + + # Try to import the analysis package. + try: + __import__(package_name, globals(), locals(), ["dummy"], -1) + # If it fails, we need to abort the analysis. + except ImportError: + raise CuckooError("Unable to import package \"{0}\", does " + "not exist.".format(package_name)) + + # Initialize the package parent abstract. + Package() + + # Enumerate the abstract subclasses. + try: + package_class = Package.__subclasses__()[0] + except IndexError as e: + raise CuckooError("Unable to select package class " + "(package={0}): {1}".format(package_name, e)) + + # Initialize the analysis package. + self.package = package_class(self.config.options, analyzer=self) + + # Move the sample to the current working directory as provided by the + # task - one is able to override the starting path of the sample. + # E.g., for some samples it might be useful to run from %APPDATA% + # instead of %TEMP%. + if self.config.category == "file": + self.target = self.package.move_curdir(self.target) + + # Initialize Auxiliary modules + Auxiliary() + prefix = auxiliary.__name__ + "." + for loader, name, ispkg in pkgutil.iter_modules(auxiliary.__path__, prefix): + if ispkg: + continue + + # Import the auxiliary module. + try: + __import__(name, globals(), locals(), ["dummy"], -1) + except ImportError as e: + log.warning("Unable to import the auxiliary module " + "\"%s\": %s", name, e) + + # Walk through the available auxiliary modules. + aux_enabled, aux_avail = [], [] + for module in Auxiliary.__subclasses__(): + # Try to start the auxiliary module. + try: + aux = module(options=self.config.options, analyzer=self) + aux_avail.append(aux) + aux.init() + aux.start() + except (NotImplementedError, AttributeError): + log.exception( + "Auxiliary module %s was not implemented", module.__name__ + ) + except CuckooDisableModule: + continue + except Exception as e: + log.exception( + "Cannot execute auxiliary module %s: %s", + module.__name__, e + ) + else: + log.debug("Started auxiliary module %s", + module.__name__) + aux_enabled.append(aux) + + # Inform zer0m0n of the ResultServer address. + zer0m0n.resultserver(self.config.ip, self.config.port) + + # Forward the command pipe and logpipe names on to zer0m0n. + zer0m0n.cmdpipe(self.config.pipe) + zer0m0n.channel(self.config.logpipe) + + # Hide the Cuckoo Analyzer & Cuckoo Agent. + zer0m0n.hidepid(self.pid) + zer0m0n.hidepid(self.ppid) + + # Initialize zer0m0n with our compiled Yara rules. + zer0m0n.yarald("bin/rules.yarac") + + # Propagate the requested dump interval, if set. + zer0m0n.dumpint(int(self.config.options.get("dumpint", "0"))) + + # Start analysis package. If for any reason, the execution of the + # analysis package fails, we have to abort the analysis. + pids = self.package.start(self.target) + + # If the analysis package returned a list of process identifiers, we + # add them to the list of monitored processes and enable the process monitor. + if pids: + self.process_list.add_pids(pids) + pid_check = True + + # If the package didn't return any process ID (for example in the case + # where the package isn't enabling any behavioral analysis), we don't + # enable the process monitor. + else: + log.info("No process IDs returned by the package, running " + "for the full timeout.") + pid_check = False + + # Check in the options if the user toggled the timeout enforce. If so, + # we need to override pid_check and disable process monitor. + if self.config.enforce_timeout: + log.info("Enabled timeout enforce, running for the full timeout.") + pid_check = False + + while self.do_run: + self.time_counter += 1 + if self.time_counter == int(self.config.timeout): + log.info("Analysis timeout hit, terminating analysis.") + break + + # If the process lock is locked, it means that something is + # operating on the list of monitored processes. Therefore we + # cannot proceed with the checks until the lock is released. + if self.process_lock.locked(): + KERNEL32.Sleep(1000) + continue + + try: + # If the process monitor is enabled we start checking whether + # the monitored processes are still alive. + if pid_check: + # We also track the PIDs provided by zer0m0n. + self.process_list.add_pids(zer0m0n.getpids()) + + for pid in self.process_list.pids: + if not Process(pid=pid).is_alive(): + log.info("Process with pid %s has terminated", pid) + self.process_list.remove_pid(pid) + + # If none of the monitored processes are still alive, we + # can terminate the analysis. + if not self.process_list.pids: + log.info("Process list is empty, " + "terminating analysis.") + break + + # Update the list of monitored processes available to the + # analysis package. It could be used for internal + # operations within the module. + self.package.set_pids(self.process_list.pids) + + try: + # The analysis packages are provided with a function that + # is executed at every loop's iteration. If such function + # returns False, it means that it requested the analysis + # to be terminate. + if not self.package.check(): + log.info("The analysis package requested the " + "termination of the analysis.") + break + + # If the check() function of the package raised some exception + # we don't care, we can still proceed with the analysis but we + # throw a warning. + except Exception as e: + log.warning("The package \"%s\" check function raised " + "an exception: %s", package_name, e) + finally: + # Zzz. + KERNEL32.Sleep(1000) + + if not self.do_run: + log.debug("The analyzer has been stopped on request by an " + "auxiliary module.") + + # Create the shutdown mutex. + KERNEL32.CreateMutexA(None, False, SHUTDOWN_MUTEX) + + try: + # Before shutting down the analysis, the package can perform some + # final operations through the finish() function. + self.package.finish() + except Exception as e: + log.warning("The package \"%s\" finish function raised an " + "exception: %s", package_name, e) + + try: + # Upload files the package created to package_files in the + # results folder. + for path, name in self.package.package_files() or []: + upload_to_host(path, os.path.join("package_files", name)) + except Exception as e: + log.warning("The package \"%s\" package_files function raised an " + "exception: %s", package_name, e) + + # Terminate the Auxiliary modules. + for aux in aux_enabled: + try: + aux.stop() + except (NotImplementedError, AttributeError): + continue + except Exception as e: + log.warning("Cannot terminate auxiliary module %s: %s", + aux.__class__.__name__, e) + + if self.config.terminate_processes: + # Try to terminate remaining active processes. + log.info("Terminating remaining processes before shutdown.") + + for pid in self.process_list.pids: + proc = Process(pid=pid) + if proc.is_alive(): + try: + proc.terminate() + except: + continue + + # Run the finish callback of every available Auxiliary module. + for aux in aux_avail: + try: + aux.finish() + except (NotImplementedError, AttributeError): + continue + except Exception as e: + log.warning("Exception running finish callback of auxiliary " + "module %s: %s", aux.__class__.__name__, e) + + # Dump all the notified files. + self.files.dump_files() + + # Hell yeah. + log.info("Analysis completed.") + return True + +if __name__ == "__main__": + success = False + error = "" + + try: + # Initialize the main analyzer class. + analyzer = Analyzer() + + # Run it and wait for the response. + success = analyzer.run() + + data = { + "status": "complete", + "description": success, + } + # This is not likely to happen. + except KeyboardInterrupt: + error = "Keyboard Interrupt" + + # If the analysis process encountered a critical error, it will raise a + # CuckooError exception, which will force the termination of the analysis. + # Notify the agent of the failure. Also catch unexpected exceptions. + except Exception as e: + # Store the error. + error_exc = traceback.format_exc() + error = "%s\n%s" % (e, error_exc) + + # Just to be paranoid. + if len(log.handlers): + log.exception(error_exc) + else: + sys.stderr.write("{0}\n".format(error_exc)) + + data = { + "status": "exception", + "description": error_exc, + } + finally: + try: + # Let's invoke the completion procedure. + analyzer.complete() + except Exception as e: + complete_excp = traceback.format_exc() + data["status"] = "exception" + if "description" in data: + data["description"] += "%s\n%s" % ( + data["description"], complete_excp + ) + else: + data["description"] = complete_excp + + # Report that we're finished. First try with the XML RPC thing and + # if that fails, attempt the new Agent. + try: + server = xmlrpclib.Server("http://127.0.0.1:8000") + server.complete(success, error, "unused_path") + except Exception as e: + urllib2.urlopen("http://127.0.0.1:8000/status", + urllib.urlencode(data)).read() diff --git a/cuckoo/data/analyzer/windows/bin/execsc.exe b/cuckoo/data/analyzer/windows/bin/execsc.exe new file mode 100755 index 0000000000..6017f565a7 Binary files /dev/null and b/cuckoo/data/analyzer/windows/bin/execsc.exe differ diff --git a/cuckoo/data/analyzer/windows/lib/__init__.py b/cuckoo/data/analyzer/windows/lib/__init__.py new file mode 100644 index 0000000000..57edaa85f5 --- /dev/null +++ b/cuckoo/data/analyzer/windows/lib/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. diff --git a/cuckoo/data/analyzer/windows/lib/api/__init__.py b/cuckoo/data/analyzer/windows/lib/api/__init__.py new file mode 100644 index 0000000000..57edaa85f5 --- /dev/null +++ b/cuckoo/data/analyzer/windows/lib/api/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. diff --git a/cuckoo/data/analyzer/windows/lib/api/process.py b/cuckoo/data/analyzer/windows/lib/api/process.py new file mode 100644 index 0000000000..ae7359795a --- /dev/null +++ b/cuckoo/data/analyzer/windows/lib/api/process.py @@ -0,0 +1,586 @@ +# Copyright (C) 2012-2013 Claudio Guarnieri. +# Copyright (C) 2014-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import os +import logging +import random +import subprocess +import tempfile + +from ctypes import ( + c_ulong, create_string_buffer, c_int, c_uint16, c_uint, c_wchar_p, + c_void_p, sizeof, byref, Structure, cast +) +from _subprocess import STARTF_USESTDHANDLES + +from lib.common.constants import SHUTDOWN_MUTEX +from lib.common.defines import KERNEL32, NTDLL, SYSTEM_INFO, STILL_ACTIVE +from lib.common.defines import THREAD_ALL_ACCESS, PROCESS_ALL_ACCESS +from lib.common.exceptions import CuckooError +from lib.common.results import upload_to_host +from lib.core.ioctl import zer0m0n + +log = logging.getLogger(__name__) + +def spCreateProcessW(application_name, command_line, process_attributes, + thread_attributes, inherit_handles, creation_flags, + environment, current_directory, startup_info): + class STARTUPINFO(Structure): + _fields_ = [ + ("cb", c_uint), + ("reserved1", c_void_p), + ("desktop", c_void_p), + ("title", c_void_p), + ("unused1", c_uint * 7), + ("flags", c_uint), + ("show_window", c_uint16), + ("reserved2", c_uint16), + ("reserved3", c_void_p), + ("std_input", c_void_p), + ("std_output", c_void_p), + ("std_error", c_void_p), + ] + + class PROCESS_INFORMATION(Structure): + _fields_ = [ + ("process_handle", c_void_p), + ("thread_handle", c_void_p), + ("process_identifier", c_uint), + ("thread_identifier", c_uint), + ] + + class Handle(int): + def Close(self): + KERNEL32.CloseHandle(self) + + if environment: + environment = "\x00".join( + "%s=%s" % (k, v) for k, v in environment.items() + ) + "\x00\x00" + + si = STARTUPINFO() + si.cb = sizeof(STARTUPINFO) + + if startup_info: + si.flags = startup_info.dwFlags + si.show_window = startup_info.wShowWindow + + if si.flags & STARTF_USESTDHANDLES: + si.std_input = cast(int(startup_info.hStdInput), c_void_p) + si.std_output = cast(int(startup_info.hStdOutput), c_void_p) + si.std_error = cast(int(startup_info.hStdError), c_void_p) + + pi = PROCESS_INFORMATION() + + result = KERNEL32.CreateProcessW( + application_name, command_line, None, None, inherit_handles, + creation_flags, environment, current_directory, byref(si), byref(pi) + ) + if not result: + # TODO We'll just assume this is correct for now. + raise WindowsError(KERNEL32.GetLastError()) + + return ( + Handle(pi.process_handle), Handle(pi.thread_handle), + pi.process_identifier, pi.thread_identifier + ) + +# We patch Python 2.7's native .CreateProcess method to be unicode-aware. +subprocess._subprocess.CreateProcess = spCreateProcessW +KERNEL32.CreateProcessW.argtypes = ( + c_wchar_p, c_wchar_p, c_void_p, c_void_p, c_uint, c_uint, c_void_p, + c_wchar_p, c_void_p, c_void_p +) + +def subprocess_checkcall(args, env=None): + return subprocess.check_call( + args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=env, + ) + +def subprocess_checkoutput(args, env=None): + return subprocess.check_output( + args, stdin=subprocess.PIPE, stderr=subprocess.PIPE, env=env, + ) + +class Process(object): + """Windows process.""" + first_process = True + config = None + + # Keeps track of the dump memory index for a particular process as in + # theory, and will be useful later, we may want to dump one process + # multiple times. + dumpmem = {} + + def __init__(self, pid=None, tid=None, process_name=None): + """ + @param pid: process identifier. + @param tid: thread identifier. + @param process_name: process name. + """ + self.pid = pid + self.tid = tid + self.process_name = process_name + + @staticmethod + def set_config(config): + """Set the analyzer configuration once.""" + Process.config = config + + def get_system_info(self): + """Get system information.""" + self.system_info = SYSTEM_INFO() + KERNEL32.GetSystemInfo(byref(self.system_info)) + + def open_process(self): + """Open a process handle.""" + return KERNEL32.OpenProcess(PROCESS_ALL_ACCESS, False, self.pid) + + def open_thread(self): + """Open a thread handle.""" + return KERNEL32.OpenThread(THREAD_ALL_ACCESS, False, self.tid) + + def exit_code(self): + """Get process exit code. + @return: exit code value. + """ + process_handle = self.open_process() + + exit_code = c_ulong(0) + KERNEL32.GetExitCodeProcess(process_handle, byref(exit_code)) + KERNEL32.CloseHandle(process_handle) + + return exit_code.value + + def get_filepath(self): + """Get process image file path. + @return: decoded file path. + """ + process_handle = self.open_process() + + NT_SUCCESS = lambda val: val >= 0 + + pbi = create_string_buffer(200) + size = c_int() + + # Set return value to signed 32bit integer. + NTDLL.NtQueryInformationProcess.restype = c_int + + ret = NTDLL.NtQueryInformationProcess(process_handle, + 27, + byref(pbi), + sizeof(pbi), + byref(size)) + + KERNEL32.CloseHandle(process_handle) + + if NT_SUCCESS(ret) and size.value > 8: + try: + fbuf = pbi.raw[8:] + fbuf = fbuf[:fbuf.find("\x00\x00")+1] + return fbuf.decode("utf16", errors="ignore") + except: + return "" + + return "" + + def is_alive(self): + """Process is alive? + @return: process status. + """ + return self.exit_code() == STILL_ACTIVE + + def get_parent_pid(self): + """Get the Parent Process ID.""" + class PROCESS_BASIC_INFORMATION(Structure): + _fields_ = [ + ("ExitStatus", c_void_p), + ("PebBaseAddress", c_void_p), + ("AffinityMask", c_void_p), + ("BasePriority", c_void_p), + ("UniqueProcessId", c_void_p), + ("InheritedFromUniqueProcessId", c_void_p), + ] + + NT_SUCCESS = lambda val: val >= 0 + + pbi = PROCESS_BASIC_INFORMATION() + size = c_int() + + # Set return value to signed 32bit integer. + NTDLL.NtQueryInformationProcess.restype = c_int + + process_handle = self.open_process() + ret = NTDLL.NtQueryInformationProcess( + process_handle, 0, byref(pbi), sizeof(pbi), byref(size) + ) + KERNEL32.CloseHandle(process_handle) + + if NT_SUCCESS(ret) and size.value == sizeof(pbi): + return pbi.InheritedFromUniqueProcessId + + def _encode_args(self, args): + """Convert a list of arguments to a string that can be passed along + on the command-line. + @param args: list of arguments + @return: the command-line equivalent + """ + ret = [] + for line in args: + if " " in line: + ret.append('"%s"' % line) + else: + ret.append(line) + return " ".join(ret) + + def is32bit(self, pid=None, process_name=None, path=None): + """Is a PE file 32-bit or does a process identifier belong to a + 32-bit process. + @param pid: process identifier. + @param process_name: process name. + @param path: path to a PE file. + @return: boolean or exception. + """ + count = (pid is None) + (process_name is None) + (path is None) + if count != 2: + raise CuckooError("Invalid usage of is32bit, only one identifier " + "should be specified") + + is32bit_exe = os.path.join("bin", "is32bit.exe") + + if pid: + args = [is32bit_exe, "-p", "%s" % pid] + elif process_name: + args = [is32bit_exe, "-n", process_name] + + # If we're running a 32-bit Python in a 64-bit Windows system and the + # path points to System32, then we hardcode it as being a 64-bit + # binary. (To be fair, a 64-bit Python on 64-bit Windows would also + # make the System32 binary 64-bit). + elif os.path.isdir("C:\\Windows\\Sysnative") and \ + path.lower().startswith("c:\\windows\\system32"): + return False + elif not os.path.exists(path): + raise CuckooError("File not found: %s" % path) + else: + args = [is32bit_exe, "-f", path] + + try: + bitsize = int(subprocess_checkoutput(args)) + except subprocess.CalledProcessError as e: + raise CuckooError("Error returned by is32bit: %s" % e.output) + + return bitsize == 32 + + def execute(self, path, args=None, dll=None, free=False, curdir=None, + source=None, mode=None, maximize=False, env=None, + trigger=None): + """Execute sample process. + @param path: sample path. + @param args: process args. + @param dll: dll path. + @param free: do not inject our monitor. + @param curdir: current working directory. + @param source: process identifier or process name which will + become the parent process for the new process. + @param mode: monitor mode - which functions to instrument. + @param maximize: whether the GUI should be maximized. + @param env: environment variables. + @param trigger: trigger to indicate analysis start + @return: operation status. + """ + if not os.access(path, os.X_OK): + log.error( + "Unable to access file at path %r, execution aborted!", path + ) + return False + + is32bit = self.is32bit(path=path) + + if source: + if isinstance(source, (int, long)) or source.isdigit(): + inject_is32bit = self.is32bit(pid=int(source)) + else: + inject_is32bit = self.is32bit(process_name=source) + else: + inject_is32bit = is32bit + + if inject_is32bit: + inject_exe = os.path.join("bin", "inject-x86.exe") + else: + inject_exe = os.path.join("bin", "inject-x64.exe") + + argv = [ + inject_exe, + "--app", path, + "--only-start", + ] + + if args: + argv += ["--args", self._encode_args(args)] + + if curdir: + argv += ["--curdir", curdir] + + if source: + if isinstance(source, (int, long)) or source.isdigit(): + argv += ["--from", "%s" % source] + else: + argv += ["--from-process", source] + + if maximize: + argv += ["--maximize"] + + try: + output = subprocess_checkoutput(argv, env) + self.pid, self.tid = map(int, output.split()) + except subprocess.CalledProcessError as e: + log.error( + "Failed to execute process from path %r with " + "arguments %r (Error: %s)", path, argv, e + ) + return False + + # Report this PID to the kernel driver (if present). + zer0m0n.addpid(self.pid) + + # With .NET for AnyCPU target, a 32-bit PE file can start a 64-bit + # process. Recheck the process bitness here after startup to make sure + # injection works. + is32bit = self.is32bit(self.pid) + + if not dll: + if is32bit: + dll = "monitor-x86.dll" + else: + dll = "monitor-x64.dll" + + dllpath = os.path.abspath(os.path.join("bin", dll)) + + if not os.path.exists(dllpath): + log.warning("No valid DLL specified to be injected, " + "injection aborted.") + return False + + if is32bit: + inject_exe = os.path.join("bin", "inject-x86.exe") + else: + inject_exe = os.path.join("bin", "inject-x64.exe") + + argv = [ + inject_exe, + "--resume-thread", + "--pid", "%s" % self.pid, + "--tid", "%s" % self.tid, + ] + + if free: + argv.append("--free") + else: + argv += [ + "--apc", + "--dll", dllpath, + "--config", self.drop_config(mode=mode, trigger=trigger), + ] + + try: + subprocess_checkoutput(argv, env) + except subprocess.CalledProcessError as e: + log.error( + "Failed to execute process from path %r with " + "arguments %r (Error: %s)", path, argv, e + ) + return False + + log.info("Successfully executed process from path %r with " + "arguments %r and pid %d", path, args or "", self.pid) + return True + + def terminate(self): + """Terminate process. + @return: operation status. + """ + process_handle = self.open_process() + + ret = KERNEL32.TerminateProcess(process_handle, 1) + KERNEL32.CloseHandle(process_handle) + + if ret: + log.info("Successfully terminated process with pid %d.", self.pid) + return True + else: + log.error("Failed to terminate process with pid %d.", self.pid) + return False + + def inject(self, dll=None, apc=False, track=True, mode=None): + """Inject our monitor into the specified process. + @param dll: Cuckoo DLL path. + @param apc: Use APC injection. + @param track: Track this process in the analyzer. + @param mode: Monitor mode - which functions to instrument. + """ + if not self.pid and not self.process_name: + log.warning("No valid pid or process name specified, " + "injection aborted.") + return False + + # Only check whether the process is still alive when it's identified + # by a process identifier. Not when it's identified by a process name. + if not self.process_name and not self.is_alive(): + log.warning("The process with pid %s is not alive, " + "injection aborted", self.pid) + return False + + if self.process_name: + is32bit = self.is32bit(process_name=self.process_name) + elif self.pid: + is32bit = self.is32bit(pid=self.pid) + + if not dll: + if is32bit: + dll = "monitor-x86.dll" + else: + dll = "monitor-x64.dll" + + dllpath = os.path.abspath(os.path.join("bin", dll)) + if not os.path.exists(dllpath): + log.warning("No valid DLL specified to be injected in process " + "with pid %s / process name %s, injection aborted.", + self.pid, self.process_name) + return False + + if is32bit: + inject_exe = os.path.join("bin", "inject-x86.exe") + else: + inject_exe = os.path.join("bin", "inject-x64.exe") + + args = [ + inject_exe, + "--dll", dllpath, + "--config", self.drop_config(track=track, mode=mode), + ] + + if self.pid: + args += ["--pid", "%s" % self.pid] + elif self.process_name: + args += ["--process-name", self.process_name] + + if apc: + args += ["--apc", "--tid", "%s" % self.tid] + else: + args += ["--crt"] + + try: + subprocess_checkcall(args) + except Exception: + log.error("Failed to inject %s-bit process with pid %s and " + "process name %s", 32 if is32bit else 64, self.pid, + self.process_name) + return False + + return True + + def drop_config(self, track=True, mode=None, trigger=None): + """Helper function to drop the configuration for a new process.""" + fd, config_path = tempfile.mkstemp() + + # The first time we come up with a random startup-time. + if Process.first_process: + # This adds 1 up to 30 times of 20 minutes to the startup + # time of the process, therefore bypassing anti-vm checks + # which check whether the VM has only been up for <10 minutes. + Process.startup_time = random.randint(1, 30) * 20 * 60 * 1000 + + lines = { + "pipe": self.config.pipe, + "logpipe": self.config.logpipe, + "analyzer": os.getcwd(), + "first-process": "1" if Process.first_process else "0", + "startup-time": Process.startup_time, + "shutdown-mutex": SHUTDOWN_MUTEX, + "force-sleepskip": self.config.options.get("force-sleepskip", "0"), + "track": "1" if track else "0", + "mode": mode or "", + "disguise": self.config.options.get("disguise", "0"), + "pipe-pid": "1", + "trigger": (trigger or "").encode("utf8"), + } + + for key, value in lines.items(): + os.write(fd, "%s=%s\n" % (key, value)) + + os.close(fd) + + # Only change the first_process attribute for processes that we + # "track", i.e., the lsass.exe injection doesn't count. + if track: + Process.first_process = False + + return config_path + + def dump_memory(self, addr=None, length=None): + """Dump process memory, optionally target only a certain memory range. + @return: operation status. + """ + if not self.pid: + log.warning("No valid pid specified, memory dump aborted") + return False + + if not self.is_alive(): + log.warning("The process with pid %d is not alive, memory " + "dump aborted", self.pid) + return False + + if self.is32bit(pid=self.pid): + inject_exe = os.path.join("bin", "inject-x86.exe") + else: + inject_exe = os.path.join("bin", "inject-x64.exe") + + # Take the memory dump. + dump_path = tempfile.mktemp() + + try: + args = [ + inject_exe, + "--pid", "%s" % self.pid, + "--dump", dump_path, + ] + + # Restrict to a certain memory block. + if addr and length: + args += [ + "--dump-block", + "0x%x" % addr, + "%s" % length, + ] + + subprocess_checkcall(args) + except subprocess.CalledProcessError: + log.error("Failed to dump memory of %d-bit process with pid %d.", + 32 if self.is32bit(pid=self.pid) else 64, self.pid) + return + + # Calculate the next index and send the process memory dump over to + # the host. Keep in mind that one process may have multiple process + # memory dumps in the future. + idx = self.dumpmem[self.pid] = self.dumpmem.get(self.pid, 0) + 1 + + if addr and length: + file_name = os.path.join( + "memory", "block-%s-0x%x-%s.dmp" % (self.pid, addr, idx) + ) + else: + file_name = os.path.join("memory", "%s-%s.dmp" % (self.pid, idx)) + + upload_to_host(dump_path, file_name) + os.unlink(dump_path) + + log.info("Memory dump of process with pid %d completed", self.pid) + return True + + # The dump_memory_block functionality has been integrated with the + # dump_memory function, this alias is just for backwards compatibility. + dump_memory_block = dump_memory diff --git a/cuckoo/data/analyzer/windows/lib/api/screenshot.py b/cuckoo/data/analyzer/windows/lib/api/screenshot.py new file mode 100644 index 0000000000..9409e5ccc2 --- /dev/null +++ b/cuckoo/data/analyzer/windows/lib/api/screenshot.py @@ -0,0 +1,82 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import math + +try: + import ImageChops + import ImageGrab + import ImageDraw + HAVE_PIL = True +except: + try: + from PIL import ImageChops + from PIL import ImageGrab + from PIL import ImageDraw + HAVE_PIL = True + except: + HAVE_PIL = False + +class Screenshot: + """Get screenshots.""" + + def _draw_rectangle(self, img, xy): + """Draw a black rectangle. + @param img: PIL Image object + @param xy: Coordinates as refined in PIL rectangle() doc + @return: Image with black rectangle + """ + dr = ImageDraw.Draw(img) + dr.rectangle(xy, fill="black", outline="black") + return img + + def have_pil(self): + """Is Python Image Library installed? + @return: installed status. + """ + return HAVE_PIL + + def equal(self, img1, img2, skip_area=None): + """Compares two screenshots using Root-Mean-Square Difference (RMS). + @param img1: screenshot to compare. + @param img2: screenshot to compare. + @return: equal status. + """ + if not HAVE_PIL: + return None + + # Trick to avoid getting a lot of screen shots only because the time in the windows + # clock is changed. + # We draw a black rectangle on the coordinates where the clock is locates, and then + # run the comparison. + # NOTE: the coordinates are changing with VM screen resolution. + if skip_area: + # Copying objects to draw in another object. + img1 = img1.copy() + img2 = img2.copy() + # Draw a rectangle to cover windows clock. + for img in (img1, img2): + self._draw_rectangle(img, skip_area) + + # To get a measure of how similar two images are, we use + # root-mean-square (RMS). If the images are exactly identical, + # this value is zero. + diff = ImageChops.difference(img1, img2) + h = diff.histogram() + sq = (value * ((idx % 256)**2) for idx, value in enumerate(h)) + sum_of_squares = sum(sq) + rms = math.sqrt(sum_of_squares/float(img1.size[0] * img1.size[1])) + + # Might need to tweak the threshold. + return rms < 8 + + def take(self): + """Take a screenshot. + @return: screenshot or None. + """ + if not HAVE_PIL: + return None + + return ImageGrab.grab() diff --git a/cuckoo/data/analyzer/windows/lib/common/__init__.py b/cuckoo/data/analyzer/windows/lib/common/__init__.py new file mode 100644 index 0000000000..57edaa85f5 --- /dev/null +++ b/cuckoo/data/analyzer/windows/lib/common/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. diff --git a/cuckoo/data/analyzer/windows/lib/common/abstracts.py b/cuckoo/data/analyzer/windows/lib/common/abstracts.py new file mode 100644 index 0000000000..e47109f3d6 --- /dev/null +++ b/cuckoo/data/analyzer/windows/lib/common/abstracts.py @@ -0,0 +1,201 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import glob +import os + +from _winreg import CreateKey, SetValueEx, CloseKey, REG_DWORD, REG_SZ + +from lib.api.process import Process +from lib.common.decide import dump_memory +from lib.common.exceptions import CuckooPackageError + +class Package(object): + """Base abstract analysis package.""" + PATHS = [] + REGKEYS = [] + + def __init__(self, options={}, analyzer=None): + """@param options: options dict.""" + self.options = options + self.analyzer = analyzer + self.pids = [] + + # Fetch the current working directory, defaults to $TEMP. + if "curdir" in options: + self.curdir = os.path.expandvars(options["curdir"]) + else: + self.curdir = os.getenv("TEMP") + + def set_pids(self, pids): + """Update list of monitored PIDs in the package context. + @param pids: list of pids. + """ + self.pids = pids + + def start(self, target): + """Run analysis package. + @raise NotImplementedError: this method is abstract. + """ + raise NotImplementedError + + def check(self): + """Check.""" + return True + + def enum_paths(self): + """Enumerate available paths.""" + basepaths = { + "System32": [ + os.path.join(os.getenv("SystemRoot"), "System32"), + os.path.join(os.getenv("SystemRoot"), "SysWOW64"), + ], + "ProgramFiles": [ + os.getenv("ProgramFiles").replace(" (x86)", ""), + os.getenv("ProgramFiles(x86)"), + ], + "HomeDrive": [ + # os.path.join() doesn't work well if you give it just "C:" + # so manually append a backslash. + os.getenv("HomeDrive") + "\\", + ], + } + + for path in self.PATHS: + basedir = path[0] + for basepath in basepaths.get(basedir, [basedir]): + if not basepath or not os.path.isdir(basepath): + continue + + yield os.path.join(basepath, *path[1:]) + + def get_path(self, application): + """Search for the application in all available paths. + @param applicaiton: application executable name + @return: executable path + """ + for path in self.enum_paths(): + if os.path.isfile(path): + return path + + raise CuckooPackageError("Unable to find any %s executable." % + application) + + def get_path_glob(self, application): + """Search for the application in all available paths with glob support. + @param applicaiton: application executable name + @return: executable path + """ + for path in self.enum_paths(): + for path in glob.iglob(path): + if os.path.isfile(path): + return path + + raise CuckooPackageError("Unable to find any %s executable." % + application) + + def move_curdir(self, filepath): + """Move a file to the current working directory so it can be executed + from there. + @param filepath: the file to be moved + @return: the new filepath + """ + outpath = os.path.join(self.curdir, os.path.basename(filepath)) + os.rename(filepath, outpath) + return outpath + + def init_regkeys(self, regkeys): + """Initialize the registry to avoid annoying popups, configure + settings, etc. + @param regkeys: the root keys, subkeys, and key/value pairs. + """ + for rootkey, subkey, values in regkeys: + key_handle = CreateKey(rootkey, subkey) + + for key, value in values.items(): + if isinstance(value, str): + SetValueEx(key_handle, key, 0, REG_SZ, value) + elif isinstance(value, int): + SetValueEx(key_handle, key, 0, REG_DWORD, value) + elif isinstance(value, dict): + self.init_regkeys([ + [rootkey, "%s\\%s" % (subkey, key), value], + ]) + else: + raise CuckooPackageError("Invalid value type: %r" % value) + + CloseKey(key_handle) + + def execute(self, path, args, mode=None, maximize=False, env=None, + source=None, trigger=None): + """Start an executable for analysis. + @param path: executable path + @param args: executable arguments + @param mode: monitor mode - which functions to instrument + @param maximize: whether the GUI should start maximized + @param env: additional environment variables + @param source: parent process of our process + @param trigger: trigger to indicate analysis start + @return: process pid + """ + dll = self.options.get("dll") + free = self.options.get("free") + analysis = self.options.get("analysis") + + # Kernel analysis overrides the free argument. + if analysis == "kernel": + free = True + + source = source or self.options.get("from") + mode = mode or self.options.get("mode") + + if not trigger and self.options.get("trigger"): + if self.options["trigger"] == "exefile": + trigger = "file:%s" % path + + # Setup pre-defined registry keys. + self.init_regkeys(self.REGKEYS) + + p = Process() + if not p.execute(path=path, args=args, dll=dll, free=free, + curdir=self.curdir, source=source, mode=mode, + maximize=maximize, env=env, trigger=trigger): + raise CuckooPackageError( + "Unable to execute the initial process, analysis aborted." + ) + + return p.pid + + def package_files(self): + """Return a list of files to upload to host. + The list should be a list of tuples (, ). + (package_files is a folder that will be created in analysis folder). + """ + return None + + def finish(self): + """Finish run. + If specified to do so, this method dumps the memory of + all running processes. + """ + if self.options.get("procmemdump"): + for pid in self.pids: + dump_memory(pid) + + return True + +class Auxiliary(object): + def __init__(self, options={}, analyzer=None): + self.options = options + self.analyzer = analyzer + + def init(self): + pass + + def start(self): + pass + + def stop(self): + pass diff --git a/cuckoo/data/analyzer/windows/lib/common/constants.py b/cuckoo/data/analyzer/windows/lib/common/constants.py new file mode 100644 index 0000000000..9f788c8a03 --- /dev/null +++ b/cuckoo/data/analyzer/windows/lib/common/constants.py @@ -0,0 +1,8 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +from lib.common.rand import random_string + +SHUTDOWN_MUTEX = "Global\\" + random_string(6, 10) diff --git a/cuckoo/data/analyzer/windows/lib/common/decide.py b/cuckoo/data/analyzer/windows/lib/common/decide.py new file mode 100644 index 0000000000..e9ff95cd7c --- /dev/null +++ b/cuckoo/data/analyzer/windows/lib/common/decide.py @@ -0,0 +1,11 @@ +# Copyright (C) 2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +from lib.api.process import Process +from lib.core.ioctl import zer0m0n + +def dump_memory(pid): + """Dump process memory using zer0m0n if available, otherwise fallback.""" + if zer0m0n.dumpmem(pid) is False: + Process(pid=pid).dump_memory() diff --git a/cuckoo/data/analyzer/windows/lib/common/defines.py b/cuckoo/data/analyzer/windows/lib/common/defines.py new file mode 100644 index 0000000000..e9a9d89e16 --- /dev/null +++ b/cuckoo/data/analyzer/windows/lib/common/defines.py @@ -0,0 +1,215 @@ +# Copyright (C) 2011-2013 Claudio Guarnieri. +# Copyright (C) 2014-2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +from ctypes import * + +NTDLL = windll.ntdll +KERNEL32 = windll.kernel32 +ADVAPI32 = windll.advapi32 +USER32 = windll.user32 +SHELL32 = windll.shell32 +PSAPI = windll.psapi + +BYTE = c_ubyte +WORD = c_ushort +DWORD = c_ulong +LONG = c_ulong +LPBYTE = POINTER(c_ubyte) +LPTSTR = POINTER(c_char) +PWSTR = c_wchar_p +HANDLE = c_void_p +PVOID = c_void_p +LPVOID = c_void_p +UINT_PTR = c_ulong +SIZE_T = c_ulong +HMODULE = c_void_p +NULL = c_int(0) + +DEBUG_PROCESS = 0x00000001 +CREATE_NEW_CONSOLE = 0x00000010 +CREATE_SUSPENDED = 0x00000004 +DBG_CONTINUE = 0x00010002 +INFINITE = 0xFFFFFFFF +PROCESS_ALL_ACCESS = 0x001F0FFF +THREAD_ALL_ACCESS = 0x001f03ff +TOKEN_ALL_ACCESS = 0x000F01FF +SE_PRIVILEGE_ENABLED = 0x00000002 +STILL_ACTIVE = 0x00000103 + +PAGE_EXECUTE_READWRITE = 0x00000040 +PAGE_EXECUTE = 0x00000010 +PAGE_EXECUTE_READ = 0x00000020 +PAGE_READONLY = 0x00000002 +PAGE_READWRITE = 0x00000004 + +MEM_COMMIT = 0x00001000 +MEM_RESERVE = 0x00002000 +MEM_DECOMMIT = 0x00004000 +MEM_RELEASE = 0x00008000 +MEM_RESET = 0x00080000 + +MEM_IMAGE = 0x01000000 +MEM_MAPPED = 0x00040000 +MEM_PRIVATE = 0x00020000 + +PAGE_NOACCESS = 0x00000001 +PAGE_READONLY = 0x00000002 +PAGE_READWRITE = 0x00000004 +PAGE_WRITECOPY = 0x00000008 +PAGE_EXECUTE = 0x00000010 +PAGE_EXECUTE_READ = 0x00000020 +PAGE_EXECUTE_READWRITE = 0x00000040 +PAGE_EXECUTE_WRITECOPY = 0x00000080 +PAGE_GUARD = 0x00000100 +PAGE_NOCACHE = 0x00000200 +PAGE_WRITECOMBINE = 0x00000400 + +PIPE_ACCESS_INBOUND = 0x00000001 +PIPE_ACCESS_DUPLEX = 0x00000003 +PIPE_TYPE_MESSAGE = 0x00000004 +PIPE_READMODE_MESSAGE = 0x00000002 +PIPE_WAIT = 0x00000000 +PIPE_UNLIMITED_INSTANCES = 0x000000ff +PIPE_TYPE_BYTE = 0x00000000 +PIPE_READMODE_BYTE = 0x00000000 +FILE_FLAG_WRITE_THROUGH = 0x80000000 +INVALID_HANDLE_VALUE = 0xffffffff +ERROR_BROKEN_PIPE = 0x0000006d +ERROR_MORE_DATA = 0x000000EA +ERROR_PIPE_CONNECTED = 0x00000217 + +WAIT_TIMEOUT = 0x00000102 + +FILE_ATTRIBUTE_HIDDEN = 0x00000002 + +WM_CLOSE = 0x00000010 +WM_GETTEXT = 0x0000000D +WM_GETTEXTLENGTH = 0x0000000E +BM_CLICK = 0x000000F5 + +SHARD_PATHA = 0x00000002 + +GENERIC_READ = 0x80000000 +GENERIC_WRITE = 0x40000000 +GENERIC_EXECUTE = 0x20000000 +GENERIC_ALL = 0x10000000 + +FILE_SHARE_READ = 0x00000001 +FILE_SHARE_WRITE = 0x00000002 +FILE_SHARE_DELETE = 0x00000004 + +CREATE_NEW = 1 +CREATE_ALWAYS = 2 +OPEN_EXISTING = 3 +OPEN_ALWAYS = 4 +TRUNCATE_EXISTING = 5 + +class STARTUPINFO(Structure): + _fields_ = [ + ("cb", DWORD), + ("lpReserved", LPTSTR), + ("lpDesktop", LPTSTR), + ("lpTitle", LPTSTR), + ("dwX", DWORD), + ("dwY", DWORD), + ("dwXSize", DWORD), + ("dwYSize", DWORD), + ("dwXCountChars", DWORD), + ("dwYCountChars", DWORD), + ("dwFillAttribute",DWORD), + ("dwFlags", DWORD), + ("wShowWindow", WORD), + ("cbReserved2", WORD), + ("lpReserved2", LPBYTE), + ("hStdInput", HANDLE), + ("hStdOutput", HANDLE), + ("hStdError", HANDLE), + ] + +class PROCESS_INFORMATION(Structure): + _fields_ = [ + ("hProcess", HANDLE), + ("hThread", HANDLE), + ("dwProcessId", DWORD), + ("dwThreadId", DWORD), + ] + +class LUID(Structure): + _fields_ = [ + ("LowPart", DWORD), + ("HighPart", LONG), + ] + +class LUID_AND_ATTRIBUTES(Structure): + _fields_ = [ + ("Luid", LUID), + ("Attributes", DWORD), + ] + +class TOKEN_PRIVILEGES(Structure): + _fields_ = [ + ("PrivilegeCount", DWORD), + ("Privileges", LUID_AND_ATTRIBUTES), + ] + +class MEMORY_BASIC_INFORMATION(Structure): + _fields_ = [ + ("BaseAddress", PVOID), + ("AllocationBase", PVOID), + ("AllocationProtect", DWORD), + ("RegionSize", SIZE_T), + ("State", DWORD), + ("Protect", DWORD), + ("Type", DWORD), + ] + +class PROC_STRUCT(Structure): + _fields_ = [ + ("wProcessorArchitecture", WORD), + ("wReserved", WORD), + ] + +class SYSTEM_INFO_UNION(Union): + _fields_ = [ + ("dwOemId", DWORD), + ("sProcStruc", PROC_STRUCT), + ] + +class SYSTEM_INFO(Structure): + _fields_ = [ + ("uSysInfo", SYSTEM_INFO_UNION), + ("dwPageSize", DWORD), + ("lpMinimumApplicationAddress", LPVOID), + ("lpMaximumApplicationAddress", LPVOID), + ("dwActiveProcessorMask", DWORD), + ("dwNumberOfProcessors", DWORD), + ("dwProcessorType", DWORD), + ("dwAllocationGranularity", DWORD), + ("wProcessorLevel", WORD), + ("wProcessorRevision", WORD), + ] + +class SYSTEMTIME(Structure): + _pack_ = 1 + _fields_ = [ + ("wYear", WORD), + ("wMonth", WORD), + ("wDayOfWeek", WORD), + ("wDay", WORD), + ("wHour", WORD), + ("wMinute", WORD), + ("wSecond", WORD), + ("wMilliseconds", WORD), + ] + +class UNICODE_STRING(Structure): + _fields_ = [ + ("Length", c_ushort), + ("MaximumLength", c_ushort), + ("Buffer", c_wchar_p), + ] + +EnumWindowsProc = WINFUNCTYPE(c_bool, POINTER(c_int), POINTER(c_int)) +EnumChildProc = WINFUNCTYPE(c_bool, POINTER(c_int), POINTER(c_int)) diff --git a/cuckoo/data/analyzer/windows/lib/common/exceptions.py b/cuckoo/data/analyzer/windows/lib/common/exceptions.py new file mode 100644 index 0000000000..c3e74767a9 --- /dev/null +++ b/cuckoo/data/analyzer/windows/lib/common/exceptions.py @@ -0,0 +1,13 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +class CuckooError(Exception): + pass + +class CuckooPackageError(Exception): + pass + +class CuckooDisableModule(CuckooError): + """Exception for disabling a module dynamically.""" diff --git a/cuckoo/data/analyzer/windows/lib/common/hashing.py b/cuckoo/data/analyzer/windows/lib/common/hashing.py new file mode 100644 index 0000000000..adbd321349 --- /dev/null +++ b/cuckoo/data/analyzer/windows/lib/common/hashing.py @@ -0,0 +1,22 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +BUFSIZE = 1024*1024 + + +def hash_file(method, path): + """Calculate a hash on a file by path. + @param method: callable hashing method + @param path: file path + @return: computed hash string + """ + f = open(path, "rb") + h = method() + while True: + buf = f.read(BUFSIZE) + if not buf: + break + h.update(buf) + return h.hexdigest() diff --git a/analyzer/windows/lib/common/rand.py b/cuckoo/data/analyzer/windows/lib/common/rand.py similarity index 100% rename from analyzer/windows/lib/common/rand.py rename to cuckoo/data/analyzer/windows/lib/common/rand.py diff --git a/cuckoo/data/analyzer/windows/lib/common/registry.py b/cuckoo/data/analyzer/windows/lib/common/registry.py new file mode 100644 index 0000000000..5c4dff1897 --- /dev/null +++ b/cuckoo/data/analyzer/windows/lib/common/registry.py @@ -0,0 +1,150 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import logging +import struct +import _winreg + +from ctypes import windll, POINTER, byref, pointer +from ctypes import c_ushort, c_wchar_p, c_void_p, create_string_buffer +from ctypes.wintypes import HANDLE, DWORD, LPCWSTR, ULONG, LONG + +from lib.common.defines import UNICODE_STRING + +log = logging.getLogger(__name__) + +RegOpenKeyExW = windll.advapi32.RegOpenKeyExW +RegOpenKeyExW.argtypes = HANDLE, LPCWSTR, DWORD, ULONG, POINTER(HANDLE) +RegOpenKeyExW.restype = LONG + +RegCreateKeyExW = windll.advapi32.RegCreateKeyExW +RegCreateKeyExW.argtypes = ( + HANDLE, LPCWSTR, DWORD, LPCWSTR, DWORD, DWORD, + DWORD, POINTER(HANDLE), POINTER(DWORD), +) +RegCreateKeyExW.restype = LONG + +RegQueryValueExW = windll.advapi32.RegQueryValueExW +RegQueryValueExW.argtypes = \ + HANDLE, LPCWSTR, POINTER(DWORD), POINTER(DWORD), c_void_p, POINTER(DWORD) +RegQueryValueExW.restype = LONG + +RegSetValueExW = windll.advapi32.RegSetValueExW +RegSetValueExW.argtypes = HANDLE, LPCWSTR, DWORD, DWORD, c_void_p, DWORD +RegSetValueExW.restype = LONG + +RegDeleteKeyW = windll.advapi32.RegDeleteKeyW +RegDeleteKeyW.argtypes = HANDLE, LPCWSTR +RegDeleteKeyW.restype = LONG + +NtRenameKey = windll.ntdll.NtRenameKey +NtRenameKey.argtypes = HANDLE, POINTER(UNICODE_STRING) + +RegCloseKey = windll.advapi32.RegCloseKey +RegCloseKey.argtypes = HANDLE, + +_rootkeys = { + "HKEY_LOCAL_MACHINE": _winreg.HKEY_LOCAL_MACHINE, + "HKEY_CURRENT_USER": _winreg.HKEY_CURRENT_USER, +} + +_regtypes = { + "REG_DWORD": _winreg.REG_DWORD, + "REG_SZ": _winreg.REG_SZ, + "REG_BINARY": _winreg.REG_BINARY, +} + +def rename_regkey(skey, ssubkey, dsubkey): + """Rename an entire tree of values in the registry. + Function by Thorsten Sick.""" + res_handle = HANDLE() + options = DWORD(0) + res = RegOpenKeyExW( + skey, ssubkey, options, _winreg.KEY_ALL_ACCESS, byref(res_handle) + ) + if not res: + bsize = c_ushort(len(dsubkey) * 2) + us = UNICODE_STRING() + us.Buffer = c_wchar_p(dsubkey) + us.Length = bsize + us.MaximumLength = bsize + + res = NtRenameKey(res_handle, pointer(us)) + if res: + log.warning("Error renaming %s\\%s to %s (0x%x)", + skey, ssubkey, dsubkey, res % 2**32) + + if res_handle: + RegCloseKey(res_handle) + +def regkey_exists(rootkey, subkey): + res_handle = HANDLE() + res = RegOpenKeyExW( + rootkey, subkey, 0, _winreg.KEY_QUERY_VALUE, byref(res_handle) + ) + RegCloseKey(res_handle) + return not res + +def set_regkey(rootkey, subkey, name, type_, value): + if type_ == _winreg.REG_SZ: + value = unicode(value) + length = len(value) * 2 + 2 + elif type_ == _winreg.REG_MULTI_SZ: + value = u"\u0000".join(value) + u"\u0000\u0000" + length = len(value) * 2 + 2 + elif type_ == _winreg.REG_DWORD: + value = struct.pack("I", value) + length = 4 + else: + length = len(value) + + res_handle = HANDLE() + res = RegCreateKeyExW( + rootkey, subkey, 0, None, 0, _winreg.KEY_ALL_ACCESS, + 0, byref(res_handle), None + ) + if not res: + RegSetValueExW(res_handle, name, 0, type_, value, length) + RegCloseKey(res_handle) + +def set_regkey_full(regkey, type_, value): + components = regkey.split("\\") + rootkey, subkey, name = components[0], components[1:-1], components[-1] + if rootkey not in _rootkeys: + log.warning("Unknown root key for registry key: %s", rootkey) + return + + set_regkey( + _rootkeys[rootkey], "\\".join(subkey), name, + _regtypes.get(type_, type_), value + ) + +def del_regkey(rootkey, regkey): + RegDeleteKeyW(rootkey, regkey) + +def query_value(rootkey, subkey, name): + res_handle = HANDLE() + type_ = DWORD() + value = create_string_buffer(1024 * 1024) + length = DWORD(1024 * 1024) + + res = RegOpenKeyExW( + rootkey, subkey, 0, _winreg.KEY_QUERY_VALUE, byref(res_handle) + ) + if not res: + res = RegQueryValueExW( + res_handle, name, None, byref(type_), value, byref(length) + ) + RegCloseKey(res_handle) + + if not res: + if type_.value == _winreg.REG_SZ: + return value.raw[:length.value].decode("utf16").rstrip("\x00") + if type_.value == _winreg.REG_MULTI_SZ: + value = value.raw[:length.value].decode("utf16") + return value.rstrip(u"\u0000").split(u"\u0000") + if type_.value == _winreg.REG_DWORD: + return struct.unpack("I", value.raw[:length.value])[0] + return value.raw[:length.value] diff --git a/cuckoo/data/analyzer/windows/lib/common/results.py b/cuckoo/data/analyzer/windows/lib/common/results.py new file mode 100644 index 0000000000..e0455458e6 --- /dev/null +++ b/cuckoo/data/analyzer/windows/lib/common/results.py @@ -0,0 +1,105 @@ +# Copyright (C) 2013 Claudio Guarnieri. +# Copyright (C) 2014-2017 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import logging +import socket +import time +import sys + +from lib.core.config import Config + +log = logging.getLogger(__name__) + +BUFSIZE = 1024*1024 + +def upload_to_host(file_path, dump_path, pids=[]): + nc = infd = None + try: + nc = NetlogFile() + nc.init(dump_path, file_path, pids) + + infd = open(file_path, "rb") + buf = infd.read(BUFSIZE) + while buf: + nc.send(buf, retry=False) + buf = infd.read(BUFSIZE) + except Exception as e: + log.error("Exception uploading file %r to host: %s", file_path, e) + finally: + if infd: + infd.close() + if nc: + nc.close() + +class NetlogConnection(object): + def __init__(self, proto=""): + config = Config(cfg="analysis.conf") + self.hostip, self.hostport = config.ip, config.port + self.sock = None + self.proto = proto + + def connect(self): + # Try to connect as quickly as possible. Just sort of force it to + # connect with a short timeout. + while not self.sock: + try: + s = socket.create_connection((self.hostip, self.hostport), 0.1) + except socket.error: + time.sleep(0.1) + continue + + s.settimeout(None) + s.sendall(self.proto) + + self.sock = s + + def send(self, data, retry=True): + if not self.sock: + self.connect() + + try: + self.sock.sendall(data) + except socket.error as e: + if retry: + self.connect() + self.send(data, retry=False) + else: + print >>sys.stderr, "Unhandled exception in NetlogConnection:", str(e) + except Exception as e: + print >>sys.stderr, "Unhandled exception in NetlogConnection:", str(e) + # We really have nowhere to log this, if the netlog connection + # does not work, we can assume that any logging won't work either. + # So we just fail silently. + self.close() + + def close(self): + try: + self.sock.shutdown(socket.SHUT_RDWR) + self.sock.close() + self.sock = None + except Exception: + pass + +class NetlogFile(NetlogConnection): + def init(self, dump_path, filepath=None, pids=[]): + if filepath: + self.proto = "FILE 2\n%s\n%s\n%s\n" % ( + dump_path.encode("utf8"), filepath.encode("utf8"), + " ".join(pids) + ) + else: + self.proto = "FILE\n%s\n" % dump_path.encode("utf8") + + self.connect() + +class NetlogHandler(logging.Handler, NetlogConnection): + def __init__(self): + logging.Handler.__init__(self) + NetlogConnection.__init__(self, proto="LOG\n") + self.connect() + + def emit(self, record): + msg = self.format(record) + self.send("{0}\n".format(msg)) diff --git a/cuckoo/data/analyzer/windows/lib/core/__init__.py b/cuckoo/data/analyzer/windows/lib/core/__init__.py new file mode 100644 index 0000000000..57edaa85f5 --- /dev/null +++ b/cuckoo/data/analyzer/windows/lib/core/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. diff --git a/cuckoo/data/analyzer/windows/lib/core/config.py b/cuckoo/data/analyzer/windows/lib/core/config.py new file mode 100644 index 0000000000..c9bb96824b --- /dev/null +++ b/cuckoo/data/analyzer/windows/lib/core/config.py @@ -0,0 +1,48 @@ +# Copyright (C) 2010-2013 Claudio Guarnieri. +# Copyright (C) 2014-2016 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import ConfigParser + +class Config: + def __init__(self, cfg): + """@param cfg: configuration file.""" + config = ConfigParser.ConfigParser(allow_no_value=True) + config.read(cfg) + + for section in config.sections(): + for name, raw_value in config.items(section): + if name == "file_name": + value = config.get(section, name).decode("utf8") + elif name == "options": + value = self.parse_options(config.get(section, name)) + else: + try: + value = config.getboolean(section, name) + except ValueError: + try: + value = config.getint(section, name) + except ValueError: + value = config.get(section, name) + setattr(self, name, value) + + # Just make sure the options field is available. + if not hasattr(self, "options"): + self.options = {} + + def parse_options(self, options): + """Get analysis options. + @return: options dict. + """ + # The analysis package can be provided with some options in the + # following format: + # option1=value1,option2=value2,option3=value3 + ret = {} + for field in options.split(","): + if "=" not in field: + continue + + key, value = field.split("=", 1) + ret[key.strip()] = value.strip() + return ret diff --git a/cuckoo/data/analyzer/windows/lib/core/driver.py b/cuckoo/data/analyzer/windows/lib/core/driver.py new file mode 100644 index 0000000000..4836f43c07 --- /dev/null +++ b/cuckoo/data/analyzer/windows/lib/core/driver.py @@ -0,0 +1,94 @@ +# Copyright (C) 2017-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import ctypes +import os.path +import platform +import shutil +import _winreg + +from lib.common.defines import NTDLL, UNICODE_STRING +from lib.common.exceptions import CuckooError +from lib.common.registry import set_regkey, del_regkey + +class Driver(object): + def __init__(self, driver_name, install_name): + self.driver_name = driver_name + self.install_name = install_name + self.is_64bit = platform.machine().endswith("64") + + def install(self): + self.copy_driver() + self.set_regkey( + "ImagePath", _winreg.REG_SZ, + "\\SystemRoot\\system32\\drivers\\%s.sys" % self.install_name + ) + self.set_regkey("Start", _winreg.REG_DWORD, 3) + self.set_regkey("Type", _winreg.REG_DWORD, 1) + self.set_regkey("ErrorControl", _winreg.REG_DWORD, 1) + self.load_driver() + self.del_regkeys() + + def copy_driver(self): + if platform.machine().endswith("64"): + self.driver_path = os.path.join( + "bin", "%s-x64.sys" % self.driver_name + ) + install_dir = os.path.expandvars( + "%SystemRoot%\\sysnative\\drivers" + ) + else: + self.driver_path = os.path.join( + "bin", "%s-x86.sys" % self.driver_name + ) + install_dir = os.path.expandvars( + "%SystemRoot%\\system32\\drivers" + ) + + if not os.path.exists(self.driver_path): + raise CuckooError("Error locating %s driver!" % self.driver_name) + + self.install_path = os.path.join( + install_dir, "%s.sys" % self.install_name + ) + + shutil.copy(self.driver_path, self.install_path) + + def set_regkey(self, key, type_, value): + set_regkey( + _winreg.HKEY_LOCAL_MACHINE, + "SYSTEM\\CurrentControlSet\\Services\\%s" % self.install_name, + key, type_, value + ) + + def del_regkeys(self): + regkeys = [ + "SYSTEM\\CurrentControlSet\\Services\\%s\\Enum", + "SYSTEM\\CurrentControlSet\\Services\\%s\\Security", + "SYSTEM\\CurrentControlSet\\Services\\%s", + ] + + for regkey in regkeys: + del_regkey(_winreg.HKEY_LOCAL_MACHINE, regkey % self.install_name) + + def load_driver(self): + regkey = ( + u"\\Registry\\Machine\\System" + u"\\CurrentControlSet\\Services\\%s" % self.install_name + ) + us = UNICODE_STRING() + us.Buffer = regkey + us.Length = len(regkey) * 2 + us.MaximumLength = us.Length + + status = NTDLL.NtLoadDriver(ctypes.byref(us)) % 2**32 + if status == 0xc0000428: + raise CuckooError( + "Driver Signature Enforcement has not been disabled." + ) + if status: + raise CuckooError( + "Unable to load the %s driver: 0x%x" % + (self.driver_name, status) + ) diff --git a/cuckoo/data/analyzer/windows/lib/core/ioctl.py b/cuckoo/data/analyzer/windows/lib/core/ioctl.py new file mode 100644 index 0000000000..abe4421db7 --- /dev/null +++ b/cuckoo/data/analyzer/windows/lib/core/ioctl.py @@ -0,0 +1,119 @@ +# Copyright (C) 2016-2018 Cuckoo Foundation. +# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org +# See the file 'docs/LICENSE' for copying permission. + +import ctypes +import logging +import socket +import struct + +from lib.common.defines import ( + KERNEL32, GENERIC_READ, GENERIC_WRITE, FILE_SHARE_READ, FILE_SHARE_WRITE, + OPEN_EXISTING +) +from lib.common.rand import random_string + +log = logging.getLogger(__name__) + +# Random name for the zer0m0n driver. +driver_name = random_string(16) + +CTL_CODE_BASE = 0x222000 + +class Ioctl(object): + def __init__(self, pipepath): + self.pipepath = pipepath + + def invoke(self, ctlcode, value, outlength=0x1000): + device_handle = KERNEL32.CreateFileA( + "\\\\.\\%s" % self.pipepath, GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, None, OPEN_EXISTING, 0, None + ) % 2**32 + + if device_handle == 0xffffffff: + # Only report an error if the error is not "name not found", + # indicating that no kernel analysis is currently taking place. + if KERNEL32.GetLastError() != 2: + log.warning( + "Error opening handle to driver (%s): %d!", + driver_name, KERNEL32.GetLastError() + ) + return False + + out = ctypes.create_string_buffer(outlength) + length = ctypes.c_uint() + + ret = KERNEL32.DeviceIoControl( + device_handle, ctlcode, value, len(value), out, + ctypes.sizeof(out), ctypes.byref(length), None + ) + KERNEL32.CloseHandle(device_handle) + + if not ret: + log.warning( + "Error performing ioctl (0x%08x): %d!", + ctlcode, KERNEL32.GetLastError() + ) + return False + + return out.raw[:length.value] + +class Zer0m0nIoctl(Ioctl): + actions = [ + "addpid", + "cmdpipe", + "channel", + "dumpmem", + "yarald", + "getpids", + "hidepid", + "dumpint", + "resultserver", + ] + + def invoke(self, action, buf): + if action not in self.actions: + raise RuntimeError("Invalid ioctl action: %s" % action) + + return Ioctl.invoke( + self, CTL_CODE_BASE + self.actions.index(action) * 4, buf, + ) + + def addpid(self, pid): + return self.invoke("addpid", struct.pack("Q", pid)) + + def cmdpipe(self, pipe): + return self.invoke("cmdpipe", "\x00".join(pipe + "\x00")) + + def channel(self, pipe): + return self.invoke("channel", "\x00".join(pipe + "\x00")) + + def dumpmem(self, pid): + return self.invoke("dumpmem", struct.pack("Q", pid)) + + def yarald(self, rulepath): + return self.invoke("yarald", open(rulepath, "rb").read()) + + def getpids(self): + pids = self.invoke("getpids", "pids") or "" + return struct.unpack("Q"*(len(pids)/8), pids) + + def hidepid(self, pid): + return self.invoke("hidepid", struct.pack("Q", pid)) + + def dumpint(self, ms): + return self.invoke("dumpint", struct.pack("I", ms)) + + def resultserver(self, ip, port): + # Just a regular SOCKADDR structure, up to 128 bytes + if ":" in ip: + rs = struct.pack("