diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml new file mode 100644 index 00000000..0056bec4 --- /dev/null +++ b/.github/workflows/lint_python.yml @@ -0,0 +1,88 @@ +name: lint_python + +on: + push: + branches: + - master + tags: + - v* + pull_request: + branches: + - '**' + +jobs: + lint: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python: + - major_dot_minor: '3.10' + safety: false + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + # This allows the matrix to specify just the major.minor version while still + # expanding it to get the latest patch version including alpha releases. + # This avoids the need to update for each new alpha, beta, release candidate, + # and then finally an actual release version. actions/setup-python doesn't + # support this for PyPy presently so we get no help there. + # + # CPython -> 3.9.0-alpha - 3.9.X + # PyPy -> pypy-3.7 + python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python.major_dot_minor), matrix.python.major_dot_minor))[startsWith(matrix.python.major_dot_minor, 'pypy')] }} + architecture: x64 + - run: pip install --upgrade pip wheel + - run: pip install bandit black codespell flake8 flake8-2020 flake8-bugbear + flake8-comprehensions isort mypy pytest pyupgrade + - run: bandit --recursive --skip B101,B102,B307,B404,B603,B607 . + - run: black --check . || true + - run: codespell # --ignore-words-list="" --skip="*.css,*.js,*.lock" + - run: flake8 . --builtins=profile --count --select=E9,F63,F7,F82 --show-source --statistics + - run: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=88 + --show-source --statistics + - run: isort --check-only --profile black . || true + - run: pip install --editable . + - run: pip install numpy pylab-sdk + - run: mkdir --parents --verbose .mypy_cache + - run: mypy --ignore-missing-imports --install-types --non-interactive . || true + - run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true + + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python: + - major_dot_minor: '3.7' + safety: false + - major_dot_minor: '3.8' + safety: true + - major_dot_minor: '3.9' + safety: true + - major_dot_minor: '3.10' + safety: true + - major_dot_minor: '3.11' + safety: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + # This allows the matrix to specify just the major.minor version while still + # expanding it to get the latest patch version including alpha releases. + # This avoids the need to update for each new alpha, beta, release candidate, + # and then finally an actual release version. actions/setup-python doesn't + # support this for PyPy presently so we get no help there. + # + # CPython -> 3.9.0-alpha - 3.9.X + # PyPy -> pypy-3.7 + python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python.major_dot_minor), matrix.python.major_dot_minor))[startsWith(matrix.python.major_dot_minor, 'pypy')] }} + architecture: x64 + - run: pip install --upgrade pip wheel + - run: pip install pytest safety + - run: pip install --editable . + - run: pip install numpy pylab-sdk + - run: make test + - if: matrix.python.safety + run: safety check diff --git a/.gitignore b/.gitignore index 1a0237e7..fe86b8cb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,16 @@ .idea +.vscode dist build -*.pyc MANIFEST +*.egg-info +*.pyc +*~ +.coverage + +# Ignore mprof generated files +mprofile_*.dat + +# virtual environment +venv/ +.python-version diff --git a/COPYING b/COPYING new file mode 100644 index 00000000..b9cc03d0 --- /dev/null +++ b/COPYING @@ -0,0 +1,32 @@ +New BSD License + +Copyright (c) 2007–2014 Fabian Pedregosa. +All rights reserved. + + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + a. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + b. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + c. Neither the name of the memory_profiler developers nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. + diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 9561fb10..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include README.rst diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..ff1ffd2b --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ +PYTHON ?= python + +.PHONY: test develop + +test: + $(PYTHON) -m memory_profiler test/test_func.py + $(PYTHON) -m memory_profiler test/test_loop.py + $(PYTHON) -m memory_profiler test/test_mprofile.py + $(PYTHON) -m memory_profiler test/test_as.py + $(PYTHON) -m memory_profiler test/test_global.py + $(PYTHON) -m memory_profiler test/test_precision_command_line.py + $(PYTHON) -m memory_profiler test/test_gen.py + $(PYTHON) -m memory_profiler test/test_unicode.py + $(PYTHON) test/test_tracemalloc.py + $(PYTHON) test/test_import.py + $(PYTHON) test/test_memory_usage.py + $(PYTHON) test/test_precision_import.py + $(PYTHON) test/test_exception.py + $(PYTHON) test/test_exit_code.py + $(PYTHON) test/test_mprof.py + $(PYTHON) test/test_async.py + mprof run test/test_func.py + +develop: + pip install -e . diff --git a/README.rst b/README.rst index c64366b6..a77982ee 100644 --- a/README.rst +++ b/README.rst @@ -1,36 +1,67 @@ +.. image:: https://travis-ci.org/pythonprofilers/memory_profiler.svg?branch=master + :target: https://travis-ci.org/pythonprofilers/memory_profiler + ================= Memory Profiler ================= + + +**Note:** This package is no longer actively maintained. I won't be actively responding to issues. + This is a python module for monitoring memory consumption of a process as well as line-by-line analysis of memory consumption for python -programs. - -It's a pure python module and has the `psutil -`_ module as optional (but highly -recommended) dependencies. +programs. It is a pure python module which depends on the `psutil +`_ module. ============== Installation ============== -To install through easy_install or pip:: +Install via pip:: + + $ pip install -U memory_profiler - $ easy_install -U memory_profiler # pip install -U memory_profiler +The package is also available on `conda-forge +`_. To install from source, download the package, extract and type:: - $ python setup.py install + $ pip install . + +=========== +Quick Start +=========== +Use `mprof` to generate a full memory usage report of your executable and to plot it. + +.. code-block:: bash + + mprof run executable + mprof plot + +The plot would be something like this: + +.. image:: https://i.stack.imgur.com/ixCH4.png ======= Usage ======= -The line-by-line profiler is used much in the same way of the -line_profiler: you must first decorate the function you would like to -profile with ``@profile``. In this example, we create a simple function -``my_func`` that allocates lists ``a``, ``b`` and then deletes ``b``:: +line-by-line memory usage +========================= + +The line-by-line memory usage mode is used much in the same way of the +`line_profiler `_: first +decorate the function you would like to profile with ``@profile`` and +then run the script with a special script (in this case with specific +arguments to the Python interpreter). + +In the following example, we create a simple function ``my_func`` that +allocates lists ``a``, ``b`` and then deletes ``b``: + +.. code-block:: python + @profile def my_func(): a = [1] * (10 ** 6) @@ -51,27 +82,29 @@ this would result in:: Output will follow:: - Line # Mem usage Increment Line Contents - ============================================== - 3 @profile - 4 5.97 MB 0.00 MB def my_func(): - 5 13.61 MB 7.64 MB a = [1] * (10 ** 6) - 6 166.20 MB 152.59 MB b = [2] * (2 * 10 ** 7) - 7 13.61 MB -152.59 MB del b - 8 13.61 MB 0.00 MB return a + Line # Mem usage Increment Occurrences Line Contents + ============================================================ + 3 38.816 MiB 38.816 MiB 1 @profile + 4 def my_func(): + 5 46.492 MiB 7.676 MiB 1 a = [1] * (10 ** 6) + 6 199.117 MiB 152.625 MiB 1 b = [2] * (2 * 10 ** 7) + 7 46.629 MiB -152.488 MiB 1 del b + 8 46.629 MiB 0.000 MiB 1 return a The first column represents the line number of the code that has been profiled, the second column (*Mem usage*) the memory usage of the Python interpreter after that line has been executed. The third column (*Increment*) represents the difference in memory of the current line -with respect to the last one. The last column (*Line Contents*) prints -the code that has been profiled. +with respect to the last one. The fourth column (*Occurrences*) shows +the number of times that profiler has executed each line. The last column +(*Line Contents*) prints the code that has been profiled. Decorator ========= +A function decorator is also available. Use as follows: -A function decorator is also available. Use as follows:: +.. code-block:: python from memory_profiler import profile @@ -82,6 +115,145 @@ A function decorator is also available. Use as follows:: del b return a +In this case the script can be run without specifying ``-m +memory_profiler`` in the command line. + +In function decorator, you can specify the precision as an argument to the +decorator function. Use as follows: + +.. code-block:: python + + from memory_profiler import profile + + @profile(precision=4) + def my_func(): + a = [1] * (10 ** 6) + b = [2] * (2 * 10 ** 7) + del b + return a + +If a python script with decorator ``@profile`` is called using ``-m +memory_profiler`` in the command line, the ``precision`` parameter is ignored. + +Time-based memory usage +========================== +Sometimes it is useful to have full memory usage reports as a function of +time (not line-by-line) of external processes (be it Python scripts or not). +In this case the executable ``mprof`` might be useful. Use it like:: + + mprof run + mprof plot + +The first line run the executable and record memory usage along time, +in a file written in the current directory. +Once it's done, a graph plot can be obtained using the second line. +The recorded file contains a timestamps, that allows for several +profiles to be kept at the same time. + +Help on each `mprof` subcommand can be obtained with the `-h` flag, +e.g. `mprof run -h`. + +In the case of a Python script, using the previous command does not +give you any information on which function is executed at a given +time. Depending on the case, it can be difficult to identify the part +of the code that is causing the highest memory usage. + +Adding the `profile` decorator to a function(ensure no +`from memory_profiler import profile` statement) and running the Python +script with + + mprof run --python python