From 642d0bc98d47855e525996af3dc535bb31fae7e9 Mon Sep 17 00:00:00 2001 From: Ian Ozsvald Date: Sat, 8 Sep 2012 00:43:43 -0700 Subject: [PATCH 001/415] allow undecorated calls with memory_profiler using command line argument --- README.rst | 43 +++++++++++------- examples/example_undecorated.py | 9 ++++ memory_profiler.py | 77 +++++++++++++++++++++++++-------- 3 files changed, 95 insertions(+), 34 deletions(-) create mode 100644 examples/example_undecorated.py diff --git a/README.rst b/README.rst index 781bdfe..5fce359 100644 --- a/README.rst +++ b/README.rst @@ -68,23 +68,34 @@ Python interpreter after that line has been executed. The third column with respect to the last one. The last column (*Line Contents*) prints the code that has been profiled. -Setting debugger breakpoints -============================= -It is possible to set breakpoints depending on the amount of memory used. -That is, you can specify a threshold and as soon as the program uses more -memory than what is specified in the threshold it will stop execution -and run into the pdb debugger. To use it, you will have to decorate -the function as done in the previous section with ``@profile`` and then -run your script with the option ``-m memory_profiler --pdb-mmem=X``, -where X is a number representing the memory threshold in MB. For example:: - $ python -m memory_profiler --pdb-mmem=100 my_script.py +The second usage pattern is to omit the decorator and to add command +line options for target-file and target-function:: -will run ``my_script.py`` and step into the pdb debugger as soon as the code -uses more than 100 MB in the decorated function. + $ python -m memory_profiler example_undecorated.py --target-file=example_undecorated.py --target-function=another_func -.. TODO: alternatives to decoration (for example when you don't want to modify - the file where your function lives). + + Line # Mem usage Increment Line Contents + ================================================ + 2 def another_func(): + 3 8.00 MB 0.00 MB """Undecorated function that allocates memory""" + 4 16.00 MB 8.00 MB c = [1] * (10 ** 6) + 5 92.00 MB 76.00 MB d = [1] * (10 ** 7) + 6 92.00 MB 0.00 MB return c, d + + +Note that you can either profile using the decorator (or many decorators) +OR with one named file and function. You cannot mix the two approaches +and you cannot profile more than one file and function. + +TODO: allow multiple files and functions rather than just one (convert +the named variables into a list internally and iterate on the list) + +TODO: the current named file approach would be confused if two files in +different directories shared the same name, and both had the same named +function. Rather than checking named files we should check for modules +in their namespace (e.g. rather than module1/afile.py and some_function, +we should check for module1.afile.some_function to remove ambiguity). ===== API @@ -115,7 +126,7 @@ arguments to be evaluated as ``f(*args, **kw)``. For example:: ===================== - IPython integration + Ipython integration ===================== After installing the module, if you use IPython, you can set up the `%mprun` and `%memit` magics by following these steps. @@ -182,7 +193,7 @@ For more details, see the docstrings of the magics. ============================ * Q: How accurate are the results ? * A: This module gets the memory consumption by querying the - operating system kernel about the amount of memory the current + operating system kernel about the ammount of memory the current process has allocated, which might be slightly different from the ammount of memory that is actually used by the Python interpreter. Also, because of how the garbage collector works in diff --git a/examples/example_undecorated.py b/examples/example_undecorated.py new file mode 100644 index 0000000..adbd206 --- /dev/null +++ b/examples/example_undecorated.py @@ -0,0 +1,9 @@ + +def another_func(): + """Undecorated function that allocates memory""" + c = [1] * (10 ** 6) + d = [1] * (10 ** 7) + return c, d + +if __name__ == '__main__': + another_func() diff --git a/memory_profiler.py b/memory_profiler.py index 8c35c27..108f469 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -1,6 +1,6 @@ """Profile the memory usage of a Python program""" -__version__ = '0.18' +__version__ = '0.17' _CMD_USAGE = "python -m memory_profiler script_file.py" @@ -16,7 +16,7 @@ def _get_memory(pid): process = psutil.Process(pid) try: - mem = float(process.get_memory_info()[0]) / (1024 ** 2) + mem = float(process.get_memory_info()[0] / (1024 ** 2)) except psutil.AccessDenied: mem = -1 return mem @@ -76,15 +76,6 @@ def memory_usage(proc=-1, interval=.1, timeout=None, run_in_place=False): """ ret = [] - if timeout is not None: - max_iter = timeout / interval - elif isinstance(proc, int): - # external process and no timeout - max_iter = 1 - else: - # for a Python function wait until it finishes - max_iter = float('inf') - if str(proc).endswith('.py'): filename = _find_script(proc) with open(filename) as f: @@ -114,6 +105,10 @@ def memory_usage(proc=-1, interval=.1, timeout=None, run_in_place=False): else: main_thread = multiprocessing.Process(target=f, args=args, kwargs=kw) i = 0 + if timeout is not None: + max_iter = timeout / interval + else: + max_iter = float('inf') main_thread.start() pid = getattr(main_thread, 'pid', os.getpid()) while i < max_iter and main_thread.is_alive(): @@ -126,9 +121,9 @@ def memory_usage(proc=-1, interval=.1, timeout=None, run_in_place=False): # external process if proc == -1: proc = os.getpid() - if max_iter == -1: - max_iter = 1 - for _ in range(max_iter): + if num == -1: + num = 1 + for _ in range(num): ret.append(_get_memory(proc)) time.sleep(interval) return ret @@ -164,6 +159,13 @@ def __init__(self, **kw): self.code_map = {} self.enable_count = 0 self.max_mem = kw.get('max_mem', None) + self.target_file = kw.get('target_file', None) + self.target_function = kw.get('target_function', None) + if self.target_file: + # if we're tracking a file+function rather than using a + # decorator then we enable settrace for all lines of code + self.enable() + def __call__(self, func): self.add_function(func) @@ -247,7 +249,27 @@ def disable_by_count(self): def trace_memory_usage(self, frame, event, arg): """Callback for sys.settrace""" - if event in ('line', 'return') and frame.f_code in self.code_map: + + # if we're profiling a named file and function then + # check this trace event to see if it matches our pattern + profile_this = False + if self.target_file: + co = frame.f_code + func_name = co.co_name + func_filename = co.co_filename + if self.target_file in func_filename: + if self.target_function == func_name: + if frame.f_code not in self.code_map: + # if we've not yet encountered this function (and it is + # the one we want to trace) then we add it to + # the code_map + self.code_map[frame.f_code] = {} + profile_this = True + + # if we've been called on our decorated function + # OR we've matched the named file and function + # then track the memory usage + if event in ('line', 'return') and (frame.f_code in self.code_map or profile_this): lineno = frame.f_lineno if event == 'return': lineno += 1 @@ -258,7 +280,7 @@ def trace_memory_usage(self, frame, event, arg): def trace_max_mem(self, frame, event, arg): # run into PDB as soon as memory is higher than MAX_MEM - if event in ('line', 'return') and frame.f_code in self.code_map: + if event in ('line', 'return'): c = _get_memory(os.getpid()) if c >= self.max_mem: t = 'Current memory {0:.2f} MB exceeded the maximum '.format(c) + \ @@ -290,7 +312,12 @@ def enable(self): def disable(self): self.last_time = {} - sys.settrace(None) + if self.target_file is None: + # if target_file is None then we're using the decorator + # and it is safe to disable settrace + # if target_file is not None then we're running settrace + # on every line of code so we can't disable the trace + sys.settrace(None) def show_results(prof, stream=None): @@ -544,6 +571,12 @@ def magic_memit(self, line=''): parser.add_option("--pdb-mmem", dest="max_mem", metavar="MAXMEM", type="float", action="store", help="step into the debugger when memory exceeds MAXMEM") + parser.add_option("--target-file", dest="target_file", + type="str", action="store", default=None, + help="Traces this file (requires target-function), disables @profile tracing") + parser.add_option("--target-function", dest="target_function", + type="str", action="store", default=None, + help="Traces this function (requires target-file), disables @profile tracing") if not sys.argv[1:]: parser.print_help() @@ -551,7 +584,15 @@ def magic_memit(self, line=''): (options, args) = parser.parse_args() - prof = LineProfiler(max_mem=options.max_mem) + # check that if the user wants to memory_profile without a decorator + # then they've specified both of the required options + if options.target_file or options.target_function: + if not (options.target_file and options.target_function): + print "Error: Both --target-file and --target-function are required" + raise SystemExit(1) + + # hardcoded module name and function name + prof = LineProfiler(max_mem=options.max_mem, target_file=options.target_file, target_function=options.target_function) __file__ = _find_script(args[0]) if sys.version_info[0] < 3: import __builtin__ From 8fa4550c36d49f180cd3869f223e5f3cc37c1c05 Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Fri, 15 Mar 2013 14:13:58 +0100 Subject: [PATCH 002/415] Added -o option to memory_profiler Output of 'python -m memory_profiler -o "bar.out" script.py' goes into file "bar.out" instead of standard output. --- memory_profiler.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/memory_profiler.py b/memory_profiler.py index 485d0eb..fc74642 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -387,7 +387,7 @@ def show_results(prof, stream=None, precision=3): # A lprun-style %mprun magic for IPython. def magic_mprun(self, parameter_s=''): """ Execute a statement under the line-by-line memory profiler from the - memory_profilser module. + memory_profiler module. Usage: %mprun -f func1 -f func2 @@ -592,7 +592,9 @@ def wrapper(*args, **kwargs): parser.add_option('--precision', dest="precision", type="int", action="store", default=3, help="precision of memory output in number of significant digits") - + parser.add_option("-o", dest="out_filename", type="str", + action="store", default=None, + help="path to a file where results will be written") if not sys.argv[1:]: parser.print_help() sys.exit(2) @@ -616,4 +618,9 @@ def wrapper(*args, **kwargs): exec(compile(open(__file__).read(), __file__, 'exec'), ns, globals()) finally: - show_results(prof, precision=options.precision) + if options.out_filename is not None: + out_file = open(options.out_filename, "w") + else: + out_file = sys.stdout + + show_results(prof, precision=options.precision, stream=out_file) From 58208ae8a888a2b34b0d2a81c5e0ac019f78ef32 Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Thu, 21 Mar 2013 15:30:08 +0100 Subject: [PATCH 003/415] Added timestamps to mprofile output The output of mprofile now includes timestamp information, in addition to memory consumption. A "timestamps" option has been added to memory_usage() --- memory_profiler.py | 37 ++++++++++++++++++++++++++----------- mprofile | 6 +++--- test/test_loop.py | 9 ++++++--- test/test_loop_decorated.py | 24 ++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 17 deletions(-) create mode 100644 test/test_loop_decorated.py diff --git a/memory_profiler.py b/memory_profiler.py index fc74642..156a0df 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -20,13 +20,16 @@ try: import psutil - def _get_memory(pid): + def _get_memory(pid, timestamps=False): process = psutil.Process(pid) try: mem = float(process.get_memory_info()[0]) / (1024 ** 2) except psutil.AccessDenied: mem = -1 - return mem + if timestamps: + return (mem, time.time()) + else: + return mem except ImportError: @@ -34,7 +37,7 @@ def _get_memory(pid): warnings.warn("psutil module not found. memory_profiler will be slow") if os.name == 'posix': - def _get_memory(pid): + def _get_memory(pid, timestamps=False): # .. # .. memory usage in MB .. # .. this should work on both Mac and Linux .. @@ -44,9 +47,16 @@ def _get_memory(pid): stdout=subprocess.PIPE).communicate()[0].split(b'\n') try: vsz_index = out[0].split().index(b'RSS') - return float(out[1].split()[vsz_index]) / 1024 + mem = float(out[1].split()[vsz_index]) / 1024 + if timestamps: + return(mem, time.time()) + else: + return mem except: - return -1 + if timestamps: + return (-1, time.time()) + else: + return -1 else: raise NotImplementedError('The psutil module is required for non-unix ' 'platforms') @@ -62,19 +72,24 @@ def __init__(self, monitor_pid, interval, pipe, *args, **kw): self.interval = interval self.pipe = pipe self.cont = True + if "timestamps" in kw: + self.timestamps = kw["timestamps"] + del kw["timestamps"] + else: + self.timestamps = False super(Timer, self).__init__(*args, **kw) def run(self): - m = _get_memory(self.monitor_pid) + m = _get_memory(self.monitor_pid, timestamps=self.timestamps) timings = [m] self.pipe.send(0) # we're ready while not self.pipe.poll(self.interval): - m = _get_memory(self.monitor_pid) + m = _get_memory(self.monitor_pid, timestamps=self.timestamps) timings.append(m) self.pipe.send(timings) -def memory_usage(proc=-1, interval=.1, timeout=None): +def memory_usage(proc=-1, interval=.1, timeout=None, timestamps=False): """ Return the memory usage of a process or piece of code @@ -131,7 +146,7 @@ def memory_usage(proc=-1, interval=.1, timeout=None): % (n_args, len(args))) child_conn, parent_conn = Pipe() # this will store Timer's results - p = Timer(os.getpid(), interval, child_conn) + p = Timer(os.getpid(), interval, child_conn, timestamps=timestamps) p.start() parent_conn.recv() # wait until we start getting memory f(*args, **kw) @@ -141,7 +156,7 @@ def memory_usage(proc=-1, interval=.1, timeout=None): elif isinstance(proc, subprocess.Popen): # external process, launched from Python while True: - ret.append(_get_memory(proc.pid)) + ret.append(_get_memory(proc.pid, timestamps=timestamps)) time.sleep(interval) if timeout is not None: max_iter -= 1 @@ -158,7 +173,7 @@ def memory_usage(proc=-1, interval=.1, timeout=None): counter = 0 while counter < max_iter: counter += 1 - ret.append(_get_memory(proc)) + ret.append(_get_memory(proc, timestamps=timestamps)) time.sleep(interval) return ret diff --git a/mprofile b/mprofile index 3daaa06..c8362de 100755 --- a/mprofile +++ b/mprofile @@ -20,7 +20,7 @@ value per line). Memory is sampled twice each second.""" outfilename = "mprofile_%s.dat" % time.strftime("%Y%m%d%H%M%S", time.localtime()) p = subprocess.Popen(sys.argv[1:]) -mu = mp.memory_usage(proc=p, interval=0.5) +mu = mp.memory_usage(proc=p, interval=0.5, timestamps=True) with open(outfilename, "w") as f: - for m in mu: - f.write(str(m) + "\n") + for m, t in mu: + f.write("{0:.6f} {1:.4f}".format(m, t) + "\n") diff --git a/test/test_loop.py b/test/test_loop.py index aaaded3..69da592 100644 --- a/test/test_loop.py +++ b/test/test_loop.py @@ -1,6 +1,7 @@ # .. an example with a for loop .. -@profile +import time + def test_1(): a = [1] * (10 ** 6) b = [2] * (2 * 10 ** 7) @@ -12,13 +13,15 @@ def test_1(): del b return a -@profile + def test_2(): a = {} for i in range(10000): - a[i] = i + 1 + a[i] = i + 1 return if __name__ == '__main__': test_1() + time.sleep(1) test_2() + time.sleep(1) diff --git a/test/test_loop_decorated.py b/test/test_loop_decorated.py new file mode 100644 index 0000000..aaaded3 --- /dev/null +++ b/test/test_loop_decorated.py @@ -0,0 +1,24 @@ +# .. an example with a for loop .. + +@profile +def test_1(): + a = [1] * (10 ** 6) + b = [2] * (2 * 10 ** 7) + del b + + for i in range(2): + a = [1] * (10 ** 6) + b = [2] * (2 * 10 ** 7) + del b + return a + +@profile +def test_2(): + a = {} + for i in range(10000): + a[i] = i + 1 + return + +if __name__ == '__main__': + test_1() + test_2() From a17bdc80f195fad45670bced6c7c2fb009407fc7 Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Thu, 21 Mar 2013 15:40:51 +0100 Subject: [PATCH 004/415] Plotting of mprofile output The "mplot" script has been added. Plots of memory consumption can be done with it. --- .gitignore | 1 + mplot | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100755 mplot diff --git a/.gitignore b/.gitignore index 1a0237e..bea3218 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ dist build *.pyc MANIFEST +*~ diff --git a/mplot b/mplot new file mode 100755 index 0000000..868f10d --- /dev/null +++ b/mplot @@ -0,0 +1,43 @@ +#! /usr/bin/env python +import pylab as pl +import sys +import glob +import time +import os + +profiles = glob.glob("mprofile_*.dat") +profiles.sort() + +if len(sys.argv) == 1: + filename = profiles[-1] +else: + filename = sys.argv[1] + if not os.path.exists(filename): + try: + n = int(filename) + except ValueError: + print("Input file not found: " + filename) + filename = profiles[n] + +datetime = time.strptime(filename, "mprofile_%Y%m%d%H%M%S.dat") +mdata = pl.loadtxt(filename) +mem = mdata[:, 0] +max_mem = mem.max() +max_mem_ind = mem.argmax() +sampling_period = 0.5 +t = pl.r_[:len(mem)] * sampling_period + +pl.figure() +pl.plot(t, mem) +pl.hlines(max_mem, + pl.xlim()[0] + 0.001, pl.xlim()[1] - 0.001, + colors="r", linestyles="--") +pl.vlines(t[max_mem_ind], + pl.ylim()[0] + 0.001, pl.ylim()[1] - 0.001, + colors="r", linestyles="--") +pl.xlabel("time [s]") +pl.ylabel("memory used [Mb]") +pl.title(time.strftime("%d / %m / %Y - start at %H:%M:%S", datetime)) +pl.grid() +pl.show() + From 4fe3646e106afa1ffe6cc0ee791ef3235a7e2b7c Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Fri, 15 Mar 2013 14:15:03 +0100 Subject: [PATCH 005/415] Added mplot to setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0abe461..40c38a2 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ author_email='fabian@fseoane.net', url='http://pypi.python.org/pypi/memory_profiler', py_modules=['memory_profiler'], - scripts=["mprofile"], + scripts=["mprofile", "mplot"], classifiers=[_f for _f in CLASSIFIERS.split('\n') if _f], license='Simplified BSD' From cac54d9bb95b051d5559456ca8aa5ab081f49387 Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Mon, 18 Mar 2013 09:30:03 +0100 Subject: [PATCH 006/415] Added timestamping feature Added a --timestamp option that when turned on, output timestamps for start and end of every decorated function, instead of memory profile. This options is intended to be used together with the mprofile script, to be able to plot the lifetime of some functions on the global memory graph. --- memory_profiler.py | 60 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/memory_profiler.py b/memory_profiler.py index 156a0df..a57cec1 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -199,6 +199,51 @@ def _find_script(script_name): raise SystemExit(1) +class TimeStamper: + """ A profiler that just records start and end execution times for + any decorated function. + """ + def __init__(self): + self.functions = {} + + def __call__(self, func): + self.add_function(func) + f = self.wrap_function(func) + f.__module__ = func.__module__ + f.__name__ = func.__name__ + f.__doc__ = func.__doc__ + f.__dict__.update(getattr(func, '__dict__', {})) + return f + + def add_function(self, func): + if not func in self.functions: + self.functions[func] = [] + + def wrap_function(self, func): + """ Wrap a function to timestamp it. + """ + def f(*args, **kwds): + # Start time + timestamps = [time.time()] + self.functions[func].append(timestamps) + try: + result = func(*args, **kwds) + finally: + # end time + timestamps.append(time.time()) + return result + return f + + def show_results(self, stream=None): + if stream is None: + stream = sys.stdout + + for func, timestamps in self.functions.iteritems(): + function_name = "%s.%s" % (func.__module__, func.__name__) + for ts in timestamps: + stream.write("%s %.4f %.4f\n" % (function_name, ts[0], ts[1])) + + class LineProfiler: """ A profiler that records the amount of memory for each line """ @@ -610,13 +655,21 @@ def wrapper(*args, **kwargs): parser.add_option("-o", dest="out_filename", type="str", action="store", default=None, help="path to a file where results will be written") + parser.add_option("--timestamp", dest="timestamp", default=False, + action="store_true", + help="""print timestamp instead of memory measurement for + decorated functions""") + if not sys.argv[1:]: parser.print_help() sys.exit(2) (options, args) = parser.parse_args() - prof = LineProfiler(max_mem=options.max_mem) + if options.timestamp: + prof = TimeStamper() + else: + prof = LineProfiler(max_mem=options.max_mem) __file__ = _find_script(args[0]) try: if sys.version_info[0] < 3: @@ -638,4 +691,7 @@ def wrapper(*args, **kwargs): else: out_file = sys.stdout - show_results(prof, precision=options.precision, stream=out_file) + if options.timestamp: + prof.show_results(stream=out_file) + else: + show_results(prof, precision=options.precision, stream=out_file) From 0dcd9ef3e5f136d22d25ea3071ecfc60c63a8d3c Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Mon, 18 Mar 2013 10:50:31 +0100 Subject: [PATCH 007/415] Timestamping is handled by mprofile and mplot An --python option has been added to mprofile, just to add the proper timestamping options to the run python program. mplot now looks for a timestamp file, and plot the additional information as vertical lines. --- mplot | 58 +++++++++++++++++++++++++++++++++---- mprofile | 50 +++++++++++++++++++++++--------- test/test_loop_decorated.py | 7 ++++- test/test_mprofile.py | 23 +++++++++++++++ 4 files changed, 118 insertions(+), 20 deletions(-) create mode 100644 test/test_mprofile.py diff --git a/mplot b/mplot index 868f10d..6acfc9a 100755 --- a/mplot +++ b/mplot @@ -1,43 +1,89 @@ #! /usr/bin/env python import pylab as pl +import math import sys import glob import time import os +import os.path as osp -profiles = glob.glob("mprofile_*.dat") + +def read_timestamp_file(ts_filename): + """Return content of ts_filename or None, if ts_filename + is invalid or does not exist""" + if not osp.isfile(ts_filename): + return None + + ret = {} + f = open(ts_filename) + for l in f: + f_name, start, end = l.split() + ts = ret.get(f_name, []) + ts.append([float(start), float(end)]) + ret[f_name] = ts + f.close() + return ret + + +profiles = glob.glob("mprofile_??????????????.dat") profiles.sort() if len(sys.argv) == 1: filename = profiles[-1] else: filename = sys.argv[1] - if not os.path.exists(filename): + if not osp.exists(filename): try: n = int(filename) except ValueError: print("Input file not found: " + filename) filename = profiles[n] -datetime = time.strptime(filename, "mprofile_%Y%m%d%H%M%S.dat") +# Check for a timestamp file +file_parts = osp.splitext(filename) +ts_filename = file_parts[0] + "_ts" + file_parts[1] +ts = read_timestamp_file(ts_filename) + mdata = pl.loadtxt(filename) +global_start = float(mdata[0, 1]) + mem = mdata[:, 0] max_mem = mem.max() max_mem_ind = mem.argmax() sampling_period = 0.5 t = pl.r_[:len(mem)] * sampling_period +all_colors=("g", "b", "k", "r") + pl.figure() pl.plot(t, mem) + +bottom, top = pl.ylim() +bottom += 0.001 +top -= 0.001 + +# plot timestamps, if any +if ts is not None: + func_num = 0 + for f, exec_ts in ts.iteritems(): + for execution in exec_ts: + pl.vlines([ts - global_start for ts in execution], + bottom, top, + colors=all_colors[func_num % len(all_colors)], + label=f.split(".")[-1]) + func_num += 1 + pl.legend() + pl.hlines(max_mem, pl.xlim()[0] + 0.001, pl.xlim()[1] - 0.001, colors="r", linestyles="--") -pl.vlines(t[max_mem_ind], - pl.ylim()[0] + 0.001, pl.ylim()[1] - 0.001, +pl.vlines(t[max_mem_ind], bottom, top, colors="r", linestyles="--") pl.xlabel("time [s]") pl.ylabel("memory used [Mb]") -pl.title(time.strftime("%d / %m / %Y - start at %H:%M:%S", datetime)) +title = time.strftime("%d / %m / %Y - start at %H:%M:%S", time.localtime(global_start)) \ + + ".{0:03d}".format(int(round(math.modf(global_start)[0]*1000))) +pl.title(title) pl.grid() pl.show() diff --git a/mprofile b/mprofile index c8362de..8e4672f 100755 --- a/mprofile +++ b/mprofile @@ -5,22 +5,46 @@ import os.path as osp import time import sys -if len(sys.argv) < 2: - print("""Memory usage monitoring -Usage: %s ... - -Output results in a file called "mprofile_.dat" (where - is the date-time of the program start) in the current -directory. This file contains the process memory consumption, in Mb (one -value per line). Memory is sampled twice each second.""" - % osp.basename(sys.argv[0]) - ) +from optparse import OptionParser + +parser = OptionParser(version=mp.__version__) +parser.disable_interspersed_args() +parser.add_option("--python", dest="python", default=False, + action="store_true", + help="""Activates extra features when the profiled executable is + a Python program (currently: function timestamping.)""") + +(options, args) = parser.parse_args() +if len(args) == 0: + print("A program to run must be provided. Use -h for help") sys.exit(1) -outfilename = "mprofile_%s.dat" % time.strftime("%Y%m%d%H%M%S", time.localtime()) +## if len(sys.argv) < 2: +## print("""Memory usage monitoring +## Usage: %s ... + +## Output results in a file called "mprofile_.dat" (where +## is the date-time of the program start) in the current +## directory. This file contains the process memory consumption, in Mb (one +## value per line). Memory is sampled twice each second.""" +## % osp.basename(sys.argv[0]) +## ) +## sys.exit(1) + +suffix = time.strftime("%Y%m%d%H%M%S", time.localtime()) +mprofile_output = "mprofile_%s.dat" % suffix + +if options.python: + print("running as a Python program...") + timestamp_output = "mprofile_%s_ts.dat" % suffix + if not args[0].startswith("python"): + args.insert(0, "python") + args[1:1] = ("-m", "memory_profiler", "--timestamp", "-o", timestamp_output) + p = subprocess.Popen(args) +else: + p = subprocess.Popen(args) -p = subprocess.Popen(sys.argv[1:]) mu = mp.memory_usage(proc=p, interval=0.5, timestamps=True) -with open(outfilename, "w") as f: +with open(mprofile_output, "w") as f: for m, t in mu: f.write("{0:.6f} {1:.4f}".format(m, t) + "\n") diff --git a/test/test_loop_decorated.py b/test/test_loop_decorated.py index aaaded3..7d64c3a 100644 --- a/test/test_loop_decorated.py +++ b/test/test_loop_decorated.py @@ -1,9 +1,12 @@ # .. an example with a for loop .. +import time + @profile def test_1(): a = [1] * (10 ** 6) b = [2] * (2 * 10 ** 7) + time.sleep(0.6) del b for i in range(2): @@ -15,8 +18,10 @@ def test_1(): @profile def test_2(): a = {} + time.sleep(0.5) for i in range(10000): - a[i] = i + 1 + a[i] = i + 1 + time.sleep(0.6) return if __name__ == '__main__': diff --git a/test/test_mprofile.py b/test/test_mprofile.py new file mode 100644 index 0000000..8b458eb --- /dev/null +++ b/test/test_mprofile.py @@ -0,0 +1,23 @@ +"""This script is intended as a test case for mprofile""" + +import time + +@profile +def test1(): + a = [1] * 100000 + time.sleep(1) + return a + +@profile +def test2(l): + b = [2 * n for n in l] + time.sleep(1) + del b + + +if __name__ == "__main__": + time.sleep(1) + l = test1() + test2(l) + time.sleep(1) + From b639130fe911b046e5cffdb9f199e504a7982b79 Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Tue, 19 Mar 2013 17:04:03 +0100 Subject: [PATCH 008/415] Added a check in TimeStamper TimeStamper now raises a clear exception when provided with something that is not callable. --- memory_profiler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/memory_profiler.py b/memory_profiler.py index a57cec1..910f615 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -207,6 +207,9 @@ def __init__(self): self.functions = {} def __call__(self, func): + if not hasattr(func, "__call__"): + raise ValueError("Value must be callable") + self.add_function(func) f = self.wrap_function(func) f.__module__ = func.__module__ From 7293915b784caf1b06758166fa6b336de5a92948 Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Thu, 21 Mar 2013 16:04:06 +0100 Subject: [PATCH 009/415] Added explanatory messages to mplot. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Improvement of error handling in mplot: a clear message is printed whether pylab is not found or no input file are automatically detected. --- mplot | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/mplot b/mplot index 6acfc9a..9a19625 100755 --- a/mplot +++ b/mplot @@ -1,7 +1,12 @@ #! /usr/bin/env python -import pylab as pl -import math import sys + +try: + import pylab as pl +except ImportError: + print("matplotlib is needed for plotting.") + sys.exit(1) +import math import glob import time import os @@ -29,6 +34,10 @@ profiles = glob.glob("mprofile_??????????????.dat") profiles.sort() if len(sys.argv) == 1: + if len(profiles) == 0: + print("""No input file found. This program looks for mprofile_*.dat files, +generated by the mprofile command.""") + sys.exit(-1) filename = profiles[-1] else: filename = sys.argv[1] From e12bc556ed278ab2610e358061963af99e9abc48 Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Mon, 25 Mar 2013 08:57:40 +0100 Subject: [PATCH 010/415] Changed the display of timestamps Execution times are now shown on the memory graph using a kind of brackets instead of just vertical lines. This makes reading the graph easier. Graphing is now correct for any sampling frequency. --- mplot | 60 +++++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 10 deletions(-) diff --git a/mplot b/mplot index 9a19625..2ea201c 100755 --- a/mplot +++ b/mplot @@ -13,6 +13,42 @@ import os import os.path as osp +def add_bracket(xloc, t, mem, color="r", label=None): + """Add two brackets on the memory line plot. + + This function uses the current figure. + + Parameters + ========== + xloc: {tuple with 2 values} + bracket location (on horizontal axis). + t, mem: + memory usage curve. Used to place bracket at the correct location. + """ + + height_ratio = 20. + yloc = pl.interp(xloc, t, mem) + vsize = (pl.ylim()[1] - pl.ylim()[0]) / height_ratio + hsize = (pl.xlim()[1] - pl.xlim()[0]) / (3.*height_ratio) + + bracket_x = pl.asarray([hsize, 0, 0, hsize]) + bracket_y = pl.asarray([vsize, vsize, -vsize, -vsize]) + + pl.plot(bracket_x + xloc[0], bracket_y + yloc[0], + "-" + color, linewidth=2, label=label) + pl.plot(-bracket_x + xloc[1], bracket_y + yloc[1], + "-" + color, linewidth=2 ) + + # TODO: use matplotlib.patches.Polygon to draw a colored background for + # each function. + + # with maplotlib 1.2, use matplotlib.path.Path to create proper markers + # see http://matplotlib.org/examples/pylab_examples/marker_path.html + # This works with matplotlib 0.99.1 + ## pl.plot(xloc[0], yloc[0], "<"+color, markersize=7, label=label) + ## pl.plot(xloc[1], yloc[1], ">"+color, markersize=7) + + def read_timestamp_file(ts_filename): """Return content of ts_filename or None, if ts_filename is invalid or does not exist""" @@ -59,13 +95,13 @@ global_start = float(mdata[0, 1]) mem = mdata[:, 0] max_mem = mem.max() max_mem_ind = mem.argmax() -sampling_period = 0.5 -t = pl.r_[:len(mem)] * sampling_period -all_colors=("g", "b", "k", "r") +t = mdata[:, 1] - global_start + +all_colors=("c", "y", "g", "r", "b") pl.figure() -pl.plot(t, mem) +pl.plot(t, mem, "b+-") bottom, top = pl.ylim() bottom += 0.001 @@ -76,12 +112,16 @@ if ts is not None: func_num = 0 for f, exec_ts in ts.iteritems(): for execution in exec_ts: - pl.vlines([ts - global_start for ts in execution], - bottom, top, - colors=all_colors[func_num % len(all_colors)], - label=f.split(".")[-1]) + add_bracket([ts - global_start for ts in execution], t, mem, + color= all_colors[func_num % len(all_colors)], + label=f.split(".")[-1] + " %.3fs" % (execution[1] - execution[0])) + ## pl.vlines([ts - global_start for ts in execution], + ## bottom, top, + ## colors=all_colors[func_num % len(all_colors)], + ## linestyles="dashdot", + ## label=f.split(".")[-1] + " %.3fs" % (execution[1] - execution[0])) func_num += 1 - pl.legend() + pl.legend(loc=0) pl.hlines(max_mem, pl.xlim()[0] + 0.001, pl.xlim()[1] - 0.001, @@ -89,7 +129,7 @@ pl.hlines(max_mem, pl.vlines(t[max_mem_ind], bottom, top, colors="r", linestyles="--") pl.xlabel("time [s]") -pl.ylabel("memory used [Mb]") +pl.ylabel("memory used [MB]") title = time.strftime("%d / %m / %Y - start at %H:%M:%S", time.localtime(global_start)) \ + ".{0:03d}".format(int(round(math.modf(global_start)[0]*1000))) pl.title(title) From 27e8a21bcb20d07cdbacc58144909b2b7a068a89 Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Mon, 25 Mar 2013 09:28:31 +0100 Subject: [PATCH 011/415] Added --interval option to mprofile This options is used to change the sampling period. Example: mprofile -T 0.2 Will execute and record its memory usage every .2s. --- mplot | 4 +++- mprofile | 10 ++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/mplot b/mplot index 2ea201c..b642df8 100755 --- a/mplot +++ b/mplot @@ -89,13 +89,15 @@ file_parts = osp.splitext(filename) ts_filename = file_parts[0] + "_ts" + file_parts[1] ts = read_timestamp_file(ts_filename) -mdata = pl.loadtxt(filename) +mdata = pl.atleast_2d(pl.loadtxt(filename)) + global_start = float(mdata[0, 1]) mem = mdata[:, 0] max_mem = mem.max() max_mem_ind = mem.argmax() + t = mdata[:, 1] - global_start all_colors=("c", "y", "g", "r", "b") diff --git a/mprofile b/mprofile index 8e4672f..840b5bd 100755 --- a/mprofile +++ b/mprofile @@ -13,8 +13,13 @@ parser.add_option("--python", dest="python", default=False, action="store_true", help="""Activates extra features when the profiled executable is a Python program (currently: function timestamping.)""") +parser.add_option("--interval", "-T", dest="interval", default="0.5", + type="float", action="store", + help="Sampling period (in seconds)") (options, args) = parser.parse_args() +print("{1}: Sampling memory every {0.interval}s".format(options, osp.basename(sys.argv[0]))) + if len(args) == 0: print("A program to run must be provided. Use -h for help") sys.exit(1) @@ -39,12 +44,13 @@ if options.python: timestamp_output = "mprofile_%s_ts.dat" % suffix if not args[0].startswith("python"): args.insert(0, "python") - args[1:1] = ("-m", "memory_profiler", "--timestamp", "-o", timestamp_output) + args[1:1] = ("-m", "memory_profiler", "--timestamp", + "-o", timestamp_output) p = subprocess.Popen(args) else: p = subprocess.Popen(args) -mu = mp.memory_usage(proc=p, interval=0.5, timestamps=True) +mu = mp.memory_usage(proc=p, interval=options.interval, timestamps=True) with open(mprofile_output, "w") as f: for m, t in mu: f.write("{0:.6f} {1:.4f}".format(m, t) + "\n") From c868ce4045ac6fdc2cb6e23a6be0dc2873a2eae1 Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Thu, 28 Mar 2013 14:48:11 +0100 Subject: [PATCH 012/415] Added option to monitor children processes To get the memory usage of the main process and its children processes, use the --include-children of mprofile. Only the sum of all memory usage is recorded, no per-process details are available. --- memory_profiler.py | 31 ++++++++++++++++++++++--------- mprofile | 6 +++++- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/memory_profiler.py b/memory_profiler.py index 910f615..aac186d 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -20,10 +20,14 @@ try: import psutil - def _get_memory(pid, timestamps=False): + def _get_memory(pid, timestamps=False, include_children=False): process = psutil.Process(pid) try: - mem = float(process.get_memory_info()[0]) / (1024 ** 2) + mem = float(process.get_memory_info()[0]) / 1048576. + if include_children: + for p in process.get_children(recursive=True): + mem += p.get_memory_info()[0] / 1048576. + except psutil.AccessDenied: mem = -1 if timestamps: @@ -31,9 +35,7 @@ def _get_memory(pid, timestamps=False): else: return mem - except ImportError: - warnings.warn("psutil module not found. memory_profiler will be slow") if os.name == 'posix': @@ -77,19 +79,28 @@ def __init__(self, monitor_pid, interval, pipe, *args, **kw): del kw["timestamps"] else: self.timestamps = False + if "include_children" in kw: + self.include_children = kw["include_children"] + del kw["include_children"] + else: + self.include_children = False + super(Timer, self).__init__(*args, **kw) def run(self): - m = _get_memory(self.monitor_pid, timestamps=self.timestamps) + m = _get_memory(self.monitor_pid, timestamps=self.timestamps, + include_children=self.include_children) timings = [m] self.pipe.send(0) # we're ready while not self.pipe.poll(self.interval): - m = _get_memory(self.monitor_pid, timestamps=self.timestamps) + m = _get_memory(self.monitor_pid, timestamps=self.timestamps, + include_children=self.include_children) timings.append(m) self.pipe.send(timings) -def memory_usage(proc=-1, interval=.1, timeout=None, timestamps=False): +def memory_usage(proc=-1, interval=.1, timeout=None, timestamps=False, + include_children=False): """ Return the memory usage of a process or piece of code @@ -156,7 +167,8 @@ def memory_usage(proc=-1, interval=.1, timeout=None, timestamps=False): elif isinstance(proc, subprocess.Popen): # external process, launched from Python while True: - ret.append(_get_memory(proc.pid, timestamps=timestamps)) + ret.append(_get_memory(proc.pid, timestamps=timestamps, + include_children=include_children)) time.sleep(interval) if timeout is not None: max_iter -= 1 @@ -173,7 +185,8 @@ def memory_usage(proc=-1, interval=.1, timeout=None, timestamps=False): counter = 0 while counter < max_iter: counter += 1 - ret.append(_get_memory(proc, timestamps=timestamps)) + ret.append(_get_memory(proc, timestamps=timestamps, + include_children=include_children)) time.sleep(interval) return ret diff --git a/mprofile b/mprofile index 840b5bd..fa6354b 100755 --- a/mprofile +++ b/mprofile @@ -16,6 +16,9 @@ parser.add_option("--python", dest="python", default=False, parser.add_option("--interval", "-T", dest="interval", default="0.5", type="float", action="store", help="Sampling period (in seconds)") +parser.add_option("--include-children", "-C", dest="include_children", default=False, + action="store_true", + help="""Monitors forked processes as well (sum up all process memory)""") (options, args) = parser.parse_args() print("{1}: Sampling memory every {0.interval}s".format(options, osp.basename(sys.argv[0]))) @@ -50,7 +53,8 @@ if options.python: else: p = subprocess.Popen(args) -mu = mp.memory_usage(proc=p, interval=options.interval, timestamps=True) +mu = mp.memory_usage(proc=p, interval=options.interval, timestamps=True, + include_children=options.include_children) with open(mprofile_output, "w") as f: for m, t in mu: f.write("{0:.6f} {1:.4f}".format(m, t) + "\n") From 5c3934209df4848fbefd2fbace8eada7719a08cb Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Thu, 4 Apr 2013 13:39:32 +0200 Subject: [PATCH 013/415] mplot can superimpose curves mplot now read every argument on its command-line, and plot every graph on the same plot. Graph layout has been clarified: legend is now outside the axis. --- mplot | 160 ++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 89 insertions(+), 71 deletions(-) diff --git a/mplot b/mplot index b642df8..6a9390c 100755 --- a/mplot +++ b/mplot @@ -66,75 +66,93 @@ def read_timestamp_file(ts_filename): return ret -profiles = glob.glob("mprofile_??????????????.dat") -profiles.sort() - -if len(sys.argv) == 1: - if len(profiles) == 0: - print("""No input file found. This program looks for mprofile_*.dat files, -generated by the mprofile command.""") - sys.exit(-1) - filename = profiles[-1] -else: - filename = sys.argv[1] - if not osp.exists(filename): - try: - n = int(filename) - except ValueError: - print("Input file not found: " + filename) - filename = profiles[n] - -# Check for a timestamp file -file_parts = osp.splitext(filename) -ts_filename = file_parts[0] + "_ts" + file_parts[1] -ts = read_timestamp_file(ts_filename) - -mdata = pl.atleast_2d(pl.loadtxt(filename)) - -global_start = float(mdata[0, 1]) - -mem = mdata[:, 0] -max_mem = mem.max() -max_mem_ind = mem.argmax() - - -t = mdata[:, 1] - global_start - -all_colors=("c", "y", "g", "r", "b") - -pl.figure() -pl.plot(t, mem, "b+-") - -bottom, top = pl.ylim() -bottom += 0.001 -top -= 0.001 - -# plot timestamps, if any -if ts is not None: - func_num = 0 - for f, exec_ts in ts.iteritems(): - for execution in exec_ts: - add_bracket([ts - global_start for ts in execution], t, mem, - color= all_colors[func_num % len(all_colors)], - label=f.split(".")[-1] + " %.3fs" % (execution[1] - execution[0])) - ## pl.vlines([ts - global_start for ts in execution], - ## bottom, top, - ## colors=all_colors[func_num % len(all_colors)], - ## linestyles="dashdot", - ## label=f.split(".")[-1] + " %.3fs" % (execution[1] - execution[0])) - func_num += 1 - pl.legend(loc=0) - -pl.hlines(max_mem, - pl.xlim()[0] + 0.001, pl.xlim()[1] - 0.001, - colors="r", linestyles="--") -pl.vlines(t[max_mem_ind], bottom, top, - colors="r", linestyles="--") -pl.xlabel("time [s]") -pl.ylabel("memory used [MB]") -title = time.strftime("%d / %m / %Y - start at %H:%M:%S", time.localtime(global_start)) \ - + ".{0:03d}".format(int(round(math.modf(global_start)[0]*1000))) -pl.title(title) -pl.grid() -pl.show() +def plot_file(filename, index=0, timestamps=True): + # Check for a timestamp file + file_parts = osp.splitext(filename) + ts_filename = file_parts[0] + "_ts" + file_parts[1] + ts = read_timestamp_file(ts_filename) + + mdata = pl.atleast_2d(pl.loadtxt(filename)) + + global_start = float(mdata[0, 1]) + + mem = mdata[:, 0] + max_mem = mem.max() + max_mem_ind = mem.argmax() + + t = mdata[:, 1] - global_start + + all_colors=("c", "y", "g", "r", "b") + mem_line_colors=('k', "b", "r") + mem_line_label = time.strftime("%d / %m / %Y - start at %H:%M:%S", + time.localtime(global_start)) \ + + ".{0:03d}".format(int(round(math.modf(global_start)[0]*1000))) + + pl.plot(t, mem, "+-" + mem_line_colors[index % len(mem_line_colors)], + label=mem_line_label) + + bottom, top = pl.ylim() + bottom += 0.001 + top -= 0.001 + + # plot timestamps, if any + if ts is not None and timestamps: + func_num = 0 + for f, exec_ts in ts.iteritems(): + for execution in exec_ts: + add_bracket([ts - global_start for ts in execution], t, mem, + color= all_colors[func_num % len(all_colors)], + label=f.split(".")[-1] + " %.3fs" % (execution[1] - execution[0])) + func_num += 1 + + if timestamps: + pl.hlines(max_mem, + pl.xlim()[0] + 0.001, pl.xlim()[1] - 0.001, + colors="r", linestyles="--") + pl.vlines(t[max_mem_ind], bottom, top, + colors="r", linestyles="--") + + +if __name__ == "__main__": + profiles = glob.glob("mprofile_??????????????.dat") + profiles.sort() + + print (sys.argv) + if len(sys.argv) == 1: + if len(profiles) == 0: + print("""No input file found. This program looks for mprofile_*.dat files, + generated by the mprofile command.""") + sys.exit(-1) + filenames = [profiles[-1]] + else: + filenames = [] + for arg in sys.argv[1:]: + if osp.exists(arg): + if not arg in filenames: + filenames.append(arg) + else: + try: + n = int(arg) + except ValueError: + print("Input file not found: " + arg) + if not profiles[n] in filenames: + filenames.append(profiles[n]) + + pl.figure(figsize=(14,6), dpi=90) + if len(filenames) > 1: + timestamps = False + else: + timestamps = True + for n, filename in enumerate(filenames): + plot_file(filename, index=n, timestamps=timestamps) + pl.xlabel("time [s]") + pl.ylabel("memory used [MB]") + + ax = pl.gca() + box = ax.get_position() + ax.set_position([0.07, 0.1, + 0.55, 0.8]) + ax.legend(loc="upper left", bbox_to_anchor=(1.05, 1.)) + pl.grid() + pl.show() From a5e3231e46024d7a5cb33374d098593220b3a6e3 Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Sun, 14 Apr 2013 20:59:29 +0200 Subject: [PATCH 014/415] cosmetic --- README.rst | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index c64366b..be3fab6 100644 --- a/README.rst +++ b/README.rst @@ -1,11 +1,10 @@ ================= Memory Profiler ================= + This is a python module for monitoring memory consumption of a process as well as line-by-line analysis of memory consumption for python -programs. - -It's a pure python module and has the `psutil +programs. It is a pure python module and has the `psutil `_ module as optional (but highly recommended) dependencies. @@ -25,10 +24,15 @@ To install from source, download the package, extract and type:: ======= Usage ======= + The line-by-line profiler is used much in the same way of the -line_profiler: you must first decorate the function you would like to -profile with ``@profile``. In this example, we create a simple function -``my_func`` that allocates lists ``a``, ``b`` and then deletes ``b``:: +`line_profiler `_: first +decorate the function you would like to profile with ``@profile`` and +then run the script with a special script (in this case with specific +arguments to the Python interpreter). + +In the following example, we create a simple function ``my_func`` that +allocates lists ``a``, ``b`` and then deletes ``b``:: @profile From 3319aee087f4f397c16de38dd036788dd9be8ccb Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Tue, 16 Apr 2013 11:28:37 +0200 Subject: [PATCH 015/415] Added maction This script is intended for various actions of profile files. Only removal is implemented in this commit. maction rm -1 # deletes most recent profile from current dir. --- maction | 120 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ mplot | 1 - 2 files changed, 120 insertions(+), 1 deletion(-) create mode 100755 maction diff --git a/maction b/maction new file mode 100755 index 0000000..85da71c --- /dev/null +++ b/maction @@ -0,0 +1,120 @@ +#! /usr/bin/env python + +import glob +import os +import os.path as osp +import sys +import re + +from optparse import OptionParser + +import memory_profiler as mp + + +def print_usage(): + print("Usage: %s " + % osp.basename(sys.argv[0])) + +def get_action(): + """Pop first argument, check it is a valid action.""" + all_actions = ("rm",) + if len(sys.argv) <= 1: + print_usage() + sys.exit(1) + if not sys.argv[1] in all_actions: + print("Valid actions are: " + " ".join(all_actions)) + sys.exit(1) + + return sys.argv.pop(1) + + +def get_profile_filenames(args): + """Return list of profile filenames. + + Parameters + ========== + args (list) + list of filename or integer. An integer is the index of the + profile in the list of existing profiles. 0 is the oldest, + -1 in the more recent. + Non-existing files cause a ValueError exception to be thrown. + + Returns + ======= + filenames (list) + list of existing memory profile filenames. It is guaranteed + that an given file name will not appear twice in this list. + """ + profiles = glob.glob("mprofile_??????????????.dat") + profiles.sort() + + filenames = [] + + for arg in args: + if arg == "--": # workaround + continue + try: + index = int(arg) + except ValueError: + index = None + if index is not None: + try: + filename = profiles[index] + except IndexError: + raise ValueError("Invalid index (non-existing file): %s" % arg) + + if filename not in filenames: + filenames.append(filename) + else: + if osp.isfile(arg): + if arg not in filenames: + filenames.append(arg) + elif osp.isdir(arg): + raise ValueError("Path %s is a directory" % arg) + else: + raise ValueError("File %s not found" % arg) + + # Add timestamp files, if any + for filename in reversed(filenames): + parts = osp.splitext(filename) + timestamp_file = parts[0] + "_ts" + parts[1] + if osp.isfile(timestamp_file) and timestamp_file not in filenames: + filenames.append(timestamp_file) + + return filenames + + +def rm_action(): + parser = OptionParser(version=mp.__version__) + parser.disable_interspersed_args() + parser.add_option("--dry-run", dest="dry_run", default=False, + action="store_true", + help="""Show what will be done, without actually doing it.""") + + (options, args) = parser.parse_args() + + if len(args) == 0: + print("A profile to remove must be provided (number or filename)") + sys.exit(1) + + filenames = get_profile_filenames(args) + if options.dry_run: + print("Files to be removed: ") + for filename in filenames: + print(filename) + else: + for filename in filenames: + os.remove(filename) + + +if __name__ == "__main__": + # Workaround for optparse limitation: insert -- before first negative number found. + negint = re.compile("-[0-9]+") + for n, arg in enumerate(sys.argv): + if negint.match(arg): + sys.argv.insert(n, "--") + break + actions = {"rm": rm_action} + actions[get_action()]() + + diff --git a/mplot b/mplot index 6a9390c..dd80bfc 100755 --- a/mplot +++ b/mplot @@ -117,7 +117,6 @@ if __name__ == "__main__": profiles = glob.glob("mprofile_??????????????.dat") profiles.sort() - print (sys.argv) if len(sys.argv) == 1: if len(profiles) == 0: print("""No input file found. This program looks for mprofile_*.dat files, From f345720bdd26ec24491f416fac5e7aa0acace2e2 Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Sun, 21 Apr 2013 16:32:43 +0200 Subject: [PATCH 016/415] Hide memory_profiler.py from argument list --- memory_profiler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/memory_profiler.py b/memory_profiler.py index 22f8f65..62af782 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -595,6 +595,7 @@ def wrapper(*args, **kwargs): sys.exit(2) (options, args) = parser.parse_args() + del sys.argv[0] # Hide "memory_profiler.py" from argument list prof = LineProfiler(max_mem=options.max_mem) __file__ = _find_script(args[0]) From 3cab3b93938c3d5035996ed3dc86642a4e9ccf6e Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Sun, 21 Apr 2013 16:34:48 +0200 Subject: [PATCH 017/415] version 0.26 --- memory_profiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/memory_profiler.py b/memory_profiler.py index 62af782..1e77a68 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -1,6 +1,6 @@ """Profile the memory usage of a Python program""" -__version__ = '0.25' +__version__ = '0.26' _CMD_USAGE = "python -m memory_profiler script_file.py" From dd421c6b1d2ea46e51ea066feb588ac6ecc0b7ea Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Thu, 25 Apr 2013 17:36:38 +0200 Subject: [PATCH 018/415] profile.timestamp is a context manager Timestamp block of codes is now possible thanks to the profile.timestamp() function that returns a context manager. Example: with profile.timestamp("block1"): f() "block1" is the label that will be used for the plotting. --- memory_profiler.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/memory_profiler.py b/memory_profiler.py index 47e3187..f85d5d9 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -214,6 +214,19 @@ def _find_script(script_name): raise SystemExit(1) +class _TimeStamperCM(object): + """Time-stamping context manager.""" + + def __init__(self, timestamps): + self._timestamps = timestamps + + def __enter__(self): + self._timestamps.append(time.time()) + + def __exit__(self, *args): + self._timestamps.append(time.time()) + + class TimeStamper: """ A profiler that just records start and end execution times for any decorated function. @@ -233,6 +246,19 @@ def __call__(self, func): f.__dict__.update(getattr(func, '__dict__', {})) return f + def timestamp(self, name=""): + """Returns a context manager for timestamping a block of code.""" + # Make a fake function + func = lambda x: x + func.__module__ = "" + func.__name__ = name + self.add_function(func) + timestamps = [] + self.functions[func].append(timestamps) + # A new object is required each time, since there can be several + # nested context managers. + return _TimeStamperCM(timestamps) + def add_function(self, func): if not func in self.functions: self.functions[func] = [] @@ -309,7 +335,7 @@ def f(*args, **kwds): return f def run(self, cmd): - """ Profile a single executable statment in the main namespace. + """ Profile a single executable statement in the main namespace. """ import __main__ main_dict = __main__.__dict__ From bd5e9007c69d59fbadc7cb858ac56c96019746c5 Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Mon, 29 Apr 2013 10:08:15 +0200 Subject: [PATCH 019/415] Added numpy_example.py This script is intended to show differences between numbers output by 'python -m memory_profiler' and 'mprofile --python' --- examples/numpy_example.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 examples/numpy_example.py diff --git a/examples/numpy_example.py b/examples/numpy_example.py new file mode 100644 index 0000000..eb2eefe --- /dev/null +++ b/examples/numpy_example.py @@ -0,0 +1,23 @@ +import numpy as np +import scipy.signal + + +#@profile +def create_data(): + ret = [] + for n in xrange(70): + ret.append(np.random.randn(1, 70, 71, 72)) + return ret + + +#@profile +def process_data(data): + data = np.concatenate(data) + detrended = scipy.signal.detrend(data, axis=0) + return detrended + + +if __name__ == "__main__": + data1 = create_data() + data2 = process_data(data1) + print (data2.shape) From a8c848f748d73c1c67e8cd050b3ace0fe6687fd3 Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Sun, 5 May 2013 21:52:32 +0200 Subject: [PATCH 020/415] _get_memory() uses the resource module from std lib. --- Makefile | 7 +++++++ memory_profiler.py | 44 +++++++++++++++++++++++++++++++------------- test/test_with.py | 1 - 3 files changed, 38 insertions(+), 14 deletions(-) create mode 100644 Makefile delete mode 100644 test/test_with.py diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d0a2e31 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ + +.PHONY: test + +test: + python -m memory_profiler test/test_func.py + python -m memory_profiler test/test_loop.py + python -m memory_profiler test/test_with.py \ No newline at end of file diff --git a/memory_profiler.py b/memory_profiler.py index 1e77a68..96f81c0 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -17,25 +17,45 @@ except ImportError: from multiprocessing.dummy import Process, Pipe +_TWO_20 = float(2 ** 20) +has_psutil = False +has_resource = False + +# .. get available packages .. try: import psutil + has_psutil = True +except ImportError: + pass + +try: + import resource + has_resource = True +except ImportError: + pass + + +def _get_memory(pid): - def _get_memory(pid): + # .. fastests but just works for current process .. + # .. and only available on unix .. + if pid == -1 and has_resource: + mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (_TWO_20) + return mem + + # .. good compromise but requires psutil .. + if has_psutil: process = psutil.Process(pid) try: - mem = float(process.get_memory_info()[0]) / (1024 ** 2) + mem = process.get_memory_info()[0] / (_TWO_20) except psutil.AccessDenied: mem = -1 return mem - -except ImportError: - - warnings.warn("psutil module not found. memory_profiler will be slow") - + # .. scary stuff .. if os.name == 'posix': - def _get_memory(pid): + warnings.warn("psutil module not found. memory_profiler will be slow") # .. # .. memory usage in MB .. # .. this should work on both Mac and Linux .. @@ -133,7 +153,7 @@ def memory_usage(proc=-1, interval=.1, timeout=None): % (n_args, len(args))) child_conn, parent_conn = Pipe() # this will store Timer's results - p = Timer(os.getpid(), interval, child_conn) + p = Timer(-1, interval, child_conn) p.start() parent_conn.recv() # wait until we start getting memory f(*args, **kw) @@ -153,8 +173,6 @@ def memory_usage(proc=-1, interval=.1, timeout=None): break else: # external process - if proc == -1: - proc = os.getpid() if max_iter == -1: max_iter = 1 counter = 0 @@ -282,14 +300,14 @@ def trace_memory_usage(self, frame, event, arg): if event == 'return': lineno += 1 entry = self.code_map[frame.f_code].setdefault(lineno, []) - entry.append(_get_memory(os.getpid())) + entry.append(_get_memory(-1)) return self.trace_memory_usage def trace_max_mem(self, frame, event, arg): # run into PDB as soon as memory is higher than MAX_MEM if event in ('line', 'return') and frame.f_code in self.code_map: - c = _get_memory(os.getpid()) + c = _get_memory(-1) if c >= self.max_mem: t = 'Current memory {0:.2f} MB exceeded the maximum '.format(c) + \ 'of {0:.2f} MB\n'.format(self.max_mem) diff --git a/test/test_with.py b/test/test_with.py deleted file mode 100644 index 5e512ae..0000000 --- a/test/test_with.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'fabian' From f75f2839e8dcc6652ed37d20999b229e24c170cf Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Sun, 5 May 2013 22:29:30 +0200 Subject: [PATCH 021/415] Clean management of global variables. Added a bunch of tests --- Makefile | 4 +++- memory_profiler.py | 13 +++++-------- test/test_as.py | 9 +++++++++ test/test_global.py | 4 ++++ test/test_import.py | 11 +++++++++++ 5 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 test/test_as.py create mode 100644 test/test_global.py create mode 100644 test/test_import.py diff --git a/Makefile b/Makefile index d0a2e31..ddddf18 100644 --- a/Makefile +++ b/Makefile @@ -4,4 +4,6 @@ test: python -m memory_profiler test/test_func.py python -m memory_profiler test/test_loop.py - python -m memory_profiler test/test_with.py \ No newline at end of file + python -m memory_profiler test/test_as.py + python -m memory_profiler test/test_global.py + python test/test_import.py \ No newline at end of file diff --git a/memory_profiler.py b/memory_profiler.py index 96f81c0..a8b4c2e 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -2,6 +2,8 @@ __version__ = '0.26' +_clean_globals = globals() + _CMD_USAGE = "python -m memory_profiler script_file.py" import time, sys, os, pdb @@ -619,17 +621,12 @@ def wrapper(*args, **kwargs): __file__ = _find_script(args[0]) try: if sys.version_info[0] < 3: - import __builtin__ - __builtin__.__dict__['profile'] = prof - ns = copy(locals()) + ns = copy(_clean_globals) ns['profile'] = prof # shadow the profile decorator defined above execfile(__file__, ns, ns) else: - import builtins - builtins.__dict__['profile'] = prof - ns = copy(locals()) + ns = copy(_clean_globals) ns['profile'] = prof # shadow the profile decorator defined above - exec(compile(open(__file__).read(), __file__, 'exec'), - ns, copy(globals())) + exec(compile(open(__file__).read(), __file__, 'exec'), ns, ns) finally: show_results(prof, precision=options.precision) diff --git a/test/test_as.py b/test/test_as.py new file mode 100644 index 0000000..dc1a4ce --- /dev/null +++ b/test/test_as.py @@ -0,0 +1,9 @@ +import math + +@profile +def f(): + o = math.sqrt(2013) + return o + +if __name__ == '__main__': + f() \ No newline at end of file diff --git a/test/test_global.py b/test/test_global.py new file mode 100644 index 0000000..c8316d3 --- /dev/null +++ b/test/test_global.py @@ -0,0 +1,4 @@ +options = None + + +# test for dc0c8aa60b5960d240b0dcea270efa1e5a314a2c diff --git a/test/test_import.py b/test/test_import.py new file mode 100644 index 0000000..9f21444 --- /dev/null +++ b/test/test_import.py @@ -0,0 +1,11 @@ +from memory_profiler import profile + +@profile +def my_func(): + a = [1] * (10 ** 6) + b = [2] * (2 * 10 ** 7) + del b + return a + +if __name__ == '__main__': + my_func() \ No newline at end of file From 5dcc68b547ee49fcc306df6c8615791dd1b293d1 Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Sun, 5 May 2013 22:33:42 +0200 Subject: [PATCH 022/415] cosmetic. --- memory_profiler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/memory_profiler.py b/memory_profiler.py index a8b4c2e..1674bd2 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -2,7 +2,8 @@ __version__ = '0.26' -_clean_globals = globals() +# .. we'll use this to pass it to the child script .. +_clean_globals = globals().copy() _CMD_USAGE = "python -m memory_profiler script_file.py" From 18b5afb651e026de7e0754c931800391dc72c193 Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Sun, 5 May 2013 22:34:31 +0200 Subject: [PATCH 023/415] cosmetic. --- memory_profiler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/memory_profiler.py b/memory_profiler.py index 1674bd2..112c967 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -1,10 +1,10 @@ """Profile the memory usage of a Python program""" -__version__ = '0.26' - # .. we'll use this to pass it to the child script .. _clean_globals = globals().copy() +__version__ = '0.26' + _CMD_USAGE = "python -m memory_profiler script_file.py" import time, sys, os, pdb From 07c7c988958428e4f2d3640d10f681dae880f5ed Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Sun, 5 May 2013 23:34:34 +0200 Subject: [PATCH 024/415] FIX: PID for ps --- memory_profiler.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/memory_profiler.py b/memory_profiler.py index 112c967..e3346fa 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -43,9 +43,12 @@ def _get_memory(pid): # .. fastests but just works for current process .. # .. and only available on unix .. - if pid == -1 and has_resource: - mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (_TWO_20) - return mem + if pid == -1: + if has_resource: + mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (_TWO_20) + return mem + else: + pid = os.getpid() # .. good compromise but requires psutil .. if has_psutil: From 9d55698b5e9b55979c8949f8abde82c8392e33cc Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Tue, 7 May 2013 10:45:19 +0200 Subject: [PATCH 025/415] Correct units (for linux) Not sure this should be the same on OSX ... --- memory_profiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/memory_profiler.py b/memory_profiler.py index e3346fa..0e89ccc 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -45,7 +45,7 @@ def _get_memory(pid): # .. and only available on unix .. if pid == -1: if has_resource: - mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (_TWO_20) + mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024. return mem else: pid = os.getpid() From 3f22f6d4b8a21d270a19e6d602150b718e623559 Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Tue, 7 May 2013 10:49:10 +0200 Subject: [PATCH 026/415] optional variable $PYTHON in makefile --- Makefile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index ddddf18..7817405 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,10 @@ +PYTHON ?= python .PHONY: test test: - python -m memory_profiler test/test_func.py - python -m memory_profiler test/test_loop.py - python -m memory_profiler test/test_as.py - python -m memory_profiler test/test_global.py - python test/test_import.py \ No newline at end of file + $(PYTHON) -m memory_profiler test/test_func.py + $(PYTHON) -m memory_profiler test/test_loop.py + $(PYTHON) -m memory_profiler test/test_as.py + $(PYTHON) -m memory_profiler test/test_global.py + $(PYTHON) test/test_import.py \ No newline at end of file From ee3f8bd78afe9f53f6251fadbfd85dcd4b04528a Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Tue, 7 May 2013 10:53:45 +0200 Subject: [PATCH 027/415] Add file README_DEV --- README_DEV.rst | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 README_DEV.rst diff --git a/README_DEV.rst b/README_DEV.rst new file mode 100644 index 0000000..3e66e51 --- /dev/null +++ b/README_DEV.rst @@ -0,0 +1,7 @@ +Some information on the internals of this package. + +Tests +----- +`make test` is the closest thing to tests on this package. It executes some +example code and prints the information. If you don't see any exceptions nor +any strange output then the tests suite "has succeeded". From 0726230a6e8781be1a9a3fd3d13df456e0966b87 Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Tue, 7 May 2013 10:55:54 +0200 Subject: [PATCH 028/415] Set version to 0.26-git --- memory_profiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/memory_profiler.py b/memory_profiler.py index 0e89ccc..e3e578e 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -3,7 +3,7 @@ # .. we'll use this to pass it to the child script .. _clean_globals = globals().copy() -__version__ = '0.26' +__version__ = '0.26-git' _CMD_USAGE = "python -m memory_profiler script_file.py" From 44b1791416ab504b65d83bb2c0b2b5781dca405f Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Tue, 7 May 2013 11:02:14 +0200 Subject: [PATCH 029/415] Cleanup --- memory_profiler.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/memory_profiler.py b/memory_profiler.py index e3e578e..a25fa8e 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -41,8 +41,7 @@ def _get_memory(pid): - # .. fastests but just works for current process .. - # .. and only available on unix .. + # .. only for current process and only on unix.. if pid == -1: if has_resource: mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024. @@ -50,7 +49,7 @@ def _get_memory(pid): else: pid = os.getpid() - # .. good compromise but requires psutil .. + # .. cross-platform but but requires psutil .. if has_psutil: process = psutil.Process(pid) try: @@ -273,16 +272,6 @@ def runctx(self, cmd, globals, locals): self.disable_by_count() return self - def runcall(self, func, *args, **kw): - """ Profile a single function call. - """ - # XXX where is this used ? can be removed ? - self.enable_by_count() - try: - return func(*args, **kw) - finally: - self.disable_by_count() - def enable_by_count(self): """ Enable the profiler if it hasn't been enabled before. """ From 99a68d176e334e1e4100f51ba40e5c037e63f5c0 Mon Sep 17 00:00:00 2001 From: xbjiang Date: Wed, 8 May 2013 16:48:09 +0800 Subject: [PATCH 030/415] using xrange in python2 --- memory_profiler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/memory_profiler.py b/memory_profiler.py index a25fa8e..0ba8f48 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -20,6 +20,11 @@ except ImportError: from multiprocessing.dummy import Process, Pipe +try: + xrange +except NameError: + xrange = range + _TWO_20 = float(2 ** 20) has_psutil = False @@ -564,7 +569,7 @@ def magic_memit(self, line=''): timeout = None mem_usage = [] - for _ in range(repeat): + for _ in xrange(repeat): tmp = memory_usage((_func_exec, (stmt, self.shell.user_ns)), timeout=timeout) mem_usage.extend(tmp) From 5f137da4c07fbf68b67d417660a0373fcc2bf8ef Mon Sep 17 00:00:00 2001 From: s7v7nislands Date: Thu, 9 May 2013 08:44:13 +0800 Subject: [PATCH 031/415] using while --- memory_profiler.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/memory_profiler.py b/memory_profiler.py index 0ba8f48..3f0a871 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -20,11 +20,6 @@ except ImportError: from multiprocessing.dummy import Process, Pipe -try: - xrange -except NameError: - xrange = range - _TWO_20 = float(2 ** 20) has_psutil = False @@ -569,7 +564,9 @@ def magic_memit(self, line=''): timeout = None mem_usage = [] - for _ in xrange(repeat): + counter = 0 + while counter < repeat: + counter += 1 tmp = memory_usage((_func_exec, (stmt, self.shell.user_ns)), timeout=timeout) mem_usage.extend(tmp) From 2ff06e707ce79fe294fa85962ae6786662273d80 Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Sat, 18 May 2013 23:40:14 +0200 Subject: [PATCH 032/415] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index be3fab6..94b6642 100644 --- a/README.rst +++ b/README.rst @@ -231,7 +231,7 @@ file ~/.ipython/ipy_user_conf.py to add the following lines:: Support, bugs & wish list =========================== For support, please ask your question on `stack overflow -`_ and add the *profiling* tag. +`_ and add the *memory-profiler* tag. Send issues, proposals, etc. to `github's issue tracker `_ . From 7ec2b316420ee192445910e5aeb4805ee24d23e7 Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Tue, 21 May 2013 18:47:11 +0200 Subject: [PATCH 033/415] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 94b6642..dadb4d3 100644 --- a/README.rst +++ b/README.rst @@ -231,7 +231,7 @@ file ~/.ipython/ipy_user_conf.py to add the following lines:: Support, bugs & wish list =========================== For support, please ask your question on `stack overflow -`_ and add the *memory-profiler* tag. +`_ and add the *memory-profiling* tag. Send issues, proposals, etc. to `github's issue tracker `_ . From 10956a8e2a6a173908137cff68cd8e00f59ad2f3 Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Mon, 27 May 2013 10:08:00 +0200 Subject: [PATCH 034/415] Memory usage at entering and leaving functions When using timestamping, memory usage is now recorded when entering and leaving functions. This provides for more accurate measurement at these points. mplot has been adapted to handle this new piece of information. --- memory_profiler.py | 12 +++++++----- mplot | 23 ++++++++++++----------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/memory_profiler.py b/memory_profiler.py index f85d5d9..79e9e5b 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -221,10 +221,10 @@ def __init__(self, timestamps): self._timestamps = timestamps def __enter__(self): - self._timestamps.append(time.time()) + self._timestamps.append(_get_memory(os.getpid(), timestamps=True)) def __exit__(self, *args): - self._timestamps.append(time.time()) + self._timestamps.append(_get_memory(os.getpid(), timestamps=True)) class TimeStamper: @@ -268,13 +268,13 @@ def wrap_function(self, func): """ def f(*args, **kwds): # Start time - timestamps = [time.time()] + timestamps = [_get_memory(os.getpid(), timestamps=True)] self.functions[func].append(timestamps) try: result = func(*args, **kwds) finally: # end time - timestamps.append(time.time()) + timestamps.append(_get_memory(os.getpid(), timestamps=True)) return result return f @@ -285,7 +285,9 @@ def show_results(self, stream=None): for func, timestamps in self.functions.iteritems(): function_name = "%s.%s" % (func.__module__, func.__name__) for ts in timestamps: - stream.write("%s %.4f %.4f\n" % (function_name, ts[0], ts[1])) + stream.write("%s %.4f %.4f %.4f %.4f\n" % ( + (function_name,) + ts[0] + ts[1])) + ## stream.write("%s %.4f %.4f\n" % (function_name, ts[0], ts[1])) class LineProfiler: diff --git a/mplot b/mplot index dd80bfc..ee86f1c 100755 --- a/mplot +++ b/mplot @@ -13,30 +13,31 @@ import os import os.path as osp -def add_bracket(xloc, t, mem, color="r", label=None): +def add_bracket(xloc, yloc, xshift=0, color="r", label=None): """Add two brackets on the memory line plot. This function uses the current figure. Parameters ========== - xloc: {tuple with 2 values} - bracket location (on horizontal axis). - t, mem: - memory usage curve. Used to place bracket at the correct location. + xloc: tuple with 2 values + brackets location (on horizontal axis). + yloc: tuple with 2 values + brackets location (on vertical axis) + xshift: float + value to subtract to xloc. """ height_ratio = 20. - yloc = pl.interp(xloc, t, mem) vsize = (pl.ylim()[1] - pl.ylim()[0]) / height_ratio hsize = (pl.xlim()[1] - pl.xlim()[0]) / (3.*height_ratio) bracket_x = pl.asarray([hsize, 0, 0, hsize]) bracket_y = pl.asarray([vsize, vsize, -vsize, -vsize]) - pl.plot(bracket_x + xloc[0], bracket_y + yloc[0], + pl.plot(bracket_x + xloc[0] - xshift, bracket_y + yloc[0], "-" + color, linewidth=2, label=label) - pl.plot(-bracket_x + xloc[1], bracket_y + yloc[1], + pl.plot(-bracket_x + xloc[1] - xshift, bracket_y + yloc[1], "-" + color, linewidth=2 ) # TODO: use matplotlib.patches.Polygon to draw a colored background for @@ -58,9 +59,9 @@ def read_timestamp_file(ts_filename): ret = {} f = open(ts_filename) for l in f: - f_name, start, end = l.split() + f_name, mem_start, start, mem_end, end = l.split() ts = ret.get(f_name, []) - ts.append([float(start), float(end)]) + ts.append([float(start), float(end), float(mem_start), float(mem_end)]) ret[f_name] = ts f.close() return ret @@ -100,7 +101,7 @@ def plot_file(filename, index=0, timestamps=True): func_num = 0 for f, exec_ts in ts.iteritems(): for execution in exec_ts: - add_bracket([ts - global_start for ts in execution], t, mem, + add_bracket(execution[:2], execution[2:], xshift=global_start, color= all_colors[func_num % len(all_colors)], label=f.split(".")[-1] + " %.3fs" % (execution[1] - execution[0])) func_num += 1 From f3ec782486a6db33118867f6c060a4ebdd8ec495 Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Mon, 27 May 2013 11:14:34 +0200 Subject: [PATCH 035/415] Added "clean" action to maction This action removes all profile files in the current directory. --- maction | 76 ++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 25 deletions(-) diff --git a/maction b/maction index 85da71c..84a0b9f 100755 --- a/maction +++ b/maction @@ -5,6 +5,7 @@ import os import os.path as osp import sys import re +import copy from optparse import OptionParser @@ -17,7 +18,7 @@ def print_usage(): def get_action(): """Pop first argument, check it is a valid action.""" - all_actions = ("rm",) + all_actions = ("rm", "clean") if len(sys.argv) <= 1: print_usage() sys.exit(1) @@ -48,31 +49,32 @@ def get_profile_filenames(args): profiles = glob.glob("mprofile_??????????????.dat") profiles.sort() - filenames = [] - - for arg in args: - if arg == "--": # workaround - continue - try: - index = int(arg) - except ValueError: - index = None - if index is not None: + if args is "all": + filenames = copy.copy(profiles) + else: + for arg in args: + if arg == "--": # workaround + continue try: - filename = profiles[index] - except IndexError: - raise ValueError("Invalid index (non-existing file): %s" % arg) - - if filename not in filenames: - filenames.append(filename) - else: - if osp.isfile(arg): - if arg not in filenames: - filenames.append(arg) - elif osp.isdir(arg): - raise ValueError("Path %s is a directory" % arg) + index = int(arg) + except ValueError: + index = None + if index is not None: + try: + filename = profiles[index] + except IndexError: + raise ValueError("Invalid index (non-existing file): %s" % arg) + + if filename not in filenames: + filenames.append(filename) else: - raise ValueError("File %s not found" % arg) + if osp.isfile(arg): + if arg not in filenames: + filenames.append(arg) + elif osp.isdir(arg): + raise ValueError("Path %s is a directory" % arg) + else: + raise ValueError("File %s not found" % arg) # Add timestamp files, if any for filename in reversed(filenames): @@ -107,6 +109,30 @@ def rm_action(): os.remove(filename) +def clean_action(): + """Remove every profile file in current directory.""" + parser = OptionParser(version=mp.__version__) + parser.disable_interspersed_args() + parser.add_option("--dry-run", dest="dry_run", default=False, + action="store_true", + help="""Show what will be done, without actually doing it.""") + + (options, args) = parser.parse_args() + + if len(args) > 0: + print("This command takes no argument.") + sys.exit(1) + + filenames = get_profile_filenames("all") + if options.dry_run: + print("Files to be removed: ") + for filename in filenames: + print(filename) + else: + for filename in filenames: + os.remove(filename) + + if __name__ == "__main__": # Workaround for optparse limitation: insert -- before first negative number found. negint = re.compile("-[0-9]+") @@ -114,7 +140,7 @@ if __name__ == "__main__": if negint.match(arg): sys.argv.insert(n, "--") break - actions = {"rm": rm_action} + actions = {"rm": rm_action, "clean": clean_action} actions[get_action()]() From 5d241dad8b8c6979d66b73bd0fdb02d780240e4e Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Mon, 27 May 2013 18:23:32 +0200 Subject: [PATCH 036/415] Corrected a small typo left after the merge. --- memory_profiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/memory_profiler.py b/memory_profiler.py index 618d2e1..7cd878f 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -59,7 +59,7 @@ def _get_memory(pid, timestamps=False, include_children=False): mem = process.get_memory_info()[0] / (_TWO_20) if include_children: for p in process.get_children(recursive=True): - mem += p.get_memory_info()[0] / (_TWO_20). + mem += p.get_memory_info()[0] / (_TWO_20) except psutil.AccessDenied: mem = -1 if timestamps: From 4e967155282a2583e1d49b30a9b722166d96dad5 Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Thu, 6 Jun 2013 07:47:59 +0200 Subject: [PATCH 037/415] Update README.rst --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index dadb4d3..84140bc 100644 --- a/README.rst +++ b/README.rst @@ -248,6 +248,11 @@ Latest sources are available from github: https://github.com/fabianp/memory_profiler +=============================== +Projects using memory_profiler +=============================== + +`Benchy `_ ========= Authors From 92a30905d37ff88a5d959a7d0afc216b0471eb62 Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Mon, 17 Jun 2013 15:53:36 +0200 Subject: [PATCH 038/415] Override builtins --- memory_profiler.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/memory_profiler.py b/memory_profiler.py index 3f0a871..fa2add0 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -616,10 +616,18 @@ def wrapper(*args, **kwargs): __file__ = _find_script(args[0]) try: if sys.version_info[0] < 3: + # we need to ovewrite the builtins to have profile + # globally defined (global variables is not enought + # for all cases, e.g. a script that imports another + # script where @profile is used) + import __builtin__ + __builtin__.__dict__['profile'] = prof ns = copy(_clean_globals) ns['profile'] = prof # shadow the profile decorator defined above execfile(__file__, ns, ns) else: + import builtins + builtins.__dict__['profile'] = prof ns = copy(_clean_globals) ns['profile'] = prof # shadow the profile decorator defined above exec(compile(open(__file__).read(), __file__, 'exec'), ns, ns) From 6be0021673351d7a6687850056dfbbe9e705e49e Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Mon, 17 Jun 2013 17:21:13 +0200 Subject: [PATCH 039/415] Timestamps and memory usage stored in one file Got rid of the _ts file for timestamp storage. The file format has been changed just for this. It opens the possibility for many new features (like per-process memory monitoring). --- memory_profiler.py | 5 ++-- mplot | 65 +++++++++++++++++++++++++--------------------- mprofile | 8 +++--- 3 files changed, 42 insertions(+), 36 deletions(-) diff --git a/memory_profiler.py b/memory_profiler.py index cacb35c..89c4277 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -312,9 +312,8 @@ def show_results(self, stream=None): for func, timestamps in self.functions.iteritems(): function_name = "%s.%s" % (func.__module__, func.__name__) for ts in timestamps: - stream.write("%s %.4f %.4f %.4f %.4f\n" % ( + stream.write("FUNC %s %.4f %.4f %.4f %.4f\n" % ( (function_name,) + ts[0] + ts[1])) - ## stream.write("%s %.4f %.4f\n" % (function_name, ts[0], ts[1])) class LineProfiler: @@ -751,7 +750,7 @@ def wrapper(*args, **kwargs): exec(compile(open(__file__).read(), __file__, 'exec'), ns, ns) finally: if options.out_filename is not None: - out_file = open(options.out_filename, "w") + out_file = open(options.out_filename, "a") else: out_file = sys.stdout diff --git a/mplot b/mplot index ee86f1c..1d4d541 100755 --- a/mplot +++ b/mplot @@ -6,6 +6,8 @@ try: except ImportError: print("matplotlib is needed for plotting.") sys.exit(1) + +import numpy as np import math import glob import time @@ -13,7 +15,7 @@ import os import os.path as osp -def add_bracket(xloc, yloc, xshift=0, color="r", label=None): +def add_brackets(xloc, yloc, xshift=0, color="r", label=None): """Add two brackets on the memory line plot. This function uses the current figure. @@ -50,38 +52,42 @@ def add_bracket(xloc, yloc, xshift=0, color="r", label=None): ## pl.plot(xloc[1], yloc[1], ">"+color, markersize=7) -def read_timestamp_file(ts_filename): - """Return content of ts_filename or None, if ts_filename - is invalid or does not exist""" - if not osp.isfile(ts_filename): - return None - +def read_mprofile_file(filename): ret = {} - f = open(ts_filename) + mdata = [] + f = open(filename, "r") for l in f: - f_name, mem_start, start, mem_end, end = l.split() - ts = ret.get(f_name, []) - ts.append([float(start), float(end), float(mem_start), float(mem_end)]) - ret[f_name] = ts + fields = l.split() + if fields[0] == "MEM": + # mem, timestamp + mdata.append((fields[1], fields[2])) + + elif fields[0] == "FUNC": + f_name, mem_start, start, mem_end, end = fields[1:] + ts = ret.get(f_name, []) + ts.append([float(start), float(end), float(mem_start), float(mem_end)]) + ret[f_name] = ts + + else: + pass f.close() - return ret + mdata = np.asarray(mdata, + dtype=[("mem", np.float), ("timestamp", np.float)]) + return mdata, ret -def plot_file(filename, index=0, timestamps=True): - # Check for a timestamp file - file_parts = osp.splitext(filename) - ts_filename = file_parts[0] + "_ts" + file_parts[1] - ts = read_timestamp_file(ts_filename) - mdata = pl.atleast_2d(pl.loadtxt(filename)) - global_start = float(mdata[0, 1]) +def plot_file(filename, index=0, timestamps=True): + mdata, ts = read_mprofile_file(filename) + + global_start = float(mdata["timestamp"][0]) - mem = mdata[:, 0] + mem = mdata["mem"] max_mem = mem.max() max_mem_ind = mem.argmax() - t = mdata[:, 1] - global_start + t = mdata["timestamp"] - global_start all_colors=("c", "y", "g", "r", "b") mem_line_colors=('k', "b", "r") @@ -97,13 +103,14 @@ def plot_file(filename, index=0, timestamps=True): top -= 0.001 # plot timestamps, if any - if ts is not None and timestamps: + if len(ts) > 0 and timestamps: func_num = 0 for f, exec_ts in ts.iteritems(): for execution in exec_ts: - add_bracket(execution[:2], execution[2:], xshift=global_start, - color= all_colors[func_num % len(all_colors)], - label=f.split(".")[-1] + " %.3fs" % (execution[1] - execution[0])) + add_brackets(execution[:2], execution[2:], xshift=global_start, + color= all_colors[func_num % len(all_colors)], + label=f.split(".")[-1] + + " %.3fs" % (execution[1] - execution[0])) func_num += 1 if timestamps: @@ -120,8 +127,8 @@ if __name__ == "__main__": if len(sys.argv) == 1: if len(profiles) == 0: - print("""No input file found. This program looks for mprofile_*.dat files, - generated by the mprofile command.""") + print("No input file found. \nThis program looks for " + "mprofile_*.dat files, generated by the mprofile command.") sys.exit(-1) filenames = [profiles[-1]] else: @@ -138,7 +145,7 @@ if __name__ == "__main__": if not profiles[n] in filenames: filenames.append(profiles[n]) - pl.figure(figsize=(14,6), dpi=90) + pl.figure(figsize=(14, 6), dpi=90) if len(filenames) > 1: timestamps = False else: diff --git a/mprofile b/mprofile index fa6354b..ebb569c 100755 --- a/mprofile +++ b/mprofile @@ -44,17 +44,17 @@ mprofile_output = "mprofile_%s.dat" % suffix if options.python: print("running as a Python program...") - timestamp_output = "mprofile_%s_ts.dat" % suffix +# timestamp_output = "mprofile_%s_ts.dat" % suffix if not args[0].startswith("python"): args.insert(0, "python") args[1:1] = ("-m", "memory_profiler", "--timestamp", - "-o", timestamp_output) + "-o", mprofile_output) p = subprocess.Popen(args) else: p = subprocess.Popen(args) mu = mp.memory_usage(proc=p, interval=options.interval, timestamps=True, include_children=options.include_children) -with open(mprofile_output, "w") as f: +with open(mprofile_output, "a") as f: for m, t in mu: - f.write("{0:.6f} {1:.4f}".format(m, t) + "\n") + f.write("MEM {0:.6f} {1:.4f}".format(m, t) + "\n") From 31a3eae45eed3eae49a7c5b0aec9686da60ee76e Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Thu, 20 Jun 2013 10:38:55 +0200 Subject: [PATCH 040/415] Added "list" command to maction And fixed a bad bug in maction ("rm" wasn't working anymore). --- maction | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/maction b/maction index 84a0b9f..a1f0dae 100755 --- a/maction +++ b/maction @@ -18,7 +18,7 @@ def print_usage(): def get_action(): """Pop first argument, check it is a valid action.""" - all_actions = ("rm", "clean") + all_actions = ("rm", "clean", "list") if len(sys.argv) <= 1: print_usage() sys.exit(1) @@ -52,6 +52,7 @@ def get_profile_filenames(args): if args is "all": filenames = copy.copy(profiles) else: + filenames = [] for arg in args: if arg == "--": # workaround continue @@ -86,6 +87,26 @@ def get_profile_filenames(args): return filenames +def list_action(): + """Display existing profiles, with indices.""" + parser = OptionParser(version=mp.__version__) + parser.disable_interspersed_args() + + (options, args) = parser.parse_args() + + if len(args) > 0: + print("This command takes no argument.") + sys.exit(1) + + filenames = get_profile_filenames("all") + for n, filename in enumerate(filenames): + ts = osp.splitext(filename)[0].split('_')[-1] + print("{index} {filename} {hour}:{min}:{sec} {day}/{month}/{year}" + .format(index=n, filename=filename, + year=ts[:4], month=ts[4:6], day=ts[6:8], + hour=ts[8:10], min=ts[10:12], sec=ts[12:14])) + + def rm_action(): parser = OptionParser(version=mp.__version__) parser.disable_interspersed_args() @@ -140,7 +161,9 @@ if __name__ == "__main__": if negint.match(arg): sys.argv.insert(n, "--") break - actions = {"rm": rm_action, "clean": clean_action} + actions = {"rm": rm_action, + "clean": clean_action, + "list": list_action} actions[get_action()]() From 7c71e32b69e44570da77e2a8fdf6225639118151 Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Thu, 20 Jun 2013 11:01:10 +0200 Subject: [PATCH 041/415] Added a workaround for matplotlib strange behaviour --- mplot | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mplot b/mplot index 1d4d541..547bdbc 100755 --- a/mplot +++ b/mplot @@ -37,6 +37,9 @@ def add_brackets(xloc, yloc, xshift=0, color="r", label=None): bracket_x = pl.asarray([hsize, 0, 0, hsize]) bracket_y = pl.asarray([vsize, vsize, -vsize, -vsize]) + # Matplotlib workaround: labels starting with _ aren't displayed + if label[0] == '_': + label = ' ' + label pl.plot(bracket_x + xloc[0] - xshift, bracket_y + yloc[0], "-" + color, linewidth=2, label=label) pl.plot(-bracket_x + xloc[1] - xshift, bracket_y + yloc[1], From fd8a891ce4e59dae8177408c983c18fee90ad7ca Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Thu, 20 Jun 2013 14:36:56 +0200 Subject: [PATCH 042/415] No memory_profiler args is transmitted Invoked script used to received the memory_profiler options in sys.argv in addition of their own. This is completely fixed now. --- memory_profiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/memory_profiler.py b/memory_profiler.py index fa2add0..deab852 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -610,7 +610,7 @@ def wrapper(*args, **kwargs): sys.exit(2) (options, args) = parser.parse_args() - del sys.argv[0] # Hide "memory_profiler.py" from argument list + sys.argv[:] = args # Remove every memory_profiler arguments prof = LineProfiler(max_mem=options.max_mem) __file__ = _find_script(args[0]) From ed8281ab786f79800439a317a473fd1802517e13 Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Thu, 20 Jun 2013 15:48:43 +0300 Subject: [PATCH 043/415] Update README.rst --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 84140bc..44393b3 100644 --- a/README.rst +++ b/README.rst @@ -271,6 +271,8 @@ cleanup. `Thomas Kluyver `_ added the IPython extension. +Philippe Gervais made several enhacements and bug fixes. + ========= From a05c9c132cc1debffb64fd606ce7fe8a5faf1420 Mon Sep 17 00:00:00 2001 From: cheesinglee Date: Fri, 21 Jun 2013 12:16:12 -0700 Subject: [PATCH 044/415] Add max_usage and retval kwargs to memory_usage() The max_usage parameter modifies the behavior of the function so that it only keeps track of the peak mem usage over the duration of the profiling. This is useful for profiling very long running processes. The retval parameter is for profiling Python functions. If set, the return value of the profiled function will be kept and returned in a tuple along with the memory usage. Signed-off-by: cheesinglee --- memory_profiler.py | 45 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/memory_profiler.py b/memory_profiler.py index deab852..47d5a66 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -83,24 +83,31 @@ class Timer(Process): Fetch memory consumption from over a time interval """ - def __init__(self, monitor_pid, interval, pipe, *args, **kw): + def __init__(self, monitor_pid, interval, pipe, max_usage=False, *args, **kw): self.monitor_pid = monitor_pid self.interval = interval self.pipe = pipe self.cont = True + self.max_usage = max_usage super(Timer, self).__init__(*args, **kw) def run(self): m = _get_memory(self.monitor_pid) - timings = [m] + if not self.max_usage: + timings = [m] + else: + timings = m self.pipe.send(0) # we're ready while not self.pipe.poll(self.interval): m = _get_memory(self.monitor_pid) - timings.append(m) + if not self.max_usage: + timings.append(m) + else: + timings = max([m,timings]) self.pipe.send(timings) -def memory_usage(proc=-1, interval=.1, timeout=None): +def memory_usage(proc=-1, interval=.1, timeout=None, max_usage=False, retval=False): """ Return the memory usage of a process or piece of code @@ -119,13 +126,25 @@ def memory_usage(proc=-1, interval=.1, timeout=None): timeout : float, optional Maximum amount of time (in seconds) to wait before returning. + + max_usage: bool, optional + Only return the maximum memory usage (default False) + + retval: bool, optional + For profiling python functions. Save the return value of the profiled + function. Return value of memory_usage becomes a tuple: + (mem_usage, retval) Returns ------- mem_usage : list of floating-poing values memory usage, in MB. It's length is always < timeout / interval """ - ret = [] + + if not max_usage: + ret = [] + else: + ret = -1 if timeout is not None: max_iter = int(timeout / interval) @@ -158,17 +177,22 @@ def memory_usage(proc=-1, interval=.1, timeout=None): % (n_args, len(args))) child_conn, parent_conn = Pipe() # this will store Timer's results - p = Timer(-1, interval, child_conn) + p = Timer(-1, interval, child_conn,max_usage) p.start() parent_conn.recv() # wait until we start getting memory - f(*args, **kw) + returned = f(*args, **kw) parent_conn.send(0) # finish timing ret = parent_conn.recv() + if retval: + ret = ret,returned p.join(5 * interval) elif isinstance(proc, subprocess.Popen): # external process, launched from Python while True: - ret.append(_get_memory(proc.pid)) + if not max_usage: + ret.append(_get_memory(proc.pid)) + else: + ret = max([ret,_get_memory(proc.pid)]) time.sleep(interval) if timeout is not None: max_iter -= 1 @@ -183,7 +207,10 @@ def memory_usage(proc=-1, interval=.1, timeout=None): counter = 0 while counter < max_iter: counter += 1 - ret.append(_get_memory(proc)) + if not max_usage: + ret.append(_get_memory(proc.pid)) + else: + ret = max([ret,_get_memory(proc.pid)]) time.sleep(interval) return ret From 673f477fda944cb66e9ae557b6a59d61da97aab6 Mon Sep 17 00:00:00 2001 From: cheesinglee Date: Sun, 23 Jun 2013 13:59:11 -0700 Subject: [PATCH 045/415] Minor changes Fixed stupid copy-paste error in external process part of memory_usage() Add ret to docstring of memory_usage() Signed-off-by: cheesinglee --- memory_profiler.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/memory_profiler.py b/memory_profiler.py index 47d5a66..c196fde 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -139,6 +139,8 @@ def memory_usage(proc=-1, interval=.1, timeout=None, max_usage=False, retval=Fal ------- mem_usage : list of floating-poing values memory usage, in MB. It's length is always < timeout / interval + ret : return value of the profiled function + Only returned if retval is set to True """ if not max_usage: @@ -177,7 +179,7 @@ def memory_usage(proc=-1, interval=.1, timeout=None, max_usage=False, retval=Fal % (n_args, len(args))) child_conn, parent_conn = Pipe() # this will store Timer's results - p = Timer(-1, interval, child_conn,max_usage) + p = Timer(os.getpid(), interval, child_conn,max_usage) p.start() parent_conn.recv() # wait until we start getting memory returned = f(*args, **kw) @@ -208,9 +210,9 @@ def memory_usage(proc=-1, interval=.1, timeout=None, max_usage=False, retval=Fal while counter < max_iter: counter += 1 if not max_usage: - ret.append(_get_memory(proc.pid)) + ret.append(_get_memory(proc)) else: - ret = max([ret,_get_memory(proc.pid)]) + ret = max([ret,_get_memory(proc)]) time.sleep(interval) return ret From d5201e1c9e03397ec7dd93322a682e76663ad643 Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Mon, 24 Jun 2013 09:26:41 -0400 Subject: [PATCH 046/415] Units of getrusage() seem to be different in OSX --- memory_profiler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/memory_profiler.py b/memory_profiler.py index c196fde..1e58603 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -38,13 +38,18 @@ except ImportError: pass +# divide resource.getrusage() by rusage_denom to get MB +rusage_denom = 1024. +if sys.platform == 'darwin': + # ... it seems that in OSX the output is different units ... + rusage_denom = rusage_denom * rusage_denom def _get_memory(pid): # .. only for current process and only on unix.. if pid == -1: if has_resource: - mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024. + mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rusage_denom return mem else: pid = os.getpid() From 825f11296efbfed98ba7a812849edb18199ddc7d Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Mon, 24 Jun 2013 09:30:48 -0400 Subject: [PATCH 047/415] test refactoring --- test/test_loop.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_loop.py b/test/test_loop.py index aaaded3..35ef48b 100644 --- a/test/test_loop.py +++ b/test/test_loop.py @@ -2,6 +2,13 @@ @profile def test_1(): + a = {} + for i in range(10000): + a[i] = i + 1 + return + +@profile +def test_2(): a = [1] * (10 ** 6) b = [2] * (2 * 10 ** 7) del b @@ -12,13 +19,6 @@ def test_1(): del b return a -@profile -def test_2(): - a = {} - for i in range(10000): - a[i] = i + 1 - return - if __name__ == '__main__': test_1() test_2() From 13e1192dfcc5d26d0c447a97a62d988d8ba791f6 Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Tue, 16 Jul 2013 14:16:09 +0200 Subject: [PATCH 048/415] Fix for issue #52 module resource seems to give wrong measurements on some cases. --- memory_profiler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/memory_profiler.py b/memory_profiler.py index 1e58603..d3d06ff 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -48,7 +48,9 @@ def _get_memory(pid): # .. only for current process and only on unix.. if pid == -1: - if has_resource: + # .. seems to get wrong measurements on some cases, see .. + # .. https://github.com/fabianp/memory_profiler/issues/52 .. + if False: #has_resource: mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rusage_denom return mem else: From d339c05826646ac3685d4623e87ad0646a9181fc Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Thu, 18 Jul 2013 15:22:17 +0200 Subject: [PATCH 049/415] 0.27 release --- memory_profiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/memory_profiler.py b/memory_profiler.py index d3d06ff..bcee515 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -3,7 +3,7 @@ # .. we'll use this to pass it to the child script .. _clean_globals = globals().copy() -__version__ = '0.26-git' +__version__ = '0.27' _CMD_USAGE = "python -m memory_profiler script_file.py" From 3b59db18b96651e4423e58ae52cf6c2ca5dd8216 Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Tue, 23 Jul 2013 13:56:37 +0200 Subject: [PATCH 050/415] Add refresh interval (-i)) option to %memit --- README.rst | 4 ++-- memory_profiler.py | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index 44393b3..2855da7 100644 --- a/README.rst +++ b/README.rst @@ -29,7 +29,7 @@ The line-by-line profiler is used much in the same way of the `line_profiler `_: first decorate the function you would like to profile with ``@profile`` and then run the script with a special script (in this case with specific -arguments to the Python interpreter). +arguments to the Python interpreter). In the following example, we create a simple function ``my_func`` that allocates lists ``a``, ``b`` and then deletes ``b``:: @@ -163,7 +163,7 @@ After installing the module, if you use IPython, you can use the `%mprun` and `%memit` magics. For IPython 0.11+, you can use the module directly as an extension, with -``%load_ext memory_profiler``. +``%load_ext memory_profiler`` To activate it whenever you start IPython, edit the configuration file for your IPython profile, ~/.ipython/profile_default/ipython_config.py, to register the diff --git a/memory_profiler.py b/memory_profiler.py index bcee515..00c03c4 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -103,7 +103,7 @@ def run(self): if not self.max_usage: timings = [m] else: - timings = m + timings = m self.pipe.send(0) # we're ready while not self.pipe.poll(self.interval): m = _get_memory(self.monitor_pid) @@ -133,10 +133,10 @@ def memory_usage(proc=-1, interval=.1, timeout=None, max_usage=False, retval=Fal timeout : float, optional Maximum amount of time (in seconds) to wait before returning. - + max_usage: bool, optional Only return the maximum memory usage (default False) - + retval: bool, optional For profiling python functions. Save the return value of the profiled function. Return value of memory_usage becomes a tuple: @@ -149,7 +149,7 @@ def memory_usage(proc=-1, interval=.1, timeout=None, max_usage=False, retval=Fal ret : return value of the profiled function Only returned if retval is set to True """ - + if not max_usage: ret = [] else: @@ -567,7 +567,7 @@ def magic_memit(self, line=''): """Measure memory usage of a Python statement Usage, in line mode: - %memit [-rt] statement + %memit [-rti] statement Options: -r: repeat the loop iteration times and take the best result. @@ -575,6 +575,9 @@ def magic_memit(self, line=''): -t: timeout after seconds. Default: None + -i: Get time information at an interval of I times per second. + Defaults to 0.1 so that there is ten measurements per second. + Examples -------- :: @@ -591,19 +594,21 @@ def magic_memit(self, line=''): maximum of 10: 0.101562 MB per loop """ - opts, stmt = self.parse_options(line, 'r:t', posix=False, strict=False) + opts, stmt = self.parse_options(line, 'r:t:i:', posix=False, strict=False) repeat = int(getattr(opts, 'r', 1)) if repeat < 1: repeat == 1 timeout = int(getattr(opts, 't', 0)) if timeout <= 0: timeout = None + interval = float(getattr(opts, 'i', 0.1)) mem_usage = [] counter = 0 while counter < repeat: counter += 1 - tmp = memory_usage((_func_exec, (stmt, self.shell.user_ns)), timeout=timeout) + tmp = memory_usage((_func_exec, (stmt, self.shell.user_ns)), + timeout=timeout, interval=interval) mem_usage.extend(tmp) if mem_usage: From 627403aaf0b7bbad7fc608cc27adef98c44f3d44 Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Tue, 23 Jul 2013 14:14:38 +0200 Subject: [PATCH 051/415] refactoring --- maction | 1 + mprofile => mprof | 5 ----- setup.py | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) rename mprofile => mprof (92%) diff --git a/maction b/maction index a1f0dae..f5aa33c 100755 --- a/maction +++ b/maction @@ -108,6 +108,7 @@ def list_action(): def rm_action(): + """TODO: merge with clean_action (@pgervais)""" parser = OptionParser(version=mp.__version__) parser.disable_interspersed_args() parser.add_option("--dry-run", dest="dry_run", default=False, diff --git a/mprofile b/mprof similarity index 92% rename from mprofile rename to mprof index ebb569c..dea0447 100755 --- a/mprofile +++ b/mprof @@ -27,10 +27,6 @@ if len(args) == 0: print("A program to run must be provided. Use -h for help") sys.exit(1) -## if len(sys.argv) < 2: -## print("""Memory usage monitoring -## Usage: %s ... - ## Output results in a file called "mprofile_.dat" (where ## is the date-time of the program start) in the current ## directory. This file contains the process memory consumption, in Mb (one @@ -44,7 +40,6 @@ mprofile_output = "mprofile_%s.dat" % suffix if options.python: print("running as a Python program...") -# timestamp_output = "mprofile_%s_ts.dat" % suffix if not args[0].startswith("python"): args.insert(0, "python") args[1:1] = ("-m", "memory_profiler", "--timestamp", diff --git a/setup.py b/setup.py index 40c38a2..749bc02 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ author_email='fabian@fseoane.net', url='http://pypi.python.org/pypi/memory_profiler', py_modules=['memory_profiler'], - scripts=["mprofile", "mplot"], + scripts=['mprof'], classifiers=[_f for _f in CLASSIFIERS.split('\n') if _f], license='Simplified BSD' From e8df9caf862b59d7b704b6318c068831275e632f Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Tue, 23 Jul 2013 15:24:12 +0200 Subject: [PATCH 052/415] refactoring --- README.rst | 11 ++- maction | 170 ---------------------------------- mprof | 264 +++++++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 225 insertions(+), 220 deletions(-) delete mode 100755 maction diff --git a/README.rst b/README.rst index 2855da7..40250b1 100644 --- a/README.rst +++ b/README.rst @@ -74,7 +74,6 @@ the code that has been profiled. Decorator ========= - A function decorator is also available. Use as follows:: from memory_profiler import profile @@ -86,6 +85,16 @@ A function decorator is also available. Use as follows:: del b return a +In this case the script can be run without specifying ``-m +memory_profiler`` in the command line. + +Executing external scripts +========================== +Sometimes it is useful to have full memory usage reports as a function of +time (not line-by-line) of external processess (be it Python scripts or not). +In this case the executable ``mprof`` might be useful. To use it + + Setting debugger breakpoints ============================= It is possible to set breakpoints depending on the amount of memory used. diff --git a/maction b/maction deleted file mode 100755 index f5aa33c..0000000 --- a/maction +++ /dev/null @@ -1,170 +0,0 @@ -#! /usr/bin/env python - -import glob -import os -import os.path as osp -import sys -import re -import copy - -from optparse import OptionParser - -import memory_profiler as mp - - -def print_usage(): - print("Usage: %s " - % osp.basename(sys.argv[0])) - -def get_action(): - """Pop first argument, check it is a valid action.""" - all_actions = ("rm", "clean", "list") - if len(sys.argv) <= 1: - print_usage() - sys.exit(1) - if not sys.argv[1] in all_actions: - print("Valid actions are: " + " ".join(all_actions)) - sys.exit(1) - - return sys.argv.pop(1) - - -def get_profile_filenames(args): - """Return list of profile filenames. - - Parameters - ========== - args (list) - list of filename or integer. An integer is the index of the - profile in the list of existing profiles. 0 is the oldest, - -1 in the more recent. - Non-existing files cause a ValueError exception to be thrown. - - Returns - ======= - filenames (list) - list of existing memory profile filenames. It is guaranteed - that an given file name will not appear twice in this list. - """ - profiles = glob.glob("mprofile_??????????????.dat") - profiles.sort() - - if args is "all": - filenames = copy.copy(profiles) - else: - filenames = [] - for arg in args: - if arg == "--": # workaround - continue - try: - index = int(arg) - except ValueError: - index = None - if index is not None: - try: - filename = profiles[index] - except IndexError: - raise ValueError("Invalid index (non-existing file): %s" % arg) - - if filename not in filenames: - filenames.append(filename) - else: - if osp.isfile(arg): - if arg not in filenames: - filenames.append(arg) - elif osp.isdir(arg): - raise ValueError("Path %s is a directory" % arg) - else: - raise ValueError("File %s not found" % arg) - - # Add timestamp files, if any - for filename in reversed(filenames): - parts = osp.splitext(filename) - timestamp_file = parts[0] + "_ts" + parts[1] - if osp.isfile(timestamp_file) and timestamp_file not in filenames: - filenames.append(timestamp_file) - - return filenames - - -def list_action(): - """Display existing profiles, with indices.""" - parser = OptionParser(version=mp.__version__) - parser.disable_interspersed_args() - - (options, args) = parser.parse_args() - - if len(args) > 0: - print("This command takes no argument.") - sys.exit(1) - - filenames = get_profile_filenames("all") - for n, filename in enumerate(filenames): - ts = osp.splitext(filename)[0].split('_')[-1] - print("{index} {filename} {hour}:{min}:{sec} {day}/{month}/{year}" - .format(index=n, filename=filename, - year=ts[:4], month=ts[4:6], day=ts[6:8], - hour=ts[8:10], min=ts[10:12], sec=ts[12:14])) - - -def rm_action(): - """TODO: merge with clean_action (@pgervais)""" - parser = OptionParser(version=mp.__version__) - parser.disable_interspersed_args() - parser.add_option("--dry-run", dest="dry_run", default=False, - action="store_true", - help="""Show what will be done, without actually doing it.""") - - (options, args) = parser.parse_args() - - if len(args) == 0: - print("A profile to remove must be provided (number or filename)") - sys.exit(1) - - filenames = get_profile_filenames(args) - if options.dry_run: - print("Files to be removed: ") - for filename in filenames: - print(filename) - else: - for filename in filenames: - os.remove(filename) - - -def clean_action(): - """Remove every profile file in current directory.""" - parser = OptionParser(version=mp.__version__) - parser.disable_interspersed_args() - parser.add_option("--dry-run", dest="dry_run", default=False, - action="store_true", - help="""Show what will be done, without actually doing it.""") - - (options, args) = parser.parse_args() - - if len(args) > 0: - print("This command takes no argument.") - sys.exit(1) - - filenames = get_profile_filenames("all") - if options.dry_run: - print("Files to be removed: ") - for filename in filenames: - print(filename) - else: - for filename in filenames: - os.remove(filename) - - -if __name__ == "__main__": - # Workaround for optparse limitation: insert -- before first negative number found. - negint = re.compile("-[0-9]+") - for n, arg in enumerate(sys.argv): - if negint.match(arg): - sys.argv.insert(n, "--") - break - actions = {"rm": rm_action, - "clean": clean_action, - "list": list_action} - actions[get_action()]() - - diff --git a/mprof b/mprof index dea0447..74670c2 100755 --- a/mprof +++ b/mprof @@ -1,55 +1,221 @@ #! /usr/bin/env python -import subprocess -import memory_profiler as mp + +import glob +import os import os.path as osp -import time import sys +import re +import copy from optparse import OptionParser -parser = OptionParser(version=mp.__version__) -parser.disable_interspersed_args() -parser.add_option("--python", dest="python", default=False, - action="store_true", - help="""Activates extra features when the profiled executable is - a Python program (currently: function timestamping.)""") -parser.add_option("--interval", "-T", dest="interval", default="0.5", - type="float", action="store", - help="Sampling period (in seconds)") -parser.add_option("--include-children", "-C", dest="include_children", default=False, - action="store_true", - help="""Monitors forked processes as well (sum up all process memory)""") - -(options, args) = parser.parse_args() -print("{1}: Sampling memory every {0.interval}s".format(options, osp.basename(sys.argv[0]))) - -if len(args) == 0: - print("A program to run must be provided. Use -h for help") - sys.exit(1) - -## Output results in a file called "mprofile_.dat" (where -## is the date-time of the program start) in the current -## directory. This file contains the process memory consumption, in Mb (one -## value per line). Memory is sampled twice each second.""" -## % osp.basename(sys.argv[0]) -## ) -## sys.exit(1) - -suffix = time.strftime("%Y%m%d%H%M%S", time.localtime()) -mprofile_output = "mprofile_%s.dat" % suffix - -if options.python: - print("running as a Python program...") - if not args[0].startswith("python"): - args.insert(0, "python") - args[1:1] = ("-m", "memory_profiler", "--timestamp", - "-o", mprofile_output) - p = subprocess.Popen(args) -else: - p = subprocess.Popen(args) - -mu = mp.memory_usage(proc=p, interval=options.interval, timestamps=True, - include_children=options.include_children) -with open(mprofile_output, "a") as f: - for m, t in mu: - f.write("MEM {0:.6f} {1:.4f}".format(m, t) + "\n") +import memory_profiler as mp + + +def print_usage(): + print("Usage: %s " + % osp.basename(sys.argv[0])) + +def get_action(): + """Pop first argument, check it is a valid action.""" + all_actions = ("run", "rm", "clean", "list") + if len(sys.argv) <= 1: + print_usage() + sys.exit(1) + if not sys.argv[1] in all_actions: + print("Valid actions are: " + " ".join(all_actions)) + sys.exit(1) + + return sys.argv.pop(1) + + +def get_profile_filenames(args): + """Return list of profile filenames. + + Parameters + ========== + args (list) + list of filename or integer. An integer is the index of the + profile in the list of existing profiles. 0 is the oldest, + -1 in the more recent. + Non-existing files cause a ValueError exception to be thrown. + + Returns + ======= + filenames (list) + list of existing memory profile filenames. It is guaranteed + that an given file name will not appear twice in this list. + """ + profiles = glob.glob("mprofile_??????????????.dat") + profiles.sort() + + if args is "all": + filenames = copy.copy(profiles) + else: + filenames = [] + for arg in args: + if arg == "--": # workaround + continue + try: + index = int(arg) + except ValueError: + index = None + if index is not None: + try: + filename = profiles[index] + except IndexError: + raise ValueError("Invalid index (non-existing file): %s" % arg) + + if filename not in filenames: + filenames.append(filename) + else: + if osp.isfile(arg): + if arg not in filenames: + filenames.append(arg) + elif osp.isdir(arg): + raise ValueError("Path %s is a directory" % arg) + else: + raise ValueError("File %s not found" % arg) + + # Add timestamp files, if any + for filename in reversed(filenames): + parts = osp.splitext(filename) + timestamp_file = parts[0] + "_ts" + parts[1] + if osp.isfile(timestamp_file) and timestamp_file not in filenames: + filenames.append(timestamp_file) + + return filenames + + +def list_action(): + """Display existing profiles, with indices.""" + parser = OptionParser(version=mp.__version__) + parser.disable_interspersed_args() + + (options, args) = parser.parse_args() + + if len(args) > 0: + print("This command takes no argument.") + sys.exit(1) + + filenames = get_profile_filenames("all") + for n, filename in enumerate(filenames): + ts = osp.splitext(filename)[0].split('_')[-1] + print("{index} {filename} {hour}:{min}:{sec} {day}/{month}/{year}" + .format(index=n, filename=filename, + year=ts[:4], month=ts[4:6], day=ts[6:8], + hour=ts[8:10], min=ts[10:12], sec=ts[12:14])) + + +def rm_action(): + """TODO: merge with clean_action (@pgervais)""" + parser = OptionParser(version=mp.__version__) + parser.disable_interspersed_args() + parser.add_option("--dry-run", dest="dry_run", default=False, + action="store_true", + help="""Show what will be done, without actually doing it.""") + + (options, args) = parser.parse_args() + + if len(args) == 0: + print("A profile to remove must be provided (number or filename)") + sys.exit(1) + + filenames = get_profile_filenames(args) + if options.dry_run: + print("Files to be removed: ") + for filename in filenames: + print(filename) + else: + for filename in filenames: + os.remove(filename) + + +def clean_action(): + """Remove every profile file in current directory.""" + parser = OptionParser(version=mp.__version__) + parser.disable_interspersed_args() + parser.add_option("--dry-run", dest="dry_run", default=False, + action="store_true", + help="""Show what will be done, without actually doing it.""") + + (options, args) = parser.parse_args() + + if len(args) > 0: + print("This command takes no argument.") + sys.exit(1) + + filenames = get_profile_filenames("all") + if options.dry_run: + print("Files to be removed: ") + for filename in filenames: + print(filename) + else: + for filename in filenames: + os.remove(filename) + + + +def run_action(): + import time, subprocess + parser = OptionParser(version=mp.__version__) + parser.disable_interspersed_args() + parser.add_option("--python", dest="python", default=False, + action="store_true", + help="""Activates extra features when the profiled executable is + a Python program (currently: function timestamping.)""") + parser.add_option("--interval", "-T", dest="interval", default="0.5", + type="float", action="store", + help="Sampling period (in seconds)") + parser.add_option("--include-children", "-C", dest="include_children", default=False, + action="store_true", + help="""Monitors forked processes as well (sum up all process memory)""") + + (options, args) = parser.parse_args() + print("{1}: Sampling memory every {0.interval}s".format(options, osp.basename(sys.argv[0]))) + + if len(args) == 0: + print("A program to run must be provided. Use -h for help") + sys.exit(1) + + ## Output results in a file called "mprofile_.dat" (where + ## is the date-time of the program start) in the current + ## directory. This file contains the process memory consumption, in Mb (one + ## value per line). Memory is sampled twice each second.""" + ## % osp.basename(sys.argv[0]) + ## ) + ## sys.exit(1) + + suffix = time.strftime("%Y%m%d%H%M%S", time.localtime()) + mprofile_output = "mprofile_%s.dat" % suffix + + if options.python: + print("running as a Python program...") + if not args[0].startswith("python"): + args.insert(0, "python") + args[1:1] = ("-m", "memory_profiler", "--timestamp", + "-o", mprofile_output) + p = subprocess.Popen(args) + else: + p = subprocess.Popen(args) + + mu = mp.memory_usage(proc=p, interval=options.interval, timestamps=True, + include_children=options.include_children) + with open(mprofile_output, "a") as f: + for m, t in mu: + f.write("MEM {0:.6f} {1:.4f}".format(m, t) + "\n") + +if __name__ == "__main__": + # Workaround for optparse limitation: insert -- before first negative number found. + negint = re.compile("-[0-9]+") + for n, arg in enumerate(sys.argv): + if negint.match(arg): + sys.argv.insert(n, "--") + break + actions = {"rm": rm_action, + "clean": clean_action, + "list": list_action, + "run" : run_action} + actions[get_action()]() + + From 9bfd761d3ea9929716d94fa055edcad8d196b0c0 Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Tue, 23 Jul 2013 15:34:07 +0200 Subject: [PATCH 053/415] refactoring --- README.rst | 5 +- mplot | 168 ------------------------------------------------- mprof | 182 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 181 insertions(+), 174 deletions(-) delete mode 100755 mplot diff --git a/README.rst b/README.rst index 40250b1..273f9e3 100644 --- a/README.rst +++ b/README.rst @@ -92,8 +92,11 @@ Executing external scripts ========================== Sometimes it is useful to have full memory usage reports as a function of time (not line-by-line) of external processess (be it Python scripts or not). -In this case the executable ``mprof`` might be useful. To use it +In this case the executable ``mprof`` might be useful. To use it like:: + ./mprof run --python name_of_script.py + +TODO: make it work without the --python option. Setting debugger breakpoints ============================= diff --git a/mplot b/mplot deleted file mode 100755 index 547bdbc..0000000 --- a/mplot +++ /dev/null @@ -1,168 +0,0 @@ -#! /usr/bin/env python -import sys - -try: - import pylab as pl -except ImportError: - print("matplotlib is needed for plotting.") - sys.exit(1) - -import numpy as np -import math -import glob -import time -import os -import os.path as osp - - -def add_brackets(xloc, yloc, xshift=0, color="r", label=None): - """Add two brackets on the memory line plot. - - This function uses the current figure. - - Parameters - ========== - xloc: tuple with 2 values - brackets location (on horizontal axis). - yloc: tuple with 2 values - brackets location (on vertical axis) - xshift: float - value to subtract to xloc. - """ - - height_ratio = 20. - vsize = (pl.ylim()[1] - pl.ylim()[0]) / height_ratio - hsize = (pl.xlim()[1] - pl.xlim()[0]) / (3.*height_ratio) - - bracket_x = pl.asarray([hsize, 0, 0, hsize]) - bracket_y = pl.asarray([vsize, vsize, -vsize, -vsize]) - - # Matplotlib workaround: labels starting with _ aren't displayed - if label[0] == '_': - label = ' ' + label - pl.plot(bracket_x + xloc[0] - xshift, bracket_y + yloc[0], - "-" + color, linewidth=2, label=label) - pl.plot(-bracket_x + xloc[1] - xshift, bracket_y + yloc[1], - "-" + color, linewidth=2 ) - - # TODO: use matplotlib.patches.Polygon to draw a colored background for - # each function. - - # with maplotlib 1.2, use matplotlib.path.Path to create proper markers - # see http://matplotlib.org/examples/pylab_examples/marker_path.html - # This works with matplotlib 0.99.1 - ## pl.plot(xloc[0], yloc[0], "<"+color, markersize=7, label=label) - ## pl.plot(xloc[1], yloc[1], ">"+color, markersize=7) - - -def read_mprofile_file(filename): - ret = {} - mdata = [] - f = open(filename, "r") - for l in f: - fields = l.split() - if fields[0] == "MEM": - # mem, timestamp - mdata.append((fields[1], fields[2])) - - elif fields[0] == "FUNC": - f_name, mem_start, start, mem_end, end = fields[1:] - ts = ret.get(f_name, []) - ts.append([float(start), float(end), float(mem_start), float(mem_end)]) - ret[f_name] = ts - - else: - pass - f.close() - - mdata = np.asarray(mdata, - dtype=[("mem", np.float), ("timestamp", np.float)]) - return mdata, ret - - - -def plot_file(filename, index=0, timestamps=True): - mdata, ts = read_mprofile_file(filename) - - global_start = float(mdata["timestamp"][0]) - - mem = mdata["mem"] - max_mem = mem.max() - max_mem_ind = mem.argmax() - - t = mdata["timestamp"] - global_start - - all_colors=("c", "y", "g", "r", "b") - mem_line_colors=('k', "b", "r") - mem_line_label = time.strftime("%d / %m / %Y - start at %H:%M:%S", - time.localtime(global_start)) \ - + ".{0:03d}".format(int(round(math.modf(global_start)[0]*1000))) - - pl.plot(t, mem, "+-" + mem_line_colors[index % len(mem_line_colors)], - label=mem_line_label) - - bottom, top = pl.ylim() - bottom += 0.001 - top -= 0.001 - - # plot timestamps, if any - if len(ts) > 0 and timestamps: - func_num = 0 - for f, exec_ts in ts.iteritems(): - for execution in exec_ts: - add_brackets(execution[:2], execution[2:], xshift=global_start, - color= all_colors[func_num % len(all_colors)], - label=f.split(".")[-1] - + " %.3fs" % (execution[1] - execution[0])) - func_num += 1 - - if timestamps: - pl.hlines(max_mem, - pl.xlim()[0] + 0.001, pl.xlim()[1] - 0.001, - colors="r", linestyles="--") - pl.vlines(t[max_mem_ind], bottom, top, - colors="r", linestyles="--") - - -if __name__ == "__main__": - profiles = glob.glob("mprofile_??????????????.dat") - profiles.sort() - - if len(sys.argv) == 1: - if len(profiles) == 0: - print("No input file found. \nThis program looks for " - "mprofile_*.dat files, generated by the mprofile command.") - sys.exit(-1) - filenames = [profiles[-1]] - else: - filenames = [] - for arg in sys.argv[1:]: - if osp.exists(arg): - if not arg in filenames: - filenames.append(arg) - else: - try: - n = int(arg) - except ValueError: - print("Input file not found: " + arg) - if not profiles[n] in filenames: - filenames.append(profiles[n]) - - pl.figure(figsize=(14, 6), dpi=90) - if len(filenames) > 1: - timestamps = False - else: - timestamps = True - for n, filename in enumerate(filenames): - plot_file(filename, index=n, timestamps=timestamps) - pl.xlabel("time [s]") - pl.ylabel("memory used [MB]") - - ax = pl.gca() - box = ax.get_position() - ax.set_position([0.07, 0.1, - 0.55, 0.8]) - ax.legend(loc="upper left", bbox_to_anchor=(1.05, 1.)) - pl.grid() - pl.show() - diff --git a/mprof b/mprof index 74670c2..f50124e 100755 --- a/mprof +++ b/mprof @@ -6,6 +6,8 @@ import os.path as osp import sys import re import copy +import time +import math from optparse import OptionParser @@ -18,7 +20,7 @@ def print_usage(): def get_action(): """Pop first argument, check it is a valid action.""" - all_actions = ("run", "rm", "clean", "list") + all_actions = ("run", "rm", "clean", "list", "plot") if len(sys.argv) <= 1: print_usage() sys.exit(1) @@ -164,7 +166,7 @@ def run_action(): action="store_true", help="""Activates extra features when the profiled executable is a Python program (currently: function timestamping.)""") - parser.add_option("--interval", "-T", dest="interval", default="0.5", + parser.add_option("--interval", "-T", dest="interval", default="0.1", type="float", action="store", help="Sampling period (in seconds)") parser.add_option("--include-children", "-C", dest="include_children", default=False, @@ -205,6 +207,177 @@ def run_action(): for m, t in mu: f.write("MEM {0:.6f} {1:.4f}".format(m, t) + "\n") + + +def add_brackets(xloc, yloc, xshift=0, color="r", label=None): + """Add two brackets on the memory line plot. + + This function uses the current figure. + + Parameters + ========== + xloc: tuple with 2 values + brackets location (on horizontal axis). + yloc: tuple with 2 values + brackets location (on vertical axis) + xshift: float + value to subtract to xloc. + """ + try: + import pylab as pl + except ImportError: + print("matplotlib is needed for plotting.") + sys.exit(1) + height_ratio = 20. + vsize = (pl.ylim()[1] - pl.ylim()[0]) / height_ratio + hsize = (pl.xlim()[1] - pl.xlim()[0]) / (3.*height_ratio) + + bracket_x = pl.asarray([hsize, 0, 0, hsize]) + bracket_y = pl.asarray([vsize, vsize, -vsize, -vsize]) + + # Matplotlib workaround: labels starting with _ aren't displayed + if label[0] == '_': + label = ' ' + label + pl.plot(bracket_x + xloc[0] - xshift, bracket_y + yloc[0], + "-" + color, linewidth=2, label=label) + pl.plot(-bracket_x + xloc[1] - xshift, bracket_y + yloc[1], + "-" + color, linewidth=2 ) + + # TODO: use matplotlib.patches.Polygon to draw a colored background for + # each function. + + # with maplotlib 1.2, use matplotlib.path.Path to create proper markers + # see http://matplotlib.org/examples/pylab_examples/marker_path.html + # This works with matplotlib 0.99.1 + ## pl.plot(xloc[0], yloc[0], "<"+color, markersize=7, label=label) + ## pl.plot(xloc[1], yloc[1], ">"+color, markersize=7) + + +def read_mprofile_file(filename): + # TODO: would be nice to do without numpy + + import numpy as np + ret = {} + mdata = [] + f = open(filename, "r") + for l in f: + fields = l.split() + if fields[0] == "MEM": + # mem, timestamp + mdata.append((fields[1], fields[2])) + + elif fields[0] == "FUNC": + f_name, mem_start, start, mem_end, end = fields[1:] + ts = ret.get(f_name, []) + ts.append([float(start), float(end), float(mem_start), float(mem_end)]) + ret[f_name] = ts + + else: + pass + f.close() + + mdata = np.asarray(mdata, + dtype=[("mem", np.float), ("timestamp", np.float)]) + return mdata, ret + + + +def plot_file(filename, index=0, timestamps=True): + try: + import pylab as pl + except ImportError: + print("matplotlib is needed for plotting.") + sys.exit(1) + + mdata, ts = read_mprofile_file(filename) + + global_start = float(mdata["timestamp"][0]) + + mem = mdata["mem"] + max_mem = mem.max() + max_mem_ind = mem.argmax() + + t = mdata["timestamp"] - global_start + + all_colors=("c", "y", "g", "r", "b") + mem_line_colors=('k', "b", "r") + mem_line_label = time.strftime("%d / %m / %Y - start at %H:%M:%S", + time.localtime(global_start)) \ + + ".{0:03d}".format(int(round(math.modf(global_start)[0]*1000))) + + pl.plot(t, mem, "+-" + mem_line_colors[index % len(mem_line_colors)], + label=mem_line_label) + + bottom, top = pl.ylim() + bottom += 0.001 + top -= 0.001 + + # plot timestamps, if any + if len(ts) > 0 and timestamps: + func_num = 0 + for f, exec_ts in ts.iteritems(): + for execution in exec_ts: + add_brackets(execution[:2], execution[2:], xshift=global_start, + color= all_colors[func_num % len(all_colors)], + label=f.split(".")[-1] + + " %.3fs" % (execution[1] - execution[0])) + func_num += 1 + + if timestamps: + pl.hlines(max_mem, + pl.xlim()[0] + 0.001, pl.xlim()[1] - 0.001, + colors="r", linestyles="--") + pl.vlines(t[max_mem_ind], bottom, top, + colors="r", linestyles="--") + +def plot_action(): + try: + import pylab as pl + except ImportError: + print("matplotlib is needed for plotting.") + sys.exit(1) + + profiles = glob.glob("mprofile_??????????????.dat") + profiles.sort() + + if len(sys.argv) == 1: + if len(profiles) == 0: + print("No input file found. \nThis program looks for " + "mprofile_*.dat files, generated by the mprofile command.") + sys.exit(-1) + filenames = [profiles[-1]] + else: + filenames = [] + for arg in sys.argv[1:]: + if osp.exists(arg): + if not arg in filenames: + filenames.append(arg) + else: + try: + n = int(arg) + except ValueError: + print("Input file not found: " + arg) + if not profiles[n] in filenames: + filenames.append(profiles[n]) + + pl.figure(figsize=(14, 6), dpi=90) + if len(filenames) > 1: + timestamps = False + else: + timestamps = True + for n, filename in enumerate(filenames): + plot_file(filename, index=n, timestamps=timestamps) + pl.xlabel("time [s]") + pl.ylabel("memory used [MB]") + + ax = pl.gca() + box = ax.get_position() + ax.set_position([0.07, 0.1, + 0.55, 0.8]) + ax.legend(loc="upper left", bbox_to_anchor=(1.05, 1.)) + pl.grid() + pl.show() + if __name__ == "__main__": # Workaround for optparse limitation: insert -- before first negative number found. negint = re.compile("-[0-9]+") @@ -215,7 +388,6 @@ if __name__ == "__main__": actions = {"rm": rm_action, "clean": clean_action, "list": list_action, - "run" : run_action} + "run": run_action, + "plot": plot_action} actions[get_action()]() - - From 9e79e9293fa7eab9223f8535e040bb02206d3810 Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Tue, 23 Jul 2013 16:38:05 +0200 Subject: [PATCH 054/415] BUG: make it work on OSX 10.8 --- memory_profiler.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/memory_profiler.py b/memory_profiler.py index a7315c3..8b12533 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -67,12 +67,13 @@ def _get_memory(pid, timestamps=False, include_children=False): if include_children: for p in process.get_children(recursive=True): mem += p.get_memory_info()[0] / (_TWO_20) + if timestamps: + return (mem, time.time()) + else: + return mem except psutil.AccessDenied: - mem = -1 - if timestamps: - return (mem, time.time()) - else: - return mem + pass + # continue and try to get this from ps # .. scary stuff .. if os.name == 'posix': From 6fed905b88f819d7b197a0cfff00b494619f62b7 Mon Sep 17 00:00:00 2001 From: dengemann Date: Wed, 24 Jul 2013 10:12:28 +0200 Subject: [PATCH 055/415] STY: pep8 and related fixes --- memory_profiler.py | 81 +++++++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 36 deletions(-) diff --git a/memory_profiler.py b/memory_profiler.py index 8b12533..259d94a 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -7,7 +7,10 @@ _CMD_USAGE = "python -m memory_profiler script_file.py" -import time, sys, os, pdb +import time +import sys +import os +import pdb import warnings import linecache import inspect @@ -44,14 +47,16 @@ # ... it seems that in OSX the output is different units ... rusage_denom = rusage_denom * rusage_denom + def _get_memory(pid, timestamps=False, include_children=False): # .. only for current process and only on unix.. if pid == -1: # .. seems to get wrong measurements on some cases, see .. # .. https://github.com/fabianp/memory_profiler/issues/52 .. - if False: #has_resource: - mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rusage_denom + if False: # has_resource: + mem = (resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / + rusage_denom) if timestamps: return (mem, time.time()) else: @@ -77,25 +82,25 @@ def _get_memory(pid, timestamps=False, include_children=False): # .. scary stuff .. if os.name == 'posix': - warnings.warn("psutil module not found. memory_profiler will be slow") - # .. - # .. memory usage in MB .. - # .. this should work on both Mac and Linux .. - # .. subprocess.check_output appeared in 2.7, using Popen .. - # .. for backwards compatibility .. - out = subprocess.Popen(['ps', 'v', '-p', str(pid)], - stdout=subprocess.PIPE).communicate()[0].split(b'\n') - try: - vsz_index = out[0].split().index(b'RSS') - mem = float(out[1].split()[vsz_index]) / 1024 - if timestamps: - return(mem, time.time()) - else: - return mem - except: - if timestamps: - return (-1, time.time()) - else: + warnings.warn("psutil module not found. memory_profiler will be slow") + # .. + # .. memory usage in MB .. + # .. this should work on both Mac and Linux .. + # .. subprocess.check_output appeared in 2.7, using Popen .. + # .. for backwards compatibility .. + out = subprocess.Popen(['ps', 'v', '-p', str(pid)], + stdout=subprocess.PIPE).communicate()[0].split(b'\n') + try: + vsz_index = out[0].split().index(b'RSS') + mem = float(out[1].split()[vsz_index]) / 1024 + if timestamps: + return(mem, time.time()) + else: + return mem + except: + if timestamps: + return (-1, time.time()) + else: return -1 else: raise NotImplementedError('The psutil module is required for non-unix ' @@ -106,8 +111,8 @@ class Timer(Process): """ Fetch memory consumption from over a time interval """ - - def __init__(self, monitor_pid, interval, pipe, max_usage=False, *args, **kw): + def __init__(self, monitor_pid, interval, pipe, max_usage=False, + *args, **kw): self.monitor_pid = monitor_pid self.interval = interval self.pipe = pipe @@ -213,9 +218,8 @@ def memory_usage(proc=-1, interval=.1, timeout=None, timestamps=False, if aspec.defaults is not None: n_args -= len(aspec.defaults) if n_args != len(args): - raise ValueError( - 'Function expects %s value(s) but %s where given' - % (n_args, len(args))) + raise ValueError('Function expects %s value(s) but %s where given' + % (n_args, len(args))) child_conn, parent_conn = Pipe() # this will store Timer's results p = Timer(os.getpid(), interval, child_conn, timestamps=timestamps, @@ -266,6 +270,7 @@ def memory_usage(proc=-1, interval=.1, timeout=None, timestamps=False, # .. # .. utility functions for line-by-line .. + def _find_script(script_name): """ Find the script. @@ -275,7 +280,7 @@ def _find_script(script_name): return script_name path = os.getenv('PATH', os.defpath).split(os.pathsep) for folder in path: - if folder == '': + if not folder: continue fn = os.path.join(folder, script_name) if os.path.isfile(fn): @@ -387,7 +392,7 @@ def add_function(self, func): except AttributeError: import warnings warnings.warn("Could not extract a code object for the object %r" - % (func,)) + % func) return if code not in self.code_map: self.code_map[code] = {} @@ -455,8 +460,8 @@ def trace_max_mem(self, frame, event, arg): if event in ('line', 'return') and frame.f_code in self.code_map: c = _get_memory(-1) if c >= self.max_mem: - t = 'Current memory {0:.2f} MB exceeded the maximum '.format(c) + \ - 'of {0:.2f} MB\n'.format(self.max_mem) + t = ('Current memory {0:.2f} MB exceeded the maximum' + ''.format(c) + 'of {0:.2f} MB\n'.format(self.max_mem)) sys.stdout.write(t) sys.stdout.write('Stepping into the debugger \n') frame.f_lineno -= 2 @@ -503,7 +508,8 @@ def show_results(prof, stream=None, precision=3): stream.write('Filename: ' + filename + '\n\n') if not os.path.exists(filename): stream.write('ERROR: Could not find file ' + filename + '\n') - if filename.startswith("ipython-input") or filename.startswith(" Date: Wed, 24 Jul 2013 16:12:26 +0200 Subject: [PATCH 056/415] Fixed parameters handling on 'mprof plot' --- mprof | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/mprof b/mprof index f50124e..bdf404a 100755 --- a/mprof +++ b/mprof @@ -169,12 +169,13 @@ def run_action(): parser.add_option("--interval", "-T", dest="interval", default="0.1", type="float", action="store", help="Sampling period (in seconds)") - parser.add_option("--include-children", "-C", dest="include_children", default=False, - action="store_true", + parser.add_option("--include-children", "-C", dest="include_children", + default=False, action="store_true", help="""Monitors forked processes as well (sum up all process memory)""") (options, args) = parser.parse_args() - print("{1}: Sampling memory every {0.interval}s".format(options, osp.basename(sys.argv[0]))) + print("{1}: Sampling memory every {0.interval}s".format( + options, osp.basename(sys.argv[0]))) if len(args) == 0: print("A program to run must be provided. Use -h for help") @@ -184,9 +185,6 @@ def run_action(): ## is the date-time of the program start) in the current ## directory. This file contains the process memory consumption, in Mb (one ## value per line). Memory is sampled twice each second.""" - ## % osp.basename(sys.argv[0]) - ## ) - ## sys.exit(1) suffix = time.strftime("%Y%m%d%H%M%S", time.localtime()) mprofile_output = "mprofile_%s.dat" % suffix @@ -348,7 +346,7 @@ def plot_action(): filenames = [profiles[-1]] else: filenames = [] - for arg in sys.argv[1:]: + for arg in sys.argv[2:]: if osp.exists(arg): if not arg in filenames: filenames.append(arg) From 55d975445327a742f953133bd4721ce176846cf1 Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Wed, 24 Jul 2013 17:32:27 +0200 Subject: [PATCH 057/415] remove code that uses the resource module (unused code) --- memory_profiler.py | 29 +++-------------------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/memory_profiler.py b/memory_profiler.py index 259d94a..bcf823c 100644 --- a/memory_profiler.py +++ b/memory_profiler.py @@ -26,7 +26,6 @@ _TWO_20 = float(2 ** 20) has_psutil = False -has_resource = False # .. get available packages .. try: @@ -35,43 +34,21 @@ except ImportError: pass -try: - import resource - has_resource = True -except ImportError: - pass - -# divide resource.getrusage() by rusage_denom to get MB -rusage_denom = 1024. -if sys.platform == 'darwin': - # ... it seems that in OSX the output is different units ... - rusage_denom = rusage_denom * rusage_denom - def _get_memory(pid, timestamps=False, include_children=False): # .. only for current process and only on unix.. if pid == -1: - # .. seems to get wrong measurements on some cases, see .. - # .. https://github.com/fabianp/memory_profiler/issues/52 .. - if False: # has_resource: - mem = (resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / - rusage_denom) - if timestamps: - return (mem, time.time()) - else: - return mem - else: - pid = os.getpid() + pid = os.getpid() # .. cross-platform but but requires psutil .. if has_psutil: process = psutil.Process(pid) try: - mem = process.get_memory_info()[0] / (_TWO_20) + mem = process.get_memory_info()[0] / _TWO_20 if include_children: for p in process.get_children(recursive=True): - mem += p.get_memory_info()[0] / (_TWO_20) + mem += p.get_memory_info()[0] / _TWO_20 if timestamps: return (mem, time.time()) else: From 8d4d71859fbf46c3016570151f7ed6fbd112a3ab Mon Sep 17 00:00:00 2001 From: Philippe Gervais Date: Fri, 26 Jul 2013 12:14:42 +0200 Subject: [PATCH 058/415] Added documentation for the 'mprof' script --- README.rst | 53 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 273f9e3..9569627 100644 --- a/README.rst +++ b/README.rst @@ -91,12 +91,57 @@ memory_profiler`` in the command line. Executing external scripts ========================== Sometimes it is useful to have full memory usage reports as a function of -time (not line-by-line) of external processess (be it Python scripts or not). -In this case the executable ``mprof`` might be useful. To use it like:: +time (not line-by-line) of external processes (be it Python scripts or not). +In this case the executable ``mprof`` might be useful. Use it like:: - ./mprof run --python name_of_script.py + mprof run + mprof plot -TODO: make it work without the --python option. +The first line run the executable and record memory usage along time, +in a file written in the current directory. +Once it's done, a graph plot can be obtained using the second line. +The recorded file contains a timestamps, that allows for several +profiles to be kept at the same time. + +Help on each `mprof` subcommand can be obtained with the `-h` flag, +e.g. `mprof run -h`. + +In the case of a Python script, using the previous command does not +give you any information on which function is executed at a given +time. Depending on the case, it can be difficult to identify the part +of the code that is causing the highest memory usage. + +Adding the `profile` decorator to a function and running the Python +script with + + mprof run --python