# -*- coding: utf-8 -*- from __future__ import with_statement import inspect import os import subprocess import sys import execnet import py import pytest from execnet import gateway from execnet import gateway_base from execnet import gateway_io from execnet.gateway_base import ChannelFactory from execnet.gateway_base import Message from execnet.gateway_base import Popen2IO try: from StringIO import StringIO as BytesIO except: from io import BytesIO skip_win_pypy = pytest.mark.xfail( condition=hasattr(sys, "pypy_version_info") and sys.platform.startswith("win"), reason="failing on Windows on PyPy (#63)", ) class TestSerializeAPI: pytestmark = [pytest.mark.parametrize("val", ["123", 42, [1, 2, 3], ["23", 25]])] def test_serializer_api(self, val): dumped = execnet.dumps(val) val2 = execnet.loads(dumped) assert val == val2 def test_mmap(self, tmpdir, val): mmap = pytest.importorskip("mmap").mmap p = tmpdir.join("data") with p.open("wb") as f: f.write(execnet.dumps(val)) f = p.open("r+b") m = mmap(f.fileno(), 0) val2 = execnet.load(m) assert val == val2 def test_bytesio(self, val): f = py.io.BytesIO() execnet.dump(f, val) read = py.io.BytesIO(f.getvalue()) val2 = execnet.load(read) assert val == val2 def test_serializer_api_version_error(monkeypatch): bchr = gateway_base.bchr monkeypatch.setattr(gateway_base, "DUMPFORMAT_VERSION", bchr(1)) dumped = execnet.dumps(42) monkeypatch.setattr(gateway_base, "DUMPFORMAT_VERSION", bchr(2)) pytest.raises(execnet.DataFormatError, lambda: execnet.loads(dumped)) def test_errors_on_execnet(): assert hasattr(execnet, "RemoteError") assert hasattr(execnet, "TimeoutError") assert hasattr(execnet, "DataFormatError") def test_subprocess_interaction(anypython): line = gateway_io.popen_bootstrapline compile(line, "xyz", "exec") args = [str(anypython), "-c", line] popen = subprocess.Popen( args, bufsize=0, universal_newlines=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, ) def send(line): popen.stdin.write(line) if sys.version_info > (3, 0) or sys.platform.startswith("java"): popen.stdin.flush() def receive(): return popen.stdout.readline() try: source = py.code.Source(read_write_loop, "read_write_loop()") repr_source = repr(str(source)) + "\n" sendline = repr_source send(sendline) s = receive() assert s == "ok\n" send("hello\n") s = receive() assert s == "received: hello\n" send("world\n") s = receive() assert s == "received: world\n" send("\n") # terminate loop finally: popen.stdin.close() popen.stdout.close() popen.wait() def read_write_loop(): sys.stdout.write("ok\n") sys.stdout.flush() while 1: try: line = sys.stdin.readline() if not line.strip(): break sys.stdout.write("received: %s" % line) sys.stdout.flush() except (IOError, EOFError): break def test_io_message(anypython, tmpdir, execmodel): check = tmpdir.join("check.py") check.write( py.code.Source( gateway_base, """ try: from io import BytesIO except ImportError: from StringIO import StringIO as BytesIO import tempfile temp_out = BytesIO() temp_in = BytesIO() io = Popen2IO(temp_out, temp_in, get_execmodel({backend!r})) for i, handler in enumerate(Message._types): print ("checking %s %s" %(i, handler)) for data in "hello", "hello".encode('ascii'): msg1 = Message(i, i, dumps(data)) msg1.to_io(io) x = io.outfile.getvalue() io.outfile.truncate(0) io.outfile.seek(0) io.infile.seek(0) io.infile.write(x) io.infile.seek(0) msg2 = Message.from_io(io) assert msg1.channelid == msg2.channelid, (msg1, msg2) assert msg1.data == msg2.data, (msg1.data, msg2.data) assert msg1.msgcode == msg2.msgcode print ("all passed") """.format( backend=execmodel.backend ), ) ) # out = py.process.cmdexec("%s %s" %(executable,check)) out = anypython.sysexec(check) print(out) assert "all passed" in out def test_popen_io(anypython, tmpdir, execmodel): check = tmpdir.join("check.py") check.write( py.code.Source( gateway_base, """ do_exec("io = init_popen_io(get_execmodel({backend!r}))", globals()) io.write("hello".encode('ascii')) s = io.read(1) assert s == "x".encode('ascii') """.format( backend=execmodel.backend ), ) ) from subprocess import Popen, PIPE args = [str(anypython), str(check)] proc = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) proc.stdin.write("x".encode("ascii")) stdout, stderr = proc.communicate() print(stderr) proc.wait() assert "hello".encode("ascii") in stdout def test_popen_io_readloop(monkeypatch, execmodel): sio = BytesIO("test".encode("ascii")) io = Popen2IO(sio, sio, execmodel) real_read = io._read def newread(numbytes): if numbytes > 1: numbytes = numbytes - 1 return real_read(numbytes) io._read = newread result = io.read(3) assert result == "tes".encode("ascii") def test_rinfo_source(anypython, tmpdir): check = tmpdir.join("check.py") check.write( py.code.Source( """ class Channel: def send(self, data): assert eval(repr(data), {}) == data channel = Channel() """, gateway.rinfo_source, """ print ('all passed') """, ) ) out = anypython.sysexec(check) print(out) assert "all passed" in out def test_geterrortext(anypython, tmpdir): check = tmpdir.join("check.py") check.write( py.code.Source( gateway_base, """ class Arg: pass errortext = geterrortext((Arg, "1", 4)) assert "Arg" in errortext import sys try: raise ValueError("17") except ValueError: excinfo = sys.exc_info() s = geterrortext(excinfo) assert "17" in s print ("all passed") """, ) ) out = anypython.sysexec(check) print(out) assert "all passed" in out @pytest.mark.skipif("not hasattr(os, 'dup')") def test_stdouterrin_setnull(execmodel): cap = py.io.StdCaptureFD() gateway_base.init_popen_io(execmodel) os.write(1, "hello".encode("ascii")) os.read(0, 1) out, err = cap.reset() assert not out assert not err class PseudoChannel: class gateway: class _channelfactory: finished = False def __init__(self): self._sent = [] self._closed = [] self.id = 1000 def send(self, obj): self._sent.append(obj) def close(self, errortext=None): self._closed.append(errortext) def test_exectask(execmodel): io = py.io.BytesIO() io.execmodel = execmodel gw = gateway_base.WorkerGateway(io, id="something") ch = PseudoChannel() gw.executetask((ch, ("raise ValueError()", None, {}))) assert "ValueError" in str(ch._closed[0]) class TestMessage: def test_wire_protocol(self): for i, handler in enumerate(Message._types): one = py.io.BytesIO() data = "23".encode("ascii") Message(i, 42, data).to_io(one) two = py.io.BytesIO(one.getvalue()) msg = Message.from_io(two) assert msg.msgcode == i assert isinstance(msg, Message) assert msg.channelid == 42 assert msg.data == data assert isinstance(repr(msg), str) class TestPureChannel: @pytest.fixture def fac(self, execmodel): class Gateway: pass Gateway.execmodel = execmodel return ChannelFactory(Gateway) def test_factory_create(self, fac): chan1 = fac.new() assert chan1.id == 1 chan2 = fac.new() assert chan2.id == 3 def test_factory_getitem(self, fac): chan1 = fac.new() assert fac._channels[chan1.id] == chan1 chan2 = fac.new() assert fac._channels[chan2.id] == chan2 def test_channel_timeouterror(self, fac): channel = fac.new() pytest.raises(IOError, channel.waitclose, timeout=0.01) def test_channel_makefile_incompatmode(self, fac): channel = fac.new() with pytest.raises(ValueError): channel.makefile("rw") class TestSourceOfFunction(object): def test_lambda_unsupported(self): pytest.raises(ValueError, gateway._source_of_function, lambda: 1) def test_wrong_prototype_fails(self): def prototype(wrong): pass pytest.raises(ValueError, gateway._source_of_function, prototype) def test_function_without_known_source_fails(self): # this one wont be able to find the source mess = {} py.builtin.exec_("def fail(channel): pass", mess, mess) print(inspect.getsourcefile(mess["fail"])) pytest.raises(ValueError, gateway._source_of_function, mess["fail"]) def test_function_with_closure_fails(self): mess = {} def closure(channel): print(mess) pytest.raises(ValueError, gateway._source_of_function, closure) def test_source_of_nested_function(self): def working(channel): pass send_source = gateway._source_of_function(working).lstrip("\r\n") expected = "def working(channel):\n pass\n" assert send_source == expected class TestGlobalFinder(object): def check(self, func): src = py.code.Source(func) code = py.code.Code(func) return gateway._find_non_builtin_globals(str(src), code.raw) def test_local(self): def f(a, b, c): d = 3 return d assert self.check(f) == [] def test_global(self): def f(a, b): sys d = 4 return d assert self.check(f) == ["sys"] def test_builtin(self): def f(): len assert self.check(f) == [] def test_function_with_global_fails(self): def func(channel): sys pytest.raises(ValueError, gateway._source_of_function, func) def test_method_call(self): # method names are reason # for the simple code object based heusteric failing def f(channel): channel.send(dict(testing=2)) assert self.check(f) == [] @skip_win_pypy def test_remote_exec_function_with_kwargs(anypython, makegateway): def func(channel, data): channel.send(data) gw = makegateway("popen//python=%s" % anypython) print("local version_info {!r}".format(sys.version_info)) print("remote info: {}".format(gw._rinfo())) ch = gw.remote_exec(func, data=1) result = ch.receive() assert result == 1 def test_remote_exc__no_kwargs(makegateway): gw = makegateway() with pytest.raises(TypeError): gw.remote_exec(gateway_base, kwarg=1) with pytest.raises(TypeError): gw.remote_exec("pass", kwarg=1) @skip_win_pypy def test_remote_exec_inspect_stack(makegateway): gw = makegateway() ch = gw.remote_exec( """ import inspect inspect.stack() import traceback channel.send('\\n'.join(traceback.format_stack())) """ ) assert 'File ""' in ch.receive() ch.waitclose()