Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Issue/pytest asyncio #846

Merged
merged 29 commits into from
Dec 18, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ Changes in this release:
- Extend mypy type annotations
- Use files for all logs and split out logs, stdout and stderr in autostarted agents (#824, #234)
- Introduce request_timeout option for transport settings
- Port unit tests to pytest-asyncio and fix deprecation warnings (#743)

v 2018.3 (2018-12-07)
Changes in this release:
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ pytest-cover==3.0.0
pytest-instafail==0.4.0
pytest-randomly==1.2.3
pytest-sugar==0.9.2
pytest-tornado==0.5.0
pytest-asyncio
pytest-xdist==1.24.1
pytest==3.10.0
python-dateutil==2.7.5
Expand Down
3 changes: 1 addition & 2 deletions src/inmanta/export.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,6 @@ def __init__(self, options=None):
self._scope = None

self._file_store = {}
self._io_loop = IOLoop.current()

def _get_instance_proxies_of_types(self, types):
"""
Expand Down Expand Up @@ -365,7 +364,7 @@ def resources_to_list(self) -> List[Dict[str, str]]:
return resources

def run_sync(self, function):
return self._io_loop.run_sync(function, 300)
return IOLoop.current(instance=True).run_sync(function, 300)

def deploy_code(self, tid, version=None):
"""
Expand Down
4 changes: 2 additions & 2 deletions src/inmanta/server/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -1529,8 +1529,8 @@ def _recompile_environment(self, environment_id, update_repo=False, wait=0, meta
stderr=process.Subprocess.STREAM,
cwd=project_dir)

out, _, _ = yield [gen.Task(sub_process.stdout.read_until_close),
gen.Task(sub_process.stderr.read_until_close),
out, _, _ = yield [sub_process.stdout.read_until_close(),
sub_process.stderr.read_until_close(),
bartv marked this conversation as resolved.
Show resolved Hide resolved
sub_process.wait_for_exit(raise_error=False)]

o = re.search("\* ([^\s]+)$", out.decode(), re.MULTILINE)
Expand Down
87 changes: 46 additions & 41 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,17 +29,14 @@
from inmanta import config, data, mongoproc
import inmanta.compiler as compiler
import pymongo
from motor import motor_tornado
from motor import motor_asyncio
from inmanta.module import Project
from inmanta import resources, export
from inmanta.agent import handler
from inmanta.ast import CompilerException
from click import testing
import inmanta.main
from concurrent.futures.thread import ThreadPoolExecutor
from tornado import gen
import re
from tornado.ioloop import IOLoop
from inmanta.server.bootloader import InmantaBootloader
from inmanta.export import cfg_env, unknown_parameters
import traceback
Expand Down Expand Up @@ -124,16 +121,19 @@ def mongo_client(mongo_db):


@pytest.fixture(scope="function")
def motor(mongo_db, mongo_client, io_loop):
client = motor_tornado.MotorClient('localhost', int(mongo_db.port), io_loop=io_loop)
def motor(mongo_db, mongo_client, event_loop):
wouterdb marked this conversation as resolved.
Show resolved Hide resolved
"""
Event_loop argument ensures this fixture is started after the eventloop is started
"""
client = motor_asyncio.AsyncIOMotorClient('localhost', int(mongo_db.port))
db = client["inmanta"]
yield db


@pytest.fixture(scope="function")
def data_module(io_loop, motor):
async def data_module(motor):
data.use_motor(motor)
io_loop.run_sync(data.create_indexes)
await data.create_indexes()


def get_free_tcp_port():
Expand Down Expand Up @@ -169,10 +169,9 @@ def inmanta_config():


@pytest.fixture(scope="function")
def server(inmanta_config, io_loop, mongo_db, mongo_client, motor):
async def server(inmanta_config, mongo_db, mongo_client, motor):
# fix for fact that pytest_tornado never set IOLoop._instance, the IOLoop of the main thread
# causes handler failure
IOLoop._instance = io_loop

state_dir = tempfile.mkdtemp()

Expand All @@ -191,25 +190,22 @@ def server(inmanta_config, io_loop, mongo_db, mongo_client, motor):
config.Config.set("server", "agent-timeout", "10")

data.use_motor(motor)
io_loop.run_sync(data.create_indexes)
await data.create_indexes()

ibl = InmantaBootloader()
ibl.start()

yield ibl.restserver

ibl.stop()
del IOLoop._instance
shutil.rmtree(state_dir)


@pytest.fixture(scope="function",
params=[(True, True, False), (True, False, False), (False, True, False),
(False, False, False), (True, True, True)],
ids=["SSL and Auth", "SSL", "Auth", "Normal", "SSL and Auth with not self signed certificate"])
def server_multi(inmanta_config, io_loop, mongo_db, mongo_client, request, motor):
IOLoop._instance = io_loop

async def server_multi(inmanta_config, mongo_db, mongo_client, request, motor):
state_dir = tempfile.mkdtemp()

ssl, auth, ca = request.param
Expand Down Expand Up @@ -257,7 +253,7 @@ def server_multi(inmanta_config, io_loop, mongo_db, mongo_client, request, motor
config.Config.set("server", "agent-timeout", "2")

data.use_motor(motor)
io_loop.run_sync(data.create_indexes)
await data.create_indexes()

ibl = InmantaBootloader()
ibl.start()
Expand All @@ -266,11 +262,6 @@ def server_multi(inmanta_config, io_loop, mongo_db, mongo_client, request, motor

ibl.stop()

try:
del IOLoop._instance
except Exception:
pass

shutil.rmtree(state_dir)


Expand All @@ -293,21 +284,21 @@ def client_multi(server_multi):


@pytest.fixture(scope="function")
def environment(client, server, io_loop):
async def environment(client, server):
"""
Create a project and environment. This fixture returns the uuid of the environment
"""
def create_project():
return client.create_project("env-test")

result = io_loop.run_sync(create_project)
result = await create_project()
assert(result.code == 200)
project_id = result.result["project"]["id"]

def create_env():
return client.create_environment(project_id=project_id, name="dev")

result = io_loop.run_sync(create_env)
result = await create_env()
env_id = result.result["environment"]["id"]

cfg_env.set(env_id)
Expand All @@ -316,21 +307,21 @@ def create_env():


@pytest.fixture(scope="function")
def environment_multi(client_multi, server_multi, io_loop):
async def environment_multi(client_multi, server_multi):
"""
Create a project and environment. This fixture returns the uuid of the environment
"""
def create_project():
return client_multi.create_project("env-test")

result = io_loop.run_sync(create_project)
result = await create_project()
assert(result.code == 200)
project_id = result.result["project"]["id"]

def create_env():
return client_multi.create_environment(project_id=project_id, name="dev")

result = io_loop.run_sync(create_env)
result = await create_env()
env_id = result.result["environment"]["id"]

yield env_id
Expand Down Expand Up @@ -364,6 +355,10 @@ def pytest_runtest_makereport(item, call):
["%s %s" % (label, resource) for label, resource in resources.items()])))


async def off_main_thread(func):
return await asyncio.get_event_loop().run_in_executor(None, func)


class SnippetCompilationTest(KeepOnFail):

def setUpClass(self):
Expand Down Expand Up @@ -414,7 +409,14 @@ def setup_for_snippet(self, snippet, autostd=True):

Project.set(Project(self.project_dir, autostd=autostd))

def do_export(self, deploy=False, include_status=False, do_raise=True):
def do_export(self, include_status=False, do_raise=True):
return self._do_export(deploy=False, include_status=include_status, do_raise=do_raise)

def _do_export(self, deploy=False, include_status=False, do_raise=True):
"""
helper function to allow actual export to be run an a different thread
i.e. export.run must run off main thread to allow it to start a new ioloop for run_sync
"""
templfile = mktemp("json", "dump", self.project_dir)

class Options(object):
Expand All @@ -439,10 +441,13 @@ class Options(object):

# Even if the compile failed we might have collected additional data such as unknowns. So
# continue the export

export = Exporter(options)

return export.run(types, scopes, model_export=False, include_status=include_status)

async def do_export_and_deploy(self, include_status=False, do_raise=True):
return await off_main_thread(lambda: self._do_export(deploy=True, include_status=include_status, do_raise=do_raise))

def setup_for_error(self, snippet, shouldbe):
self.setup_for_snippet(snippet)
try:
Expand Down Expand Up @@ -482,23 +487,23 @@ def snippetcompiler(snippetcompiler_global):


class CLI(object):

def __init__(self, io_loop):
self.io_loop = io_loop
self._thread_pool = ThreadPoolExecutor(1)

@gen.coroutine
def run(self, *args):
async def run(self, *args):
os.environ["COLUMNS"] = "1000"
runner = testing.CliRunner()
cmd_args = ["--host", "localhost", "--port", config.Config.get("cmdline_rest_transport", "port")]
cmd_args.extend(args)
result = yield self._thread_pool.submit(runner.invoke, cli=inmanta.main.cmd, args=cmd_args, obj=self.io_loop,
catch_exceptions=False)
return result

def invoke():
return runner.invoke(
cli=inmanta.main.cmd,
args=cmd_args,
catch_exceptions=False
)

return await asyncio.get_event_loop().run_in_executor(None, invoke)


@pytest.fixture
def cli(io_loop):
o = CLI(io_loop)
def cli():
o = CLI()
yield o
Loading