Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Worker quitting then stopping via web UI bug fix #1324

Merged
merged 21 commits into from
Apr 16, 2020
Merged
Show file tree
Hide file tree
Changes from 15 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
b80d945
Made it so stop only performs if state isn't already stopping/stopped…
Trouv Apr 13, 2020
6f4473d
Added second fix in web.py
Trouv Apr 13, 2020
a918d81
made it so test_stop event is fired when all workers quit
Trouv Apr 13, 2020
00b4da8
Added log message
Trouv Apr 13, 2020
1ca93b6
Fixed import merge conflict in web.py
Trouv Apr 14, 2020
358ffa5
Removed the alternative api message
Trouv Apr 14, 2020
26c0b01
Moved STATE_STOPPED check to the heartbeat method, as well as checkin…
Trouv Apr 14, 2020
9a414ba
Added info arg to stop() to log messages on stop only when it actuall…
Trouv Apr 14, 2020
72a7b87
Added heartbeat mocking for broken test case
Trouv Apr 14, 2020
6b2c81a
Alternative mock heatbeat solution, less things to pass around
Trouv Apr 14, 2020
2ab666f
Merge branch 'master' of github.com:Trouv/locust into worker-quit-ui-fix
Trouv Apr 15, 2020
1021d8e
Made it so UI updates to stopped form when workers quit
Trouv Apr 15, 2020
12f32e7
Added .stopping wherever .stopped is in css
Trouv Apr 15, 2020
4b575b9
Undid some no-longer-necessary changes
Trouv Apr 15, 2020
6b511f4
Removed newline in locust.js
Trouv Apr 15, 2020
82e8b66
Added check_stopped function and made new stops specific to reason. R…
Trouv Apr 15, 2020
71f98b2
Merge branch 'master' of github.com:Trouv/locust into worker-quit-ui-fix
Trouv Apr 16, 2020
7b40b18
Moved stop check to inside if, moved jquery stop set to inside button…
Trouv Apr 16, 2020
fc2fbd4
Added tests
Trouv Apr 16, 2020
a80d02b
Merge branch 'master' of github.com:Trouv/locust into worker-quit-ui-fix
Trouv Apr 16, 2020
59f117f
made new tests faster
Trouv Apr 16, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 20 additions & 7 deletions locust/runners.py
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,10 @@ def hatching(self):
@property
def running(self):
return self.get_by_state(STATE_RUNNING)

@property
def missing(self):
return self.get_by_state(STATE_MISSING)

self.clients = WorkerNodesDict()
self.server = rpc.Server(master_bind_host, master_bind_port)
Expand Down Expand Up @@ -410,11 +414,15 @@ def start(self, locust_count, hatch_rate):

self.state = STATE_HATCHING

def stop(self):
self.state = STATE_STOPPING
for client in self.clients.all:
self.server.send_to_client(Message("stop", None, client.id))
self.environment.events.test_stop.fire(environment=self.environment)
def stop(self, info=None):
Trouv marked this conversation as resolved.
Show resolved Hide resolved
if self.state not in [STATE_INIT, STATE_STOPPED, STATE_STOPPING]:
if info is not None:
logger.info(info)

self.state = STATE_STOPPING
for client in self.clients.all:
self.server.send_to_client(Message("stop", None, client.id))
self.environment.events.test_stop.fire(environment=self.environment)

def quit(self):
if self.state not in [STATE_INIT, STATE_STOPPED, STATE_STOPPING]:
Expand All @@ -432,6 +440,7 @@ def heartbeat_worker(self):
if self.connection_broken:
self.reset_connection()
continue

for client in self.clients.all:
if client.heartbeat < 0 and client.state != STATE_MISSING:
logger.info('Worker %s failed to send heartbeat, setting state to missing.' % str(client.id))
Expand All @@ -440,6 +449,12 @@ def heartbeat_worker(self):
else:
client.heartbeat -= 1

if self.worker_count - len(self.clients.missing) <= 0:
self.stop(info='No workers remaining, stopping test.')
Trouv marked this conversation as resolved.
Show resolved Hide resolved

if not self.state == STATE_INIT and all(map(lambda x: x.state != STATE_RUNNING and x.state != STATE_HATCHING, self.clients.all)):
self.state = STATE_STOPPED

def reset_connection(self):
logger.info("Reset connection to slave")
try:
Expand Down Expand Up @@ -499,8 +514,6 @@ def client_listener(self):
elif msg.type == "exception":
self.log_exception(msg.node_id, msg.data["msg"], msg.data["traceback"])

if not self.state == STATE_INIT and all(map(lambda x: x.state != STATE_RUNNING and x.state != STATE_HATCHING, self.clients.all)):
self.state = STATE_STOPPED
Trouv marked this conversation as resolved.
Show resolved Hide resolved

@property
def worker_count(self):
Expand Down
14 changes: 10 additions & 4 deletions locust/static/locust.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,18 @@ $(window).ready(function() {
}
});

$("#box_stop a.stop-button").click(function(event) {
event.preventDefault();
$.get($(this).attr("href"));
function appearStopped() {
$("body").attr("class", "stopped");
$(".box_stop").hide();
$("a.new_test").show();
$("a.edit_test").hide();
$(".user_count").hide();
}

$("#box_stop a.stop-button").click(function(event) {
event.preventDefault();
$.get($(this).attr("href"));
appearStopped()
});

$("#box_stop a.reset-button").click(function(event) {
Expand Down Expand Up @@ -173,6 +177,8 @@ function updateStats() {
rpsChart.addValue([total.current_rps, total.current_fail_per_sec]);
responseTimeChart.addValue([report.current_response_time_percentile_50, report.current_response_time_percentile_95]);
usersChart.addValue([report.user_count]);
} else {
appearStopped()
}

setTimeout(updateStats, 2000);
Expand All @@ -187,4 +193,4 @@ function updateExceptions() {
setTimeout(updateExceptions, 5000);
});
}
updateExceptions();
updateExceptions();
22 changes: 11 additions & 11 deletions locust/static/style.css
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,8 @@ a:hover {
}
.hatching .boxes .box_running {display: block;}
.running .boxes .box_running {display: block;}
.stopped .boxes .box_running {display: block;}
.stopped .boxes .box_stop {display: none;}
.stopped .boxes .box_running, .stopping .boxes .box_running {display: block;}
.stopped .boxes .box_stop, .stopping .boxes .box_stop {display: none;}

.container {
max-width: 1800px;
Expand Down Expand Up @@ -206,7 +206,7 @@ a:hover {
}


.stopped .start {
.stopped .start, .stopping .start {
display: none;
border-radius: 5px;
-moz-border-radius: 5px;
Expand All @@ -216,7 +216,7 @@ a:hover {
box-shadow: 0 0 60px rgba(0,0,0,0.3);
}

.stopped .edit {display: none;}
.stopped .edit, .stopping .edit {display: none;}
.running .edit, .hatching .edit {
display: none;
border-radius: 5px;
Expand All @@ -232,25 +232,25 @@ a:hover {
.ready .start {display: block;}

.running .status, .hatching .status {display: block;}
.stopped .status {display: block;}
.stopped .status, .stopping .status {display: block;}
.ready .status {display: none;}

.stopped .boxes .edit_test, .ready .boxes .edit_test {display: none;}
.stopped .boxes .user_count, .ready .boxes .user_count {display: none;}
.stopped .boxes .edit_test, .stopping .boxes .edit_test, .ready .boxes .edit_test {display: none;}
.stopped .boxes .user_count, .stopping .boxes .user_count, .ready .boxes .user_count {display: none;}

.running a.new_test, .ready a.new_test, .hatching a.new_test {display: none;}
.running a.new_test {display: none;}
.stopped a.new_test {display: block;}
.stopped a.new_test, .stopping a.new_test {display: block;}

.start a.close_link, .edit a.close_link{
position: absolute;
right: 10px;
top: 10px;
}
.stopped .start a.close_link {display: inline;}
.stopped .start a.close_link, .stopping .start a.close_link {display: inline;}
.running .start a.close_link, .ready .start a.close_link, .hatching .start a.close_link {display: none;}

.stopped .edit a.close_link, .ready .edit a.close_link {display: none;}
.stopped .edit a.close_link, .stopping .edit a.close_link, .ready .edit a.close_link {display: none;}
.running .edit a.close_link, .hatching .edit a.close_link {display: inline;}

.stats_label {
Expand Down Expand Up @@ -461,7 +461,7 @@ ul.tabs li a.current:after {
}

.running .hostname, .hatching .hostname {display: block;}
.stopped .hostname {display: block;}
.stopped .hostname, .stopping .hostname {display: block;}
.ready .hostname {display: none;}

.footer {
Expand Down
22 changes: 18 additions & 4 deletions locust/test/test_runners.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import unittest

from time import time

import gevent
from gevent import sleep
from gevent.queue import Queue
Expand All @@ -12,7 +14,7 @@
from locust.exception import LocustError, RPCError, StopLocust
from locust.rpc import Message
from locust.runners import LocustRunner, LocalLocustRunner, MasterLocustRunner, WorkerNode, \
WorkerLocustRunner, STATE_INIT, STATE_HATCHING, STATE_RUNNING, STATE_MISSING
WorkerLocustRunner, STATE_INIT, STATE_HATCHING, STATE_RUNNING, STATE_MISSING, HEARTBEAT_INTERVAL
from locust.stats import RequestStats
from locust.test.testcases import LocustTestCase
from locust.wait_time import between, constant
Expand Down Expand Up @@ -386,7 +388,14 @@ def tearDown(self):

def get_runner(self):
return MasterLocustRunner(self.environment, [], master_bind_host="*", master_bind_port=5557)


def sleep_with_given_heartbeat(self, seconds, heartbeat):
start = time()
while time() - start < seconds:
heartbeat()
sleep(min(HEARTBEAT_INTERVAL, seconds - time() + start))


def test_worker_connect(self):
with mock.patch("locust.rpc.rpc.Server", mocked_rpc()) as server:
master = self.get_runner()
Expand Down Expand Up @@ -734,11 +743,15 @@ def test_spawn_locusts_in_stepload_mode(self):
for i in range(5):
server.mocked_send(Message("client_ready", None, "fake_client%i" % i))

def heartbeat():
for i in range(5):
server.mocked_send(Message("heartbeat", {'state': STATE_RUNNING, 'current_cpu_usage': 50}, "fake_client%i" % i))

Trouv marked this conversation as resolved.
Show resolved Hide resolved
# start a new swarming in Step Load mode: total locust count of 10, hatch rate of 2, step locust count of 5, step duration of 2s
master.start_stepload(10, 2, 5, 2)

# make sure the first step run is started
sleep(0.5)
self.sleep_with_given_heartbeat(0.5, heartbeat)
self.assertEqual(5, len(server.outbox))

num_clients = 0
Expand All @@ -749,7 +762,8 @@ def test_spawn_locusts_in_stepload_mode(self):
self.assertEqual(5, num_clients, "Total number of locusts that would have been spawned for first step is not 5")

# make sure the first step run is complete
sleep(2)
self.sleep_with_given_heartbeat(2, heartbeat)

num_clients = 0
idx = end_of_last_step
while idx < len(server.outbox):
Expand Down