Remove broken final state loop (#874)

Fixes #856

The code this PR removes introduces an artificial and erroneous loop
transition in every final state that is always traversed, regardless of
the generation.

The comment doesn't make sense in my opinion, as the `if` above just
handles exactly this case.

Removing this piece of code fixes the bug that surfaced in the upgrade
of outlines in the vLLM integration.

outlines/fsm/guide.py

@@ -193,12 +193,8 @@ def get_next_state(self, state: int, token_id: int) -> int:
         The new state of the guide.
 
         """
-        if token_id == self.eos_token_id:
+        if token_id == self.eos_token_id or state not in self.states_to_token_maps:
             return -1
-        elif (
-            state in self.final_states
-        ):  # Necessary because we keep generating EOS tokens when finished
-            return state
 
         last_token_to_end_state = self.states_to_token_maps[state]
         next_state = last_token_to_end_state.get(token_id)

tests/fsm/test_fsm.py

@@ -82,9 +82,7 @@ def convert_token_to_string(self, token):
     assert fsm.is_final_state(state)
 
     state = fsm.next_state(state=5, token_id=103)
-    assert state == 5
-
-    assert fsm.is_final_state(-1)
+    assert fsm.is_final_state(state)
 
 
 def test_cfg():

tests/fsm/test_guide.py

@@ -180,9 +180,7 @@ def convert_token_to_string(self, token):
     assert fsm.is_final_state(state)
 
     state = fsm.get_next_state(state=5, token_id=103)
-    assert state == 5
-
-    assert fsm.is_final_state(-1)
+    assert fsm.is_final_state(state)
 
 
 def test_cfg():