diff --git a/.circleci/config.yml b/.circleci/config.yml index 1e710e7..2b7d8ce 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -78,7 +78,7 @@ jobs: . env/bin/activate pip install -e . - check-version-updated: + check-validator-version-updated: docker: - image: cimg/python:3.10 steps: @@ -87,18 +87,7 @@ jobs: - run: name: Version is updated command: | - [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep prompting/__init__.py | wc -l) == 1 ]] && echo "prompting/__init__.py has changed" - [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep VERSION | wc -l) == 1 ]] && echo "VERSION has changed" - - check-changelog-updated: - docker: - - image: cimg/python:3.10 - steps: - - checkout - - run: - name: File CHANGELOG.md is updated - command: | - [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep CHANGELOG.md | wc -l) == 1 ]] && echo "CHANGELOG.md has changed" + [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep prompting/validators/__init__.py | wc -l) == 1 ]] && echo "prompting/validators/__init__.py has changed" check-version-not-released: docker: @@ -123,12 +112,7 @@ workflows: release-branches-requirements: jobs: - - check-version-updated: - filters: - branches: - only: - - /^(release|hotfix)/.*/ - - check-changelog-updated: + - check-validator-version-updated: filters: branches: only: diff --git a/prompting/validators/__init__.py b/prompting/validators/__init__.py index 38fe3a4..f75e905 100644 --- a/prompting/validators/__init__.py +++ b/prompting/validators/__init__.py @@ -27,7 +27,7 @@ from . import event from . import dataset -__version__ = "2.1.3" +__version__ = "2.1.4" version_split = __version__.split(".") __spec_version__ = ( (1000 * int(version_split[0])) diff --git a/prompting/validators/forward.py b/prompting/validators/forward.py index c1971c8..b354962 100644 --- a/prompting/validators/forward.py +++ b/prompting/validators/forward.py @@ -230,7 +230,7 @@ async def forward(self): base_text = ".".join(data.split(".", maxsplit=random_cutoff)[:-1]) # Create a summary task from the context. - summary_task: Task = create_summarization_task(base_text) + summary_task: Task = create_summarization_task(base_text) # Request a summary, given the original context. summarization_event = await run_step( diff --git a/prompting/validators/mock.py b/prompting/validators/mock.py index 461aaff..12d2c14 100644 --- a/prompting/validators/mock.py +++ b/prompting/validators/mock.py @@ -60,6 +60,12 @@ def __init__(self, mock_name: str = "MockReward"): self.question_blacklist = [] self.answer_blacklist = [] + def add(self, texts: List[str]): + pass + + def set_counter_to_half(self): + pass + def apply(self, prompt: str, completion: List[str], name: str) -> torch.FloatTensor: mock_reward = torch.tensor([1 for _ in completion], dtype=torch.float32) return mock_reward, {} diff --git a/prompting/validators/reward/diversity.py b/prompting/validators/reward/diversity.py index 744254d..3f16977 100644 --- a/prompting/validators/reward/diversity.py +++ b/prompting/validators/reward/diversity.py @@ -164,7 +164,7 @@ def get_rewards( ) -> List[DiversityRewardEvent]: # Check if completions are empty, return 0 if so if len(completions) == 0: - return torch.tensor([]).to(self.device), None + return torch.tensor([]).to(self.device) # Get embeddings for all completions. embeddings = self.get_embeddings(completions) diff --git a/prompting/validators/reward/reward.py b/prompting/validators/reward/reward.py index 54acea4..abfecb4 100644 --- a/prompting/validators/reward/reward.py +++ b/prompting/validators/reward/reward.py @@ -30,6 +30,11 @@ class BaseRewardEvent: @staticmethod def parse_reward_events(reward_events): + if reward_events == None or len(reward_events) == 0: + field_names = [field.name for field in fields(BaseRewardEvent())] + empty_reward_event = dict(zip(field_names, [[]] * len(field_names))) + return empty_reward_event + field_names = [field.name for field in fields(reward_events[0])] reward_events = [ asdict(reward_event).values() for reward_event in reward_events @@ -167,11 +172,13 @@ def apply( reward_events = {f"{self.name}_{k}": v for k, v in reward_events.items()} reward_events[self.name] = filled_rewards.tolist() reward_events[self.name + "_normalized"] = filled_rewards_normalized.tolist() - + # Warns unexpected behavior for rewards if torch.isnan(filled_rewards_normalized).any(): - bt.logging.warning(f"The tensor from {self.name} contains NaN values: {filled_rewards_normalized}") - filled_rewards_normalized.nan_to_num_(nan=0.0) + bt.logging.warning( + f"The tensor from {self.name} contains NaN values: {filled_rewards_normalized}" + ) + filled_rewards_normalized.nan_to_num_(nan=0.0) # Return the filled rewards. return filled_rewards_normalized, reward_events diff --git a/requirements.txt b/requirements.txt index 6e3213b..d42f5d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ bittensor==6.1.0 torch transformers==4.30.0 wandb==0.15.10 -datasets==2.14.0 +datasets==2.14.6 plotly==5.14.1 networkx==3.1 scipy==1.10.1 diff --git a/tests/validators/reward/test_reward_event.py b/tests/validators/reward/test_reward_event.py index e413dd5..d61d41a 100644 --- a/tests/validators/reward/test_reward_event.py +++ b/tests/validators/reward/test_reward_event.py @@ -20,6 +20,7 @@ from dataclasses import fields import prompting.validators.reward as reward + class RewardEventTestCase(unittest.TestCase): """ This class contains unit tests for the RewardEvent classes. @@ -30,16 +31,15 @@ class RewardEventTestCase(unittest.TestCase): def setUp(self): self.event_classes = [ - reward.reward.BaseRewardEvent, # Represents a reward model (float) - reward.nsfw.NSFWRewardEvent, # Remaining events are filters + reward.reward.BaseRewardEvent, # Represents a reward model (float) + reward.nsfw.NSFWRewardEvent, # Remaining events are filters reward.blacklist.BlacklistRewardEvent, reward.relevance.RelevanceRewardEvent, - reward.diversity.DiversityRewardEvent + reward.diversity.DiversityRewardEvent, ] self.reward_events = {} for event in self.event_classes: - event_type = event.__name__ self.reward_events[event_type] = [] @@ -53,7 +53,7 @@ def setUp(self): for field in fields(ev): # don't modify the is_filter_model field - if field.name == 'is_filter_model': + if field.name == "is_filter_model": continue # otherwise set the field to a float (including reward) setattr(ev, field.name, 1.234) @@ -61,23 +61,53 @@ def setUp(self): self.reward_events[event_type].append(ev) def test_no_missing_rewards(self): - for name, events in self.reward_events.items(): - parsed = reward.reward.BaseRewardEvent.parse_reward_events(events) # Ensure that all rewards are not None - self.assertTrue(all(r is not None for r in parsed['reward']), f'Events for {name} are missing rewards') - + self.assertTrue( + all(r is not None for r in parsed["reward"]), + f"Events for {name} are missing rewards", + ) def test_imputed_reward_values_are_correct(self): - for name, events in self.reward_events.items(): - - expected_value = 1 if events[0].is_filter_model else 0 - indices_missing_reward = [i for i, ev in enumerate(events) if ev.reward is None] + expected_value = 1 + indices_missing_reward = [ + i for i, ev in enumerate(events) if ev.reward is None + ] parsed = reward.reward.BaseRewardEvent.parse_reward_events(events) # Ensure that all rewards are not None - self.assertTrue(all(parsed['reward'][i]==expected_value for i in indices_missing_reward), f'Events for {name} were imputed with incorrect reward value') + self.assertTrue( + all( + parsed["reward"][i] == expected_value + for i in indices_missing_reward + ), + f"Events for {name} were imputed with incorrect reward value", + ) + + def test_parse_reward_events_with_reward_events(self): + # Create sample reward events + event1 = reward.reward.BaseRewardEvent(reward=1, normalized_reward="event1") + event2 = reward.reward.BaseRewardEvent(reward=2, normalized_reward="event2") + events = [event1, event2] + + # Expected result + expected = {"reward": (1, 2), "normalized_reward": ("event1", "event2")} + + # Call the function and check if the result matches the expected output + result = reward.reward.BaseRewardEvent.parse_reward_events(events) + self.assertEqual(result, expected) + + def test_parse_reward_events_with_no_reward_events(self): + # Test with None + result_none = reward.reward.BaseRewardEvent.parse_reward_events(None) + self.assertTrue(all(len(lst) == 0 for lst in result_none.values())) + self.assertEqual(result_none, {"reward": [], "normalized_reward": []}) + + # Test with empty list + result_empty = reward.reward.BaseRewardEvent.parse_reward_events([]) + self.assertTrue(all(len(lst) == 0 for lst in result_empty.values())) + self.assertEqual(result_empty, {"reward": [], "normalized_reward": []})