Skip to content

Commit

Permalink
tests for blended competitor and API consistency
Browse files Browse the repository at this point in the history
  • Loading branch information
Will McGinnis committed Mar 29, 2020
1 parent f91a1f4 commit ad70f4d
Show file tree
Hide file tree
Showing 8 changed files with 245 additions and 39 deletions.
17 changes: 17 additions & 0 deletions docs/source/arenas.rst
Original file line number Diff line number Diff line change
@@ -1,6 +1,23 @@
Arenas
======

Arenas are objects that manage populations of competitors and their matchups. Currently there is only one
type of arena implemented, LambdaArenas

Lambda Arena
------------

.. autoclass:: elote.arenas.lambda_arena.LambdaArena
:members:


Helpers
-------

.. autoclass:: elote.arenas.base.History
:members:

.. autoclass:: elote.arenas.base.Bout
:members:


25 changes: 20 additions & 5 deletions docs/source/competitors.rst
Original file line number Diff line number Diff line change
@@ -1,17 +1,32 @@
Competitors
===========

Elo Competitor
--------------

.. autoclass:: elote.competitors.elo.EloCompetitor
:members: export_state,expected_score,beat,tied
:members: export_state,expected_score,beat,tied,rating

Glicko Competitor
-----------------

.. autoclass:: elote.competitors.glicko.GlickoCompetitor
:members: export_state,expected_score,beat,tied
:members: export_state,expected_score,beat,tied,rating

DWZ Competitor
--------------

.. autoclass:: elote.competitors.dwz.DWZCompetitor
:members: export_state,expected_score,beat,tied
:members: export_state,expected_score,beat,tied,rating

ECF Competitor
--------------

.. autoclass:: elote.competitors.ecf.ECFCompetitor
:members: export_state,expected_score,beat,tied
:members: export_state,expected_score,beat,tied,rating

BlendedCompetitor
-----------------

.. autoclass:: elote.competitors.ensemble.BlendedCompetitor
:members: export_state,expected_score,beat,tied
:members: export_state,expected_score,beat,tied,rating
78 changes: 74 additions & 4 deletions elote/arenas/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,26 @@ def export_state(self):

class History:
def __init__(self):
"""
"""
self.bouts = []

def add_bout(self, bout):
"""
:param bout:
:return:
"""
self.bouts.append(bout)

def report_results(self, lower_threshold=0.5, upper_threshold=0.5):
"""
:param lower_threshold:
:param upper_threshold:
:return:
"""
report = list()
for bout in self.bouts:
report.append({
Expand All @@ -44,6 +58,13 @@ def report_results(self, lower_threshold=0.5, upper_threshold=0.5):
return report

def confusion_matrix(self, lower_threshold=0.5, upper_threshold=0.5, attribute_filter=None):
"""
:param lower_threshold:
:param upper_threshold:
:param attribute_filter:
:return:
"""
tp, fp, tn, fn, do_nothing = 0, 0, 0, 0, 0
for bout in self.bouts:
match = True
Expand All @@ -67,6 +88,11 @@ def confusion_matrix(self, lower_threshold=0.5, upper_threshold=0.5, attribute_f
return tp, fp, tn, fn, do_nothing

def random_search(self, trials=1000):
"""
:param trials:
:return:
"""
best_net, best_thresholds = 0, list()
for _ in range(trials):
thresholds = sorted([random.random(), random.random()])
Expand All @@ -80,53 +106,97 @@ def random_search(self, trials=1000):

class Bout:
def __init__(self, a, b, predicted_outcome, outcome, attributes=None):
"""
:param a:
:param b:
:param predicted_outcome:
:param outcome:
:param attributes:
"""
self.a = a
self.b = b
self.predicted_outcome = predicted_outcome
self.outcome = outcome
self.attributes = attributes or dict()

def true_positive(self, threshold=0.5):
"""
:param threshold:
:return:
"""
if self.predicted_outcome > threshold and self.outcome == 'win':
return True
else:
return False

def false_positive(self, threshold=0.5):
"""
:param threshold:
:return:
"""
if self.predicted_outcome > threshold and self.outcome != 'win':
return True
else:
return False

def true_negative(self, threshold=0.5):
"""
:param threshold:
:return:
"""
if self.predicted_outcome <= threshold and self.outcome == 'loss':
return True
else:
return False

def false_negative(self, threshold=0.5):
"""
:param threshold:
:return:
"""
if self.predicted_outcome <= threshold and self.outcome != 'loss':
return True
else:
return False

def predicted_winner(self, lower_treshold=0.5, upper_threshold=0.5):
def predicted_winner(self, lower_threshold=0.5, upper_threshold=0.5):
"""
:param lower_threshold:
:param upper_threshold:
:return:
"""
if self.predicted_outcome > upper_threshold:
return self.a
elif self.predicted_outcome < lower_treshold:
elif self.predicted_outcome < lower_threshold:
return self.b
else:
return None

def predicted_loser(self, lower_treshold=0.5, upper_threshold=0.5):
def predicted_loser(self, lower_threshold=0.5, upper_threshold=0.5):
"""
:param lower_threshold:
:param upper_threshold:
:return:
"""
if self.predicted_outcome > upper_threshold:
return self.b
elif self.predicted_outcome < lower_treshold:
elif self.predicted_outcome < lower_threshold:
return self.a
else:
return None

def actual_winner(self):
"""
:return:
"""
if self.outcome == 'win':
return self.a
elif self.outcome == 'loss':
Expand Down
29 changes: 19 additions & 10 deletions elote/competitors/dwz.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,27 +15,36 @@ class vars:
:param initial_rating: the initial rating to use for a new competitor who has no history. Default 400
"""
self._count = 0
self.rating = initial_rating
self._rating = initial_rating

def __repr__(self):
return '<DWZCompetitor: %s>' % (self.__hash__())

def __str__(self):
return '<DWZCompetitor>'

@property
def rating(self):
return self._rating

@rating.setter
def rating(self, value):
self._rating = value

def export_state(self):
"""
Exports all information needed to re-create this competitor from scratch later on.
:return: dictionary of kwargs and class-args to re-instantiate this object
"""
return {
"initial_rating": self.rating,
"initial_rating": self._rating,
"class_vars": {
"_J": self._J
}
}


def expected_score(self, competitor: BaseCompetitor):
"""
The expected outcome of a match between this competitor and one passed in. Scaled between 0-1, where 1 is a strong
Expand All @@ -46,15 +55,15 @@ def expected_score(self, competitor: BaseCompetitor):
"""
self.verify_competitor_types(competitor)

return 1 / (1 + 10 ** ((competitor.rating - self.rating) / 400 ))
return 1 / (1 + 10 ** ((competitor.rating - self._rating) / 400))

@property
def _E(self):
E0 = (self.rating / 1000) ** 4 + self._J
a = max([0.5, min([self.rating / 2000, 1])])
E0 = (self._rating / 1000) ** 4 + self._J
a = max([0.5, min([self._rating / 2000, 1])])

if self.rating < 1300:
B = math.exp((1300 - self.rating) / 150) - 1
if self._rating < 1300:
B = math.exp((1300 - self._rating) / 150) - 1
else:
B = 0

Expand All @@ -65,7 +74,7 @@ def _E(self):
return max([5, min([E, 150])])

def _new_rating(self, competitor, W_a):
return self.rating + (800 / (self._E + self._count)) * (W_a - self.expected_score(competitor))
return self._rating + (800 / (self._E + self._count)) * (W_a - self.expected_score(competitor))

def beat(self, competitor: BaseCompetitor):
"""
Expand All @@ -80,7 +89,7 @@ def beat(self, competitor: BaseCompetitor):
self_rating = self._new_rating(competitor, 1)
competitor_rating = competitor._new_rating(self, 0)

self.rating = self_rating
self._rating = self_rating
self._count += 1

competitor.rating = competitor_rating
Expand All @@ -98,7 +107,7 @@ def tied(self, competitor: BaseCompetitor):
self_rating = self._new_rating(competitor, 0.5)
competitor_rating = competitor._new_rating(self, 0.5)

self.rating = self_rating
self._rating = self_rating
self._count += 1

competitor.rating = competitor_rating
Expand Down
18 changes: 13 additions & 5 deletions elote/competitors/elo.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def __init__(self, initial_rating: float = 400):
:param initial_rating: the initial rating to use for a new competitor who has no history. Default 400
:type initial_rating: int
"""
self.rating = initial_rating
self._rating = initial_rating

def __repr__(self):
return '<EloCompetitor: %s>' % (self.__hash__())
Expand All @@ -58,7 +58,7 @@ def export_state(self):
:return: dictionary of kwargs and class-args to re-instantiate this object
"""
return {
"initial_rating": self.rating,
"initial_rating": self._rating,
"class_vars": {
"_k_factor": self._k_factor,
"_base_rating": self._base_rating
Expand All @@ -67,7 +67,15 @@ def export_state(self):

@property
def transformed_rating(self):
return 10 ** (self.rating / self._base_rating)
return 10 ** (self._rating / self._base_rating)

@property
def rating(self):
return self._rating

@rating.setter
def rating(self, value):
self._rating = value

def expected_score(self, competitor: BaseCompetitor):
"""
Expand Down Expand Up @@ -97,7 +105,7 @@ def beat(self, competitor: BaseCompetitor):
lose_es = competitor.expected_score(self)

# update the winner's rating
self.rating = self.rating + self._k_factor * (1 - win_es)
self._rating = self._rating + self._k_factor * (1 - win_es)

# update the loser's rating
competitor.rating = competitor.rating + self._k_factor * (0 - lose_es)
Expand All @@ -116,7 +124,7 @@ def tied(self, competitor: BaseCompetitor):
lose_es = competitor.expected_score(self)

# update the winner's rating
self.rating = self.rating + self._k_factor * (0.5 - win_es)
self._rating = self._rating + self._k_factor * (0.5 - win_es)

# update the loser's rating
competitor.rating = competitor.rating + self._k_factor * (0.5 - lose_es)
Loading

0 comments on commit ad70f4d

Please sign in to comment.