Predict ranks and their odds of entire match outcome. (#74)

Author: vivekjoshy

README.md

@@ -110,6 +110,26 @@ You can compare two or more teams to get the probabilities of the match drawing.
 0.09025541153402594
 ```
 
+## Predicting Ranks
+
+Sometimes you want to know what the likelihood is someone will place at a particular rank. You can use this library to predict those odds.
+
+```python
+>>> from openskill import predict_rank, predict_draw
+>>> a1 = a2 = a3 = Rating(mu=34, sigma=0.25)
+>>> b1 = b2 = b3 = Rating(mu=32, sigma=0.5)
+>>> c1 = c2 = c3 = Rating(mu=30, sigma=1)
+>>> team_1, team_2, team_3 = [a1, a2, a3], [b1, b2, b3], [c1, c2, c3]
+>>> draw_probability = predict_draw(teams=[team_1, team_2, team_3])
+>>> draw_probability
+0.3295385074666581
+>>> rank_probability = predict_rank(teams=[team_1, team_2, team_3])
+>>> rank_probability
+[(1, 0.4450361350569973), (2, 0.19655022513040032), (3, 0.028875132345944337)]
+>>> sum([y for x, y in rank_probability]) + draw_probability
+1.0
+```
+
 ## Choosing Models
 
 The default model is `PlackettLuce`. You can import alternate models from `openskill.models` like so:

benchmark/benchmark.py benchmark/rank_benchmark.pybenchmark/benchmark.py renamed to benchmark/rank_benchmark.py

@@ -4,12 +4,14 @@
 from typing import Union
 
 import jsonlines
+import numpy as np
 import trueskill
 from prompt_toolkit import HTML
 from prompt_toolkit import print_formatted_text as print
 from prompt_toolkit import prompt
 from prompt_toolkit.completion import WordCompleter
 from prompt_toolkit.shortcuts import ProgressBar
+from sklearn.model_selection import train_test_split
 
 import openskill
 from openskill.models import (
@@ -24,11 +26,19 @@
 os_players = {}
 ts_players = {}
 
+match_count = {}
+
+matches = []
+training_set = {}
+test_set = {}
+valid_test_set_matches = []
+
 # Counters
 os_correct_predictions = 0
 os_incorrect_predictions = 0
 ts_correct_predictions = 0
 ts_incorrect_predictions = 0
+confident_matches = 0
 
 
 print(HTML("<u><b>Benchmark Starting</b></u>"))
@@ -144,11 +154,15 @@ def predict_os_match(match: dict):
     for player in red_team:
         os_red_players[player] = os_players[player]
 
-    blue_win_probability, red_win_probability = openskill.predict_win(
+    blue_win_probability, red_win_probability = openskill.predict_rank(
         [list(os_blue_players.values()), list(os_red_players.values())]
     )
-    if (blue_win_probability > red_win_probability) == won:
-        global os_correct_predictions
+    blue_win_probability = blue_win_probability[0]
+    red_win_probability = red_win_probability[0]
+    global os_correct_predictions
+    if (blue_win_probability < red_win_probability) == won:
+        os_correct_predictions += 1
+    elif blue_win_probability == red_win_probability:  # Draw
         os_correct_predictions += 1
     else:
         global os_incorrect_predictions
@@ -179,7 +193,7 @@ def predict_ts_match(match: dict):
         ts_blue_players[player] = ts_players[player]
 
     for player in red_team:
-        ts_red_players[player] = os_players[player]
+        ts_red_players[player] = ts_players[player]
 
     blue_win_probability = win_probability(
         list(ts_blue_players.values()), list(ts_red_players.values())
@@ -193,6 +207,52 @@ def predict_ts_match(match: dict):
         ts_incorrect_predictions += 1
 
 
+def process_match(match: dict):
+    teams: dict = match.get("teams")
+    blue_team: dict = teams.get("blue")
+    red_team: dict = teams.get("red")
+
+    for player in blue_team:
+        match_count[player] = match_count.get(player, 0) + 1
+
+    for player in red_team:
+        match_count[player] = match_count.get(player, 0) + 1
+
+
+def valid_test_set(match: dict):
+    teams: dict = match.get("teams")
+    blue_team: dict = teams.get("blue")
+    red_team: dict = teams.get("red")
+
+    for player in blue_team:
+        if player not in os_players:
+            return False
+
+    for player in red_team:
+        if player not in os_players:
+            return False
+
+    return True
+
+
+def confident_in_match(match: dict) -> bool:
+    teams: dict = match.get("teams")
+    blue_team: dict = teams.get("blue")
+    red_team: dict = teams.get("red")
+
+    global confident_matches
+    for player in blue_team:
+        if match_count[player] < 2:
+            return False
+
+    for player in red_team:
+        if match_count[player] < 2:
+            return False
+
+    confident_matches += 1
+    return True
+
+
 models = [
     BradleyTerryFull,
     BradleyTerryPart,
@@ -203,6 +263,7 @@ def predict_ts_match(match: dict):
 model_names = [m.__name__ for m in models]
 model_completer = WordCompleter(model_names)
 input_model = prompt("Enter Model: ", completer=model_completer)
+
 if input_model in model_names:
     index = model_names.index(input_model)
 else:
@@ -211,41 +272,71 @@ def predict_ts_match(match: dict):
 with jsonlines.open("v2_jsonl_teams.jsonl") as reader:
     lines = list(reader.iter())
 
-    # Process OpenSkill Ratings
-    title = HTML(f'Updating Ratings with <style fg="Green">{input_model}</style> Model')
+    title = HTML(f'<style fg="Red">Processing Matches</style>')
     with ProgressBar(title=title) as progress_bar:
-        os_process_time_start = time.time()
         for line in progress_bar(lines, total=len(lines)):
             if data_verified(match=line):
-                process_os_match(match=line, model=models[index])
+                process_match(match=line)
+
+    # Measure Confidence
+    title = HTML(f'<style fg="Red">Splitting Data</style>')
+    with ProgressBar(title=title) as progress_bar:
+        for line in progress_bar(lines, total=len(lines)):
+            if data_verified(match=line):
+                if confident_in_match(match=line):
+                    matches.append(line)
+
+    # Split Data
+    training_set, test_set = train_test_split(
+        matches, test_size=0.33, random_state=True
+    )
+
+    # Process OpenSkill Ratings
+    title = HTML(
+        f'Updating Ratings with <style fg="Green">{input_model}</style> Model:'
+    )
+    with ProgressBar(title=title) as progress_bar:
+        os_process_time_start = time.time()
+        for line in progress_bar(training_set, total=len(training_set)):
+            process_os_match(match=line, model=models[index])
     os_process_time_stop = time.time()
     os_time = os_process_time_stop - os_process_time_start
 
     # Process TrueSkill Ratings
-    title = HTML(f'Updating Ratings with <style fg="Green">TrueSkill</style> Model')
+    title = HTML(f'Updating Ratings with <style fg="Green">TrueSkill</style> Model:')
     with ProgressBar(title=title) as progress_bar:
         ts_process_time_start = time.time()
-        for line in progress_bar(lines, total=len(lines)):
-            if data_verified(match=line):
-                process_ts_match(match=line)
+        for line in progress_bar(training_set, total=len(training_set)):
+            process_ts_match(match=line)
     ts_process_time_stop = time.time()
     ts_time = ts_process_time_stop - ts_process_time_start
 
+    # Process Test Set
+    title = HTML(f'<style fg="Red">Processing Test Set</style>')
+    with ProgressBar(title=title) as progress_bar:
+        for line in progress_bar(test_set, total=len(test_set)):
+            if valid_test_set(match=line):
+                valid_test_set_matches.append(line)
+
     # Predict OpenSkill Matches
     title = HTML(f'<style fg="Blue">Predicting OpenSkill Matches:</style>')
     with ProgressBar(title=title) as progress_bar:
-        for line in progress_bar(lines, total=len(lines)):
-            if data_verified(match=line):
-                predict_os_match(match=line)
+        for line in progress_bar(
+            valid_test_set_matches, total=len(valid_test_set_matches)
+        ):
+            predict_os_match(match=line)
 
     # Predict TrueSkill Matches
     title = HTML(f'<style fg="Blue">Predicting TrueSkill Matches:</style>')
     with ProgressBar(title=title) as progress_bar:
-        for line in progress_bar(lines, total=len(lines)):
-            if data_verified(match=line):
-                predict_ts_match(match=line)
+        for line in progress_bar(
+            valid_test_set_matches, total=len(valid_test_set_matches)
+        ):
+            predict_ts_match(match=line)
 
+mean = float(np.array(list(match_count.values())).mean())
 
+print(HTML(f"Confident Matches:  <style fg='Yellow'>{confident_matches}</style>"))
 print(
     HTML(
         f"Predictions Made with OpenSkill's <style fg='Green'><u>{input_model}</u></style> Model:"
@@ -281,3 +372,4 @@ def predict_ts_match(match: dict):
     )
 )
 print(HTML(f"Process Duration: <style fg='Yellow'>{ts_time}</style>"))
+print(HTML(f"Mean Matches: <style fg='Yellow'>{mean}</style>"))

benchmark/split_benchmark.py benchmark/win_benchmark.pybenchmark/split_benchmark.py renamed to benchmark/win_benchmark.py

@@ -119,6 +119,26 @@ You can compare two or more teams to get the probabilities of the match drawing.
    0.09025541153402594
 
 
+Predicting Ranks
+----------------
+
+.. code:: python
+
+   >>> from openskill import predict_rank, predict_draw
+   >>> a1 = a2 = a3 = Rating(mu=34, sigma=0.25)
+   >>> b1 = b2 = b3 = Rating(mu=32, sigma=0.5)
+   >>> c1 = c2 = c3 = Rating(mu=30, sigma=1)
+   >>> team_1, team_2, team_3 = [a1, a2, a3], [b1, b2, b3], [c1, c2, c3]
+   >>> draw_probability = predict_draw(teams=[team_1, team_2, team_3])
+   >>> draw_probability
+   0.3295385074666581
+   >>> rank_probability = predict_rank(teams=[team_1, team_2, team_3])
+   >>> rank_probability
+   [(1, 0.4450361350569973), (2, 0.19655022513040032), (3, 0.028875132345944337)]
+   >>> sum([y for x, y in rank_probability]) + draw_probability
+   1.0
+
+
 Choosing Models
 ---------------
 

docs/manual.rst

@@ -3,6 +3,7 @@
     create_rating,
     ordinal,
     predict_draw,
+    predict_rank,
     predict_win,
     rate,
     team_rating,

openskill/__init__.py

@@ -2,7 +2,9 @@
 import itertools
 import math
 from functools import reduce
-from typing import List, Optional, Union
+from typing import List, Optional, Tuple, Union
+
+from scipy.stats import rankdata
 
 from openskill.constants import Constants, beta
 from openskill.constants import mu as default_mu
@@ -412,3 +414,55 @@ def predict_draw(teams: List[List[Rating]], **options) -> Union[int, float]:
         denom = n * (n - 1)
 
     return abs(sum(pairwise_probabilities)) / denom
+
+
+def predict_rank(
+    teams: List[List[Rating]], **options
+) -> List[Tuple[int, Union[int, float]]]:
+    """
+    Predict the shape of a match outcome.
+    This algorithm has a time complexity of O(n!/(n - 2)!) where 'n' is the number of teams.
+
+    :param teams: A list of two or more teams, where teams are lists of :class:`~openskill.rate.Rating` objects.
+    :return: A list of team ranks with their probabilities.
+    """
+    if len(teams) < 2:
+        raise ValueError(f"Expected at least two teams.")
+
+    n = len(teams)
+    total_player_count = sum([len(_) for _ in teams])
+    denom = (n * (n - 1)) / 2
+    draw_probability = 1 / n
+    draw_margin = (
+        math.sqrt(total_player_count)
+        * beta(**options)
+        * phi_major_inverse((1 + draw_probability) / 2)
+    )
+
+    pairwise_probabilities = []
+    for pairwise_subset in itertools.permutations(teams, 2):
+        current_team_a_rating = team_rating([pairwise_subset[0]])
+        current_team_b_rating = team_rating([pairwise_subset[1]])
+        mu_a = current_team_a_rating[0][0]
+        sigma_a = current_team_a_rating[0][1]
+        mu_b = current_team_b_rating[0][0]
+        sigma_b = current_team_b_rating[0][1]
+        pairwise_probabilities.append(
+            phi_major(
+                (mu_a - mu_b - draw_margin)
+                / math.sqrt(n * beta(**options) ** 2 + sigma_a**2 + sigma_b**2)
+            )
+        )
+    win_probability = [
+        (sum(team_prob) / denom)
+        for team_prob in itertools.zip_longest(
+            *[iter(pairwise_probabilities)] * (n - 1)
+        )
+    ]
+
+    ranked_probability = [abs(_) for _ in win_probability]
+    ranks = list(rankdata(ranked_probability, method="min"))
+    max_ordinal = max(ranks)
+    ranks = [abs(_ - max_ordinal) + 1 for _ in ranks]
+    predictions = list(zip(ranks, ranked_probability))
+    return predictions

openskill/rate.py

@@ -0,0 +1,26 @@
+import pytest
+
+from openskill import Rating
+from openskill.rate import predict_draw, predict_rank
+
+
+def test_predict_rank():
+    a1 = Rating(mu=34, sigma=0.25)
+    a2 = Rating(mu=32, sigma=0.25)
+    a3 = Rating(mu=34, sigma=0.25)
+
+    b1 = Rating(mu=24, sigma=0.5)
+    b2 = Rating(mu=22, sigma=0.5)
+    b3 = Rating(mu=20, sigma=0.5)
+
+    team_1 = [a1, b1]
+    team_2 = [a2, b2]
+    team_3 = [a3, b3]
+
+    ranks = predict_rank(teams=[team_1, team_2, team_3])
+    total_rank_probability = sum([y for x, y in ranks])
+    draw_probability = predict_draw(teams=[team_1, team_2, team_3])
+    assert total_rank_probability + draw_probability == pytest.approx(1)
+
+    with pytest.raises(ValueError):
+        predict_rank(teams=[team_1])