Slim down site and fix many database query issues

backend/blueprints/spa_api/service_layers/homepage/recent.py

@@ -1,8 +1,11 @@
+import json
+
 from sqlalchemy import desc
 
 from backend.blueprints.spa_api.service_layers.replay.replay import CompactReplay
 from backend.blueprints.spa_api.service_layers.utils import with_session
 from backend.database.objects import Game
+from backend.database.startup import lazy_get_redis
 
 
 class RecentReplays:
@@ -17,8 +20,14 @@ def create(cls):
     @staticmethod
     @with_session
     def get_recent_replays(session=None):
-        replays = session.query(Game).order_by(desc(Game.upload_date))[:5][::-1]
-        return [CompactReplay.create_from_game(r).__dict__ for r in replays]
+        r = lazy_get_redis()
+        if r is not None and r.get('recent_replays') is not None:
+            return json.loads(r.get('recent_replays'))
+        replays = session.query(Game).order_by(desc(Game.upload_date))[:5]
+        replays = [CompactReplay.create_from_game(r).__dict__ for r in replays]
+        if r is not None:
+            r.set('recent_replays', json.dumps(replays), ex=60 * 60)
+        return replays
 
 
 if __name__ == '__main__':

backend/blueprints/spa_api/service_layers/ml/ml.py

@@ -1,12 +1,13 @@
 import glob
 import os
 
+import numpy as np
+import pandas as pd
 import torch
 import torch.nn as nn
-import pandas as pd
+from sqlalchemy import inspect
 
 from backend.database.objects import PlayerGame
-from sqlalchemy import inspect
 
 
 def object_as_dict(obj):
@@ -40,18 +41,25 @@ class RankPredictor:
     # MODEL_DIR = os.path.abspath(os.path.join('..', '..', '..', '..', '..', 'data', 'models'))
     MODEL_DIR = os.path.abspath(os.path.join('data', 'models'))
     models = {}
-    maxs = None
-    mins = None
+    maxs = {}
+    mins = {}
 
     def __init__(self):
-        # Load models
-        for model in sorted(glob.glob(os.path.join(self.MODEL_DIR, '*.mdl'))):
-            state = torch.load(model, map_location='cpu')
-            m = RankPredictorEnsemble()
-            m.load_state_dict(state)
-            self.models[os.path.basename(model).split('.')[0]] = m
-        self.maxs = pd.read_csv(os.path.join(self.MODEL_DIR, 'maxs.csv'), index_col=0, squeeze=True, header=None)
-        self.mins = pd.read_csv(os.path.join(self.MODEL_DIR, 'mins.csv'), index_col=0, squeeze=True, header=None)
+        possible_playlists = [playlist for playlist in os.listdir(self.MODEL_DIR) if
+                              os.path.isdir(os.path.join(self.MODEL_DIR, playlist))]
+        for playlist_dir in possible_playlists:
+            self.models[playlist_dir] = {}
+            playlist_loc = os.path.join(self.MODEL_DIR, playlist_dir)
+            # Load models
+            for model in sorted(glob.glob(os.path.join(playlist_loc, '*.mdl'))):
+                state = torch.load(model, map_location='cpu')
+                m = RankPredictorEnsemble()
+                m.load_state_dict(state)
+                self.models[playlist_dir][os.path.basename(model).split('.')[0]] = m
+            self.maxs[playlist_dir] = pd.read_csv(os.path.join(playlist_loc, 'maxs.csv'), index_col=0, squeeze=True,
+                                                  header=None)
+            self.mins[playlist_dir] = pd.read_csv(os.path.join(playlist_loc, 'mins.csv'), index_col=0, squeeze=True,
+                                                  header=None)
 
     @staticmethod
     def process_input_data(input_, maxs, mins):
@@ -73,31 +81,37 @@ def process_input_data(input_, maxs, mins):
         #     mins = input.min()
         #     maxs = input.max()
         input = (input - mins) / (maxs - mins)
+        input = input.fillna(0.0)
+        input = input.replace([np.inf, -np.inf], 0.0)
         output = nonzero['rank']
         return input, output, maxs, mins
 
-    def convert_sql_object_to_numpy(self, obj: PlayerGame):
+    def convert_sql_object_to_numpy(self, obj: PlayerGame, playlist: int):
+        playlist = str(playlist)
         obj = object_as_dict(obj)
         a = pd.Series(obj)
-        input, _, __, ___ = self.process_input_data(a, self.maxs, self.mins)
+        input, _, __, ___ = self.process_input_data(a, self.maxs[playlist], self.mins[playlist])
         obj = input.values
         return obj
 
-    def predict_rank(self, x: PlayerGame) -> int:
-        x = self.convert_sql_object_to_numpy(x)
+    def predict_rank(self, x: PlayerGame, playlist: int) -> int:
+        print(playlist)
+        x = self.convert_sql_object_to_numpy(x, playlist)
         result = pd.DataFrame(index=list(range(len(x))))
-        for rank, m in self.models.items():
+        if str(playlist) not in self.models:
+            return None
+        models = self.models[str(playlist)]
+        for rank, m in models.items():
             result = result.merge(
                 pd.Series(100 * m(x.astype(float)).detach().cpu().numpy(),
                           name=str(rank)).to_frame(), left_index=True, right_index=True)
-        # print(result)
+        print(result)
         result = (result > 50).apply(lambda x: int(x.idxmin()), axis=1).values
         return int(result[0])
 
 
 model_holder = RankPredictor()
 
-
 if __name__ == '__main__':
     from backend.database.startup import lazy_startup
 

backend/blueprints/spa_api/service_layers/player/player_profile_stats.py

@@ -1,14 +1,14 @@
 from typing import List
 
-from sqlalchemy import func, desc, cast, String, literal_column
+from sqlalchemy import func, desc, cast, String
 from sqlalchemy.dialects import postgresql
 
 from backend.blueprints.spa_api.service_layers.replay.replay_player import Loadout
 from backend.blueprints.spa_api.service_layers.utils import with_session
+from backend.data.constants.car import get_car
 from backend.database.objects import PlayerGame, Game, Player
 from backend.database.wrapper.player_wrapper import PlayerWrapper
 from backend.database.wrapper.stats.player_stat_wrapper import PlayerStatWrapper
-from backend.data.constants.car import get_car
 
 player_wrapper = PlayerWrapper(limit=10)
 player_stat_wrapper = PlayerStatWrapper(player_wrapper)
@@ -48,15 +48,13 @@ def _get_most_played_with(id_: str, session):
         result = session.query(p,
                                func.count(Game.players).label('count')).filter(
             Game.players.contains(cast([id_],
-                                       postgresql.ARRAY(String)))).group_by('player').order_by(desc('count')).subquery(
-            't')
-        result = session.query(result, Player.platformname).join(Player,
-                                                                 Player.platformid == result.c.player).filter(
-            Player.platformid != id_).filter(literal_column('count') > 1)[:3]
+                                       postgresql.ARRAY(String)))).group_by('player').order_by(desc('count'))
+        result = result[1:4]
         for p in result:
             player = session.query(Player).filter(Player.platformid == p[0]).first()
             if player is None or player.platformname == "":
-                players_in_common.append(PlayerInCommonStats(name=p[2], count=p[1], id=p[0], avatar=player.avatar))
+                print("unknown player")
+                players_in_common.append(PlayerInCommonStats(name="Unknown", count=p[1], id=p[0], avatar=player.avatar))
             else:
                 players_in_common.append(
                     PlayerInCommonStats(name=player.platformname, count=p[1], id=p[0], avatar=player.avatar))
@@ -86,7 +84,6 @@ def _get_favourite_car(id_: str, session):
     def _get_most_recent_loadout(id_: str, session):
         pg = session.query(PlayerGame) \
             .join(Game, PlayerGame.game == Game.hash) \
-            .distinct(PlayerGame.player) \
             .filter(PlayerGame.player == id_) \
             .order_by(desc(PlayerGame.player), desc(Game.match_date)) \
             .first()

backend/blueprints/spa_api/service_layers/replay/predicted_ranks.py

@@ -9,7 +9,8 @@
     from backend.blueprints.spa_api.service_layers.ml.ml import model_holder
 except ModuleNotFoundError:
     model_holder = None
-    print("Not using ML because required packages are not installed. Run `pip install -r requirements-ml.txt` to use ML.")
+    print(
+        "Not using ML because required packages are not installed. Run `pip install -r requirements-ml.txt` to use ML.")
 
 
 class PredictedRank:
@@ -33,13 +34,38 @@ def create_from_id(id_: str, session=None) -> List['PredictedRank']:
         accepted_playlists = [
             13,  # standard
             3,  # unranked standard
-            6,  # custom
+            6,  # custom,
+            11,  # doubles
+            2,  # unranked doubles
+            10,  # ranked duels
+            1,  # unranked duels
+            27,  # hoops
+            28,  # rumble
+            29,  # dropshot
+            30,  # snow day
         ]
         if game.playlist not in accepted_playlists:
             raise UnsupportedPlaylist
 
         playergames = session.query(PlayerGame).filter(PlayerGame.game == id_).all()
-
+        adjusted_playlist_map = {
+            2: 11,
+            11: 11,
+            3: 13,
+            6: 13,
+            13: 13,
+            10: 10,
+            1: 10,
+            27: 27,
+            28: 28,
+            29: 29,
+            30: 30
+        }
+        playlist = adjusted_playlist_map[game.playlist]
+        if game.playlist == 6 and len(game.players) == 4:
+            playlist = 11
+        elif game.playlist == 6 and len(game.players) == 2:
+            playlist = 10
         if is_local_dev():
             import random
             ranks = [
@@ -48,7 +74,7 @@ def create_from_id(id_: str, session=None) -> List['PredictedRank']:
             ]
         else:
             ranks = [
-                PredictedRank(pg.player, model_holder.predict_rank(pg))
+                PredictedRank(pg.player, model_holder.predict_rank(pg, playlist=playlist))
                 for pg in playergames
             ]
         return ranks

backend/blueprints/spa_api/service_layers/replay/replay.py

@@ -1,11 +1,11 @@
-from typing import List, cast
+from typing import List, cast, Dict
 
 from backend.blueprints.spa_api.errors.errors import ReplayNotFound, Redirect
 from backend.blueprints.spa_api.service_layers.replay.json_tag import JsonTag
 from backend.blueprints.spa_api.service_layers.replay.replay_player import ReplayPlayer
 from backend.blueprints.spa_api.service_layers.utils import sort_player_games_by_team_then_id, with_session
 from backend.data.constants.playlist import get_playlist
-from backend.database.objects import Game, PlayerGame, GameVisibilitySetting
+from backend.database.objects import Game, PlayerGame, GameVisibilitySetting, Player
 from backend.utils.safe_flask_globals import get_current_user_id
 
 
@@ -23,7 +23,7 @@ class Replay:
     def __init__(self, id_: str, name: str, date: str, map: str,
                  game_mode: str, game_score: GameScore,
                  players: List[ReplayPlayer], tags: List[JsonTag], visibility: GameVisibilitySetting,
-                 ranks: List[int], mmrs: List[int]):
+                 ranks: List[int], mmrs: List[int], group_map: Dict[str, List[int]]):
         self.id = id_
         self.name = name
         self.date = date
@@ -35,6 +35,7 @@ def __init__(self, id_: str, name: str, date: str, map: str,
         self.visibility = visibility.value
         self.ranks = ranks
         self.mmrs = mmrs
+        self.groupMap = group_map
 
     @staticmethod
     @with_session
@@ -45,11 +46,15 @@ def create_from_id(id_: str, session=None) -> 'Replay':
             if game is None:
                 raise ReplayNotFound()
             raise Redirect("/replays/" + game.hash)
-        replay = Replay.create_from_game(game)
+        replay = Replay.create_from_game(game, session=session, groups=True)
         return replay
 
     @staticmethod
-    def create_from_game(game: Game, full=True) -> 'Replay':
+    def create_from_game(game: Game, full=True, groups=False, session=None) -> 'Replay':
+        group_map = None
+        if groups and session is not None:
+            players = session.query(Player).filter(Player.platformid.in_(game.players)).all()
+            group_map = {player.platformid: player.groups for player in players}
         return Replay(
             id_=game.hash,
             name=game.name,
@@ -68,8 +73,8 @@ def create_from_game(game: Game, full=True) -> 'Replay':
             ],
             visibility=game.visibility,
             ranks=game.ranks,
-            mmrs=game.mmrs
-
+            mmrs=game.mmrs,
+            group_map=group_map
         )
 
 

backend/blueprints/spa_api/spa_api.py

@@ -131,7 +131,12 @@ def encode_bot_name(w):
 @bp.route('/global/replay_count')
 @with_session
 def api_get_replay_count(session=None):
+    r = lazy_get_redis()
+    if r is not None and r.get('replay_count') is not None:
+        return jsonify(int(r.get('replay_count')))
     count = session.query(Game.hash).count()
+    if r is not None:
+        r.set('replay_count', str(count), ex=60 * 60)
     return jsonify(count)
 
 

backend/database/objects.py

@@ -315,7 +315,7 @@ def validate_code(self, key, value):
 
 class Player(DBObjectBase):
     __tablename__ = 'players'
-    platformid = Column(String(40), primary_key=True)
+    platformid = Column(String(40), primary_key=True, unique=True)
     platformname = Column(String(50))
     avatar = Column(String(150))
     ranks = Column(postgresql.ARRAY(Integer, dimensions=1))  # foreign key

backend/database/startup.py

@@ -10,12 +10,22 @@
 
 from backend.database.objects import DBObjectBase
 
+try:
+    import config
+    DB_IP = config.DB_IP
+    DB_USER = config.DB_USER
+    DB_PASSWORD = config.DB_PASSWORD
+except:
+    DB_IP = None
+    DB_USER = None
+    DB_PASSWORD = None
+
 logger = logging.getLogger(__name__)
 
 
 def login(connection_string, recreate_database=False) -> Tuple[create_engine, sessionmaker]:
     print(connection_string)
-    engine = create_engine(connection_string, echo=False)
+    engine = create_engine(connection_string, echo=False, pool_recycle=1800)
     if recreate_database:
         conn = engine.connect()
         conn.execute("commit")
@@ -32,20 +42,6 @@ def login(connection_string, recreate_database=False) -> Tuple[create_engine, se
     return engine, session
 
 
-def startup() -> sessionmaker:
-    try:
-        # Sql Stuff
-        connection_string = 'postgresql:///saltie'
-        engine, session = login(connection_string)
-    except OperationalError as e:
-        print('trying backup info', e)
-        try:
-            engine, session = login('postgresql://postgres:postgres@localhost/saltie')
-        except Exception as e:
-            engine, session = login('postgresql://postgres:postgres@localhost', recreate_database=True)
-    return session
-
-
 def get_current_session():
     return EngineStartup.get_current_session()
 
@@ -81,6 +77,10 @@ def get_strict_redis():
 class EngineStartup:
     @staticmethod
     def login_db() -> Tuple[any, sessionmaker]:
+        if DB_IP is not None:
+            connection_string = f'postgresql://{DB_USER}:{DB_PASSWORD}@{DB_IP}/saltie'
+            engine, session = login(connection_string)
+            return engine, session
         try:
             # Sql Stuff
             connection_string = 'postgresql:///saltie'
@@ -90,6 +90,7 @@ def login_db() -> Tuple[any, sessionmaker]:
             try:
                 engine, session = login('postgresql://postgres:postgres@localhost/saltie')
             except Exception as e:
+                print(e)
                 engine, session = login('postgresql://postgres:postgres@localhost', recreate_database=True)
         return engine, session
 
@@ -118,6 +119,7 @@ def get_strict_redis():
     def get_current_session():
         try:
             return current_app.config['db']()
-        except:
+        except Exception as e:
+            print("Error getting session", e)
             _session = lazy_startup()
             return _session()