fix: add more fields to AniDB characters

Added parsing of the character type (character, organization, etc.) and last updated timestamp for AniDB characters. The first one will allow us to distinguish between characters and organizations, while the latter will allow us to do conditional updates of the data found in the xml like we do for the episodes and tags, in case you ever attempt to refresh from a partially outdated xml where it has a character that's shared between multiple anime or whatnot. There is also user votes and user ratings to be found in the xmls we can add in the future, if the need ever arise. Also fixed it so organizations show up in APIv3. And last; the database migration for SQLite is tested, but for the two other databases it's not. So if somebody can test that they work, and report back on it then that would be great.
ShokoAnime · Jan 14, 2025 · 677b0e9 · 677b0e9
1 parent 0fb3ac3
commit 677b0e9
Show file tree

Hide file tree

Showing 12 changed files with 101 additions and 27 deletions.
diff --git a/Shoko.Server/API/v3/Models/Common/Role.cs b/Shoko.Server/API/v3/Models/Common/Role.cs
@@ -41,7 +41,7 @@ public class Role
     [Required]
     public string RoleDetails { get; set; } = string.Empty;
 
-    public Role(AniDB_Anime_Character xref, AniDB_Creator staff, AniDB_Character? character)
+    public Role(AniDB_Anime_Character xref, AniDB_Character character, AniDB_Creator? staff = null)
     {
         Character = character == null ? null : new()
         {
@@ -51,16 +51,27 @@ public Role(AniDB_Anime_Character xref, AniDB_Creator staff, AniDB_Character? ch
             Description = character.Description ?? string.Empty,
             Image = character.GetImageMetadata() is { } characterImage ? new Image(characterImage) : null,
         };
-        Staff = new()
-        {
-            ID = staff.CreatorID,
-            Name = staff.Name,
-            AlternateName = staff.OriginalName ?? string.Empty,
-            Description = string.Empty,
-            Image = staff.GetImageMetadata() is { } staffImage ? new Image(staffImage) : null,
-        };
+        Staff = staff is not null
+            ? new()
+            {
+                ID = staff.CreatorID,
+                Name = staff.Name,
+                AlternateName = staff.OriginalName ?? string.Empty,
+                Description = string.Empty,
+                Image = staff.GetImageMetadata() is { } staffImage ? new Image(staffImage) : null,
+            }
+            : new()
+            {
+                ID = 0,
+                Name = string.Empty,
+                AlternateName = string.Empty,
+                Description = string.Empty,
+                Image = null,
+            };
         RoleName = CreatorRoleType.Actor;
-        RoleDetails = xref.AppearanceType.ToString().Replace("_", " ");
+        RoleDetails = staff is not null
+            ? xref.AppearanceType.ToString().Replace("_", " ")
+            : "Appears In";
     }
 
     public Role(AniDB_Anime_Staff xref, AniDB_Creator staff)

diff --git a/Shoko.Server/API/v3/Models/Shoko/Series.cs b/Shoko.Server/API/v3/Models/Shoko/Series.cs
@@ -263,14 +263,14 @@ public static List<Role> GetCast(int animeID, HashSet<CreatorRoleType>? roleType
             var characterXrefs = RepoFactory.AniDB_Anime_Character.GetByAnimeID(animeID);
             foreach (var xref in characterXrefs.OrderBy(x => x.Ordering))
             {
-                if (xref.Creators is not { Count: > 0 } creators)
-                    continue;
-
                 if (xref.Character is not { } character)
                     continue;
 
-                foreach (var creator in creators)
-                    roles.Add(new(xref, creator, character));
+                if (character.Type is CharacterType.Organization)
+                    roles.Add(new(xref, character));
+                else
+                    foreach (var creator in xref.Creators)
+                        roles.Add(new(xref, character, creator));
             }
         }
 

diff --git a/Shoko.Server/Databases/DatabaseFixes.cs b/Shoko.Server/Databases/DatabaseFixes.cs
@@ -775,8 +775,16 @@ public static void RepairMissingTMDBPersons()
             missingIds.Count, updateCount, skippedCount);
     }
 
+    private static bool _ranRecreateAnimeCharactersAndCreators = false;
+
     public static void RecreateAnimeCharactersAndCreators()
     {
+        // Hack to prevent running the migration twice in the same lifecycle,
+        // since it is scheduled to run for two migrations one after another,
+        // but if we run them back to back then we only need to run it once.
+        if (_ranRecreateAnimeCharactersAndCreators) return;
+        _ranRecreateAnimeCharactersAndCreators = true;
+
         var xmlUtils = Utils.ServiceContainer.GetRequiredService<HttpXmlUtils>();
         var animeParser = Utils.ServiceContainer.GetRequiredService<HttpAnimeParser>();
         var animeCreator = Utils.ServiceContainer.GetRequiredService<AnimeCreator>();

diff --git a/Shoko.Server/Databases/MySQL.cs b/Shoko.Server/Databases/MySQL.cs
@@ -27,7 +27,7 @@ namespace Shoko.Server.Databases;
 public class MySQL : BaseDatabase<MySqlConnection>
 {
     public override string Name { get; } = "MySQL";
-    public override int RequiredVersion { get; } = 147;
+    public override int RequiredVersion { get; } = 148;
 
     private List<DatabaseCommand> createVersionTable = new()
     {
@@ -921,6 +921,9 @@ public class MySQL : BaseDatabase<MySqlConnection>
         new(147, 10, "CREATE TABLE `AniDB_Anime_Character_Creator` (`AniDB_Anime_Character_CreatorID` INT NOT NULL AUTO_INCREMENT, `AnimeID` INT NOT NULL, `CharacterID` INT NOT NULL, `CreatorID` INT NOT NULL, `Ordering` INT NOT NULL, PRIMARY KEY (`AniDB_Anime_Character_CreatorID`));"),
         new(147, 11, "CREATE INDEX IX_AniDB_Anime_Staff_CreatorID ON AniDB_Anime_Staff(CreatorID);"),
         new(147, 12, DatabaseFixes.RecreateAnimeCharactersAndCreators),
+        new(148, 01, "ALTER TABLE `AniDB_Character` ADD `Type` int NOT NULL DEFAULT 0;"),
+        new(148, 02, "ALTER TABLE `AniDB_Character` ADD `LastUpdated` datetime NOT NULL DEFAULT '1970-01-01 00:00:00';"),
+        new(148, 03, DatabaseFixes.RecreateAnimeCharactersAndCreators),
     };
 
     private DatabaseCommand linuxTableVersionsFix = new("RENAME TABLE versions TO Versions;");

diff --git a/Shoko.Server/Databases/SQLServer.cs b/Shoko.Server/Databases/SQLServer.cs
@@ -28,7 +28,7 @@ namespace Shoko.Server.Databases;
 public class SQLServer : BaseDatabase<SqlConnection>
 {
     public override string Name { get; } = "SQLServer";
-    public override int RequiredVersion { get; } = 140;
+    public override int RequiredVersion { get; } = 141;
 
     public override void BackupDatabase(string fullfilename)
     {
@@ -870,6 +870,9 @@ public override bool HasVersionsTable()
         new DatabaseCommand(140, 10, "CREATE TABLE AniDB_Anime_Character_Creator (AniDB_Anime_Character_CreatorID INT IDENTITY(1,1), AnimeID INT NOT NULL, CharacterID INT NOT NULL, CreatorID INT NOT NULL, Ordering INT NOT NULL);"),
         new DatabaseCommand(140, 11, "CREATE INDEX IX_AniDB_Anime_Staff_CreatorID ON AniDB_Anime_Staff(CreatorID);"),
         new DatabaseCommand(140, 12, DatabaseFixes.RecreateAnimeCharactersAndCreators),
+        new DatabaseCommand(141, 01, "ALTER TABLE AniDB_Character ADD Type int NOT NULL DEFAULT 0;"),
+        new DatabaseCommand(141, 02, "ALTER TABLE AniDB_Character ADD LastUpdated datetime2 NOT NULL DEFAULT '1970-01-01 00:00:00';"),
+        new DatabaseCommand(141, 03, DatabaseFixes.RecreateAnimeCharactersAndCreators),
     };
 
     private static void AlterImdbMovieIDType()

diff --git a/Shoko.Server/Databases/SQLite.cs b/Shoko.Server/Databases/SQLite.cs
@@ -28,7 +28,7 @@ public class SQLite : BaseDatabase<SqliteConnection>
 {
     public override string Name => "SQLite";
 
-    public override int RequiredVersion => 130;
+    public override int RequiredVersion => 131;
 
     public override void BackupDatabase(string fullfilename)
     {
@@ -842,6 +842,9 @@ public override void CreateDatabase()
         new(130, 10, "CREATE TABLE AniDB_Anime_Character_Creator (AniDB_Anime_Character_CreatorID INTEGER PRIMARY KEY AUTOINCREMENT, AnimeID INTEGER NOT NULL, CharacterID INTEGER NOT NULL, CreatorID INTEGER NOT NULL, Ordering INTEGER NOT NULL);"),
         new(130, 11, "CREATE INDEX IX_AniDB_Anime_Staff_CreatorID ON AniDB_Anime_Staff(CreatorID);"),
         new(130, 12, DatabaseFixes.RecreateAnimeCharactersAndCreators),
+        new(131, 01, "ALTER TABLE AniDB_Character ADD COLUMN Type INTEGER NOT NULL DEFAULT 0;"),
+        new(131, 02, "ALTER TABLE AniDB_Character ADD COLUMN LastUpdated DATETIME NOT NULL DEFAULT '1970-01-01 00:00:00';"),
+        new(131, 03, DatabaseFixes.RecreateAnimeCharactersAndCreators),
     };
 
     private static Tuple<bool, string> MigrateRenamers(object connection)

diff --git a/Shoko.Server/Mappings/AniDB_CharacterMap.cs b/Shoko.Server/Mappings/AniDB_CharacterMap.cs
@@ -1,6 +1,7 @@
 using FluentNHibernate.Mapping;
 using Shoko.Server.Models.AniDB;
 using Shoko.Server.Providers.TMDB;
+using Shoko.Server.Server;
 
 namespace Shoko.Server.Mappings;
 
@@ -18,5 +19,7 @@ public AniDB_CharacterMap()
         Map(x => x.OriginalName).Not.Nullable();
         Map(x => x.Name).Not.Nullable();
         Map(x => x.Gender).CustomType<PersonGender>().Not.Nullable();
+        Map(x => x.Type).CustomType<CharacterType>().Not.Nullable();
+        Map(x => x.LastUpdated).Not.Nullable();
     }
 }
diff --git a/Shoko.Server/Models/AniDB/AniDB_Character.cs b/Shoko.Server/Models/AniDB/AniDB_Character.cs
@@ -1,7 +1,8 @@
-
-#nullable enable
+using System;
 using Shoko.Server.Providers.TMDB;
+using Shoko.Server.Server;
 
+#nullable enable
 namespace Shoko.Server.Models.AniDB;
 
 public class AniDB_Character
@@ -22,5 +23,9 @@ public class AniDB_Character
 
     public PersonGender Gender { get; set; }
 
+    public CharacterType Type { get; set; }
+
+    public DateTime LastUpdated { get; set; }
+
     #endregion
 }
diff --git a/Shoko.Server/Providers/AniDB/HTTP/AnimeCreator.cs b/Shoko.Server/Providers/AniDB/HTTP/AnimeCreator.cs
@@ -815,14 +815,19 @@ public void CreateCharacters(List<ResponseCharacter> chars, SVR_AniDB_Anime anim
             foreach (var (rawCharacter, _) in charLookup)
             {
                 var characterIndex = characterOrdering++;
-                if (rawCharacter.AnimeID != anime.AnimeID || rawCharacter.CharacterID <= 0 || string.IsNullOrEmpty(rawCharacter.CharacterType))
+                if (rawCharacter.AnimeID != anime.AnimeID || rawCharacter.CharacterID <= 0 || string.IsNullOrEmpty(rawCharacter.CharacterAppearanceType))
                     continue;
 
                 var gender = rawCharacter.Gender switch
                 {
                     null => PersonGender.Unknown,
                     _ => Enum.TryParse<PersonGender>(rawCharacter.Gender, true, out var result) ? result : PersonGender.Unknown
                 };
+                var characterType = rawCharacter.CharacterType switch
+                {
+                    null => CharacterType.Unknown,
+                    _ => Enum.TryParse<CharacterType>(rawCharacter.CharacterType, true, out var result) ? result : CharacterType.Unknown
+                };
                 var character = RepoFactory.AniDB_Character.GetByCharacterID(rawCharacter.CharacterID) ?? new()
                 {
                     CharacterID = rawCharacter.CharacterID,
@@ -837,9 +842,10 @@ public void CreateCharacters(List<ResponseCharacter> chars, SVR_AniDB_Anime anim
                     character.Name = rawCharacter.CharacterName;
                     character.ImagePath = rawCharacter.PicName ?? string.Empty;
                     character.Gender = gender;
+                    character.Type = characterType;
                     charactersToSave.Add(character);
                 }
-                else
+                else if (rawCharacter.LastUpdated >= character.LastUpdated)
                 {
                     if (string.IsNullOrEmpty(rawCharacter?.CharacterName)) continue;
 
@@ -869,12 +875,26 @@ public void CreateCharacters(List<ResponseCharacter> chars, SVR_AniDB_Anime anim
                         character.Gender = gender;
                         updated = true;
                     }
+                    if (character.Type != characterType)
+                    {
+                        character.Type = characterType;
+                        updated = true;
+                    }
+                    if (character.LastUpdated != rawCharacter.LastUpdated)
+                    {
+                        character.LastUpdated = rawCharacter.LastUpdated;
+                        updated = true;
+                    }
                     if (updated)
                         charactersToSave.Add(character);
                     charactersToKeep.Add(character.AniDB_CharacterID);
                 }
+                else
+                {
+                    charactersToKeep.Add(character.AniDB_CharacterID);
+                }
 
-                var appearance = rawCharacter.CharacterType;
+                var appearance = rawCharacter.CharacterAppearanceType;
                 var appearanceType = appearance switch
                 {
                     "main character in" => CharacterAppearanceType.Main_Character,

diff --git a/Shoko.Server/Providers/AniDB/HTTP/GetAnime/ResponseCharacter.cs b/Shoko.Server/Providers/AniDB/HTTP/GetAnime/ResponseCharacter.cs
@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 
 namespace Shoko.Server.Providers.AniDB.HTTP.GetAnime;
 
@@ -10,7 +11,9 @@ public class ResponseCharacter
     public string CharacterName { get; set; }
     public string CharacterKanjiName { get; set; }
     public string CharacterDescription { get; set; }
+    public string CharacterAppearanceType { get; set; }
     public string CharacterType { get; set; }
     public string Gender { get; set; }
     public List<ResponseSeiyuu> Seiyuus { get; set; }
+    public DateTime LastUpdated { get; set; }
 }
diff --git a/Shoko.Server/Providers/AniDB/HTTP/HttpAnimeParser.cs b/Shoko.Server/Providers/AniDB/HTTP/HttpAnimeParser.cs
@@ -599,11 +599,14 @@ private static ResponseCharacter ParseCharacter(int animeID, XmlNode node)
             return null;
         }
 
-        var charType = TryGetAttribute(node, "type");
+        var characterType = TryGetProperty(node, "charactertype") ?? "Character";
+        var characterAppearanceType = TryGetAttribute(node, "type");
         var charName = TryGetProperty(node, "name")?.Replace('`', '\'');
         var charGender = TryGetProperty(node, "gender")?.Replace('`', '\'');
         var charDescription = TryGetProperty(node, "description")?.Replace('`', '\'');
         var picName = TryGetProperty(node, "picture");
+        if (!DateTime.TryParse(TryGetAttribute(node, "update"), out var lastUpdated))
+            lastUpdated = DateTime.UnixEpoch;
 
         // parse seiyuus
         var seiyuus = new List<ResponseSeiyuu>();
@@ -628,12 +631,14 @@ private static ResponseCharacter ParseCharacter(int animeID, XmlNode node)
         {
             AnimeID = animeID,
             CharacterID = charID,
-            CharacterType = charType,
+            CharacterAppearanceType = characterAppearanceType,
+            CharacterType = characterType,
             CharacterName = charName,
             CharacterDescription = charDescription,
             PicName = picName,
             Gender = charGender,
-            Seiyuus = seiyuus
+            Seiyuus = seiyuus,
+            LastUpdated = lastUpdated,
         };
     }
 

diff --git a/Shoko.Server/Server/Enums.cs b/Shoko.Server/Server/Enums.cs
@@ -140,6 +140,7 @@ public enum CreatorRoleType
     SourceWork,
 }
 
+[JsonConverter(typeof(StringEnumConverter))]
 public enum CharacterAppearanceType
 {
     Unknown = 0,
@@ -148,3 +149,12 @@ public enum CharacterAppearanceType
     Background_Character,
     Cameo
 }
+
+[JsonConverter(typeof(StringEnumConverter))]
+public enum CharacterType
+{
+    Unknown = 0,
+    Character = 1,
+    // ??? = 2,
+    Organization = 3,
+}