1.5.4 <=> Fixed PlexDB schema change crash

CHANGELOG.md

@@ -1,3 +1,11 @@
+## 1.5.4
+- Removed spammy messages in resolvement log
+- Better handling for mitigation system
+- Added mitigation for Plex ICU database change -> **USE ON YOUR OWN RISK UNTIL TESTED FIERCELY BY THE COMMUNITY!**
+- Mitigation works by removing two database triggers before updating the ratings and adding them immediatly after again to bypass an SQLite crash due to no ICU extension included in the sqlite-jdbc build
+- Bumped id error resolvement caching up to 30 days as most resolvement errors are 404s of dead shows anyways
+- Fixed spammy "cached new movie agent xxx id message" that should not appear once an id has been cached
+
 ## 1.5.3
 - One time mitigation added to combat cache name typo fix via cache reset
 

VERSION

@@ -1 +1 @@
-1.5.3
+1.5.4

build.gradle

@@ -5,7 +5,7 @@ plugins {
     id 'com.github.spotbugs' version '2.0.1'
 }
 
-version = '1.5.3'
+version = '1.5.4'
 sourceCompatibility = '11'
 
 new File(projectDir, "VERSION").text = version;

src/main/java/updatetool/Mitigations.java

@@ -8,20 +8,52 @@
 
 public final class Mitigations {
     private static final KeyValueStore MITIGATIONS = KeyValueStore.of(Main.PWD.resolve("mitigations.json"));
+    private static final int VERSION = Integer.parseInt(Main.VERSION.replaceAll("(\\.|[A-Za-z]|\\-)", ""));
 
     private Mitigations() {};
 
     public static void executeMitigations() {
         executeTypoSwitchCacheResetMitigation();
+        executeCacheParameterWrongOrderMitigation();
         MITIGATIONS.dump();
     }
 
+    private static void executeCacheParameterWrongOrderMitigation() {
+        String KEY = "executeCacheParameterWrongOrderMitigation";
+
+        if(MITIGATIONS.lookup(KEY) != null)
+            return;
+
+        if(VERSION > 154) {
+            MITIGATIONS.cache(KEY, "");
+            return;
+        }
+
+        Logger.info("One time mitigation executed: Switched caches in function parameter require conversion.");
+        Logger.info("This mitigation will only be executed once.");
+
+        var actualBlacklist = KeyValueStore.of(Main.PWD.resolve("cache-tvdb2imdb.json"));
+        var actualCache = KeyValueStore.of(Main.PWD.resolve("cache-tvdbBlacklist.json"));
+        actualCache.remove("__EXPIRE");
+
+        actualCache.withChangedPath(Main.PWD.resolve("cache-tvdb2imdb.json")).dump();
+        actualBlacklist.withChangedPath(Main.PWD.resolve("cache-tvdbBlacklist.json")).dump();
+
+        Logger.info("Mitigation completed!");
+        MITIGATIONS.cache(KEY, "");
+    }
+
     private static void executeTypoSwitchCacheResetMitigation() {
         String KEY = "executeTypoSwitchCacheResetMitigation";
 
         if(MITIGATIONS.lookup(KEY) != null)
             return;
 
+        if(VERSION > 151) {
+            MITIGATIONS.cache(KEY, "");
+            return;
+        }
+
         Logger.info("One time mitigation executed: Typo in cache names requires a full cache reset.");
         Logger.info("This mitigation will only be executed once.");
 

src/main/java/updatetool/common/KeyValueStore.java

@@ -45,8 +45,18 @@ public synchronized String lookup(String key) {
         return map.get(key);
     }
 
-    public synchronized void cache(String key, String value) {
-        map.put(key, value);
+    public synchronized boolean cache(String key, String value) {
+        return map.put(key, value) == null;
+    }
+
+    public synchronized boolean remove(String key) {
+        return map.remove(key) != null;
+    }
+
+    public synchronized KeyValueStore withChangedPath(Path p) {
+        var kv = new KeyValueStore(p);
+        kv.map.putAll(this.map);
+        return kv;
     }
 
     public synchronized void reset() {

src/main/java/updatetool/common/externalapis/TvdbApiV3.java

@@ -62,7 +62,7 @@ private String getImdbId() {
     private class Token { String token; };
 
     @SuppressFBWarnings("DM_EXIT")
-    public TvdbApiV3(String key, KeyValueStore blacklist, KeyValueStore cache, KeyValueStore cacheMovie, KeyValueStore blacklistMovie) throws ApiCallFailedException {
+    public TvdbApiV3(String key, KeyValueStore cache, KeyValueStore blacklist, KeyValueStore cacheMovie, KeyValueStore blacklistMovie) throws ApiCallFailedException {
         Logger.info("Testing TVDB API (v3) authorization apikey: {}", key);
 
         try {
@@ -79,7 +79,9 @@ public TvdbApiV3(String key, KeyValueStore blacklist, KeyValueStore cache, KeyVa
         this.blacklistMovie = blacklistMovie;
         this.cacheMovie = cacheMovie;
 
-        Converter<String, UnmarshalTvdb> converter = resp -> Objects.requireNonNull(gson.fromJson(resp.body(), UnmarshalTvdb.class));
+        Converter<String, UnmarshalTvdb> converter = resp -> {
+            return Objects.requireNonNull(gson.fromJson(resp.body(), UnmarshalTvdb.class));
+        };
 
         Handler<String, UnmarshalTvdb, ImdbMetadataResult> handler = (resp, res, payload) -> {
             if(res.Error != null) {

src/main/java/updatetool/imdb/ImdbDatabaseSupport.java

@@ -5,15 +5,18 @@
 import java.sql.Statement;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import org.sqlite.SQLiteException;
 import org.tinylog.Logger;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import updatetool.Globals;
+import updatetool.common.DatabaseSupport.LibraryType;
 import updatetool.common.KeyValueStore;
 import updatetool.common.SqliteDatabaseProvider;
 import updatetool.common.Utility;
-import updatetool.common.DatabaseSupport.LibraryType;
 
 public class ImdbDatabaseSupport {
     private final SqliteDatabaseProvider provider;
@@ -125,10 +128,11 @@ private void updateNewAgentMetadataMapping(ImdbMetadataResult m) throws SQLExcep
                     result = id;
             }
         }
-
+                
         if(result != null) {
-            newAgentMapping.cache(m.guid, result);
-            Logger.info("Associated and cached {} with new movie agent guid {} ({}).", result, m.guid, m.title);
+            if(newAgentMapping.cache(m.guid, result)) {
+                Logger.info("Associated and cached {} with new movie agent guid {} ({}).", result, m.guid, m.title);
+            }
         } else {
             Logger.warn("No external metadata provider id associated with this guid {} ({}). This item will not be processed any further.", m.guid, m.title);
         }
@@ -160,44 +164,129 @@ public void requestBatchUpdateOf(List<ImdbMetadataResult> items) throws SQLiteEx
         }
     }
 
+    private static final String ICU_MIG_1 = "fts4_metadata_titles_after_update_icu";
+    private static final String ICU_MIG_2 = "fts4_metadata_titles_before_update_icu";
+    private static final String ICU_MIG_1_SQL = "CREATE TRIGGER fts4_metadata_titles_after_update_icu AFTER UPDATE ON metadata_items BEGIN INSERT INTO fts4_metadata_titles_icu(docid, title, title_sort, original_title) VALUES(new.rowid, new.title, new.title_sort, new.original_title); END";
+    private static final String ICU_MIG_2_SQL = "CREATE TRIGGER fts4_metadata_titles_before_update_icu BEFORE UPDATE ON metadata_items BEGIN DELETE FROM fts4_metadata_titles_icu WHERE docid=old.rowid; END";
+
+    @SuppressFBWarnings({"DM_EXIT", "REC_CATCH_EXCEPTION", "OBL_UNSATISFIED_OBLIGATION", "DE_MIGHT_IGNORE"})
     private void internalBatchUpdate(List<ImdbMetadataResult> items, boolean isNewAgent) {
         boolean success = true;
-        try(var s = provider.connection.prepareStatement(isNewAgent ? "UPDATE metadata_items SET audience_rating = ?, extra_data = ?, rating = NULL WHERE id = ?" 
-                : "UPDATE metadata_items SET rating = ?, extra_data = ? WHERE id = ?")) {
-            for(var item : items) {
-                Double d = isNewAgent ? item.audienceRating : item.rating;
-
-                //TODO: hotfix, investigate further only happened to one person over the entire tool lifetime
-                if(d == null) {
-                    Logger.error("Null value encountered. Should not be possible. Skipping entry to not crash tool. Contact maintainer with this dump: " + Objects.toString(item));
-                    continue;
+        boolean mitigationIcuNeeded, mitigationIcuTriggersDisabled = false;
+        List<AutoCloseable> close = new ArrayList<>();
+
+        try {
+
+            // Mitigation
+            var mitigationIcu = provider.connection.createStatement();
+            close.add(mitigationIcu);
+            var rs = mitigationIcu.executeQuery("SELECT name, sql FROM sqlite_master WHERE type = 'trigger' AND name = '" + ICU_MIG_1 + "' OR name = '" + ICU_MIG_2 + "';");
+            Map<String, String> storedSql = new HashMap<>();
+            while(rs.next()) {
+                storedSql.put(rs.getString(1), rs.getString(2));
+            }
+            rs.close();
+
+            int count = storedSql.size();
+            if(!(count == 2 || count == 0)) {
+                throw new UnsupportedOperationException("@@!!! INCONSISTENT TRIGGER COUNT STATE !!!@@ - Parameters for ICU SQLite3 Mitigation have changed! Contact the author of this tool immediatly and create a github issue here and include the output of '.schema metadata_items' on your SQLite3 PlexDB: https://github.com/mynttt/UpdateTool/issues");
+            }
+
+            mitigationIcuNeeded = count == 2;
+            Logger.info("PlexDB ICU Mitigation enabled: {}", mitigationIcuNeeded);
+
+            if(mitigationIcuNeeded) {
+                if(!Objects.equals(ICU_MIG_1_SQL, storedSql.get(ICU_MIG_1)) || !Objects.equals(ICU_MIG_2_SQL, storedSql.get(ICU_MIG_2))) {
+                    Logger.error("IS  : {}",storedSql.get(ICU_MIG_1));
+                    Logger.error("MUST: {}",ICU_MIG_1_SQL);
+                    Logger.error("IS  : {}",storedSql.get(ICU_MIG_2));
+                    Logger.error("MUST: {}",ICU_MIG_2_SQL);
+                    throw new UnsupportedOperationException("@@!!! INCONSISTENT TRIGGER STATE !!!@@ - Parameters for ICU SQLite3 Mitigation have changed! Contact the author of this tool immediatly and create a github issue here and include the output of '.schema metadata_items' on your SQLite3 PlexDB: https://github.com/mynttt/UpdateTool/issues");
                 }
 
-                s.setDouble(1, d);
-                s.setString(2, item.extraData);
-                s.setInt(3, item.id);
-                s.addBatch();
+                var s1 = provider.connection.createStatement();
+                close.add(s1);
+                s1.executeUpdate("DROP TRIGGER fts4_metadata_titles_before_update_icu;");
+                var s2 = provider.connection.createStatement();
+                close.add(s2);
+                s2.executeUpdate("DROP TRIGGER fts4_metadata_titles_after_update_icu;");
+                mitigationIcuTriggersDisabled = true;
             }
-            int[] records = s.executeBatch();
-            for(int c : records) {
-                if (c == Statement.EXECUTE_FAILED) {
-                    Logger.error("Batch Update failed: " + c + " | All: " + Arrays.toString(records));
-                    success = false;
-                    break;
+
+            try(var s = provider.connection.prepareStatement(isNewAgent ? "UPDATE metadata_items SET audience_rating = ?, extra_data = ?, rating = NULL WHERE id = ?" 
+                    : "UPDATE metadata_items SET rating = ?, extra_data = ? WHERE id = ?")) {
+                for(var item : items) {
+                    Double d = isNewAgent ? item.audienceRating : item.rating;
+
+                    //TODO: hotfix, investigate further only happened to one person over the entire tool lifetime
+                    if(d == null) {
+                        Logger.error("Null value encountered. Should not be possible. Skipping entry to not crash tool. Contact maintainer with this dump: " + Objects.toString(item));
+                        continue;
+                    }
+
+                    s.setDouble(1, d);
+                    s.setString(2, item.extraData);
+                    s.setInt(3, item.id);
+                    s.addBatch();
+                }
+                int[] records = s.executeBatch();
+                for(int c : records) {
+                    if (c == Statement.EXECUTE_FAILED) {
+                        Logger.error("Batch Update failed: " + c + " | All: " + Arrays.toString(records));
+                        success = false;
+                        break;
+                    }
+                }
+            } catch (SQLException e) {
+                success = false;
+                throw Utility.rethrow(e);
+            } finally {
+                try {
+                    if(success) {
+                        provider.connection.commit();
+                    } else {
+                        provider.connection.rollback();
+                    }
+                } catch(SQLException e) {
+                    throw Utility.rethrow(e);
                 }
             }
-        } catch (SQLException e) {
-            success = false;
+        } catch(Exception e) {
             throw Utility.rethrow(e);
         } finally {
-            try {
-                if(success) {
-                    provider.connection.commit();
-                } else {
-                    provider.connection.rollback();
+            for(AutoCloseable a : close) {
+                try {
+                    a.close();
+                } catch (Exception e) {}
+                close.clear();
+            }
+
+            if(mitigationIcuTriggersDisabled) {
+                try {
+                    var s1 = provider.connection.createStatement();
+                    close.add(s1);
+                    s1.executeUpdate(ICU_MIG_1_SQL);
+                    var s2 = provider.connection.createStatement();
+                    close.add(s2);
+                    s2.executeUpdate(ICU_MIG_2_SQL);
+                } catch (SQLException ex) {
+                    Logger.error(ex);
+                    Logger.error("======================================");
+                    Logger.error("WARNING!!! COULD NOT RESTORE DISABLED TRIGGERS IN ICU MITIGATION!!!");
+                    Logger.error("RUN THE QUERIES BELOW ON YOUR PLEX DATABASE TO RESTORE TRIGGERS!");
+                    Logger.error("======================================");
+                    Logger.error(ICU_MIG_1_SQL);
+                    Logger.error(ICU_MIG_2_SQL);
+                    Logger.error("======================================");
+                    Logger.error("TOOL WILL EXIT NOW! DON'T USE BEVORE HAVING EXECUTED THESE COMMANDS!");
+                    System.exit(-1);
+                } finally {
+                    for(AutoCloseable a : close) {
+                        try {
+                            a.close();
+                        } catch (Exception e) {}
+                    }
                 }
-            } catch(SQLException e) {
-                throw Utility.rethrow(e);
             }
         }
     }

src/main/java/updatetool/imdb/ImdbDockerImplementation.java

@@ -116,7 +116,8 @@ public void bootstrap(Map<String, String> args) throws Exception {
             capabilities.remove(Capabilities.TVDB);
         } else {
             ApiVersion version;
-            if(tvdbApiKey.length() == 32) {
+            if(tvdbApiKey.length() >= 32) {
+                tvdbApiKey = tvdbApiKey.trim();
                 version = new TvdbApiV3(tvdbApiKey, null, null, null, null).version();
             } else {
                 version = new TvdbApiV4(tvdbApiKey, null, null, null, null, null).version();
@@ -195,11 +196,11 @@ public ImdbBatchJob(ExecutorService service, ImdbPipelineConfiguration config, P
         @Override
         @SuppressFBWarnings("DM_EXIT")
         public void run() {
-            KeyValueStore.expiredCheck(14, caches.get("tvdb-blacklist"));
-            KeyValueStore.expiredCheck(14, caches.get("tmdb-series-blacklist"));
-            KeyValueStore.expiredCheck(14, caches.get("tmdb-blacklist"));
-            KeyValueStore.expiredCheck(14, caches.get("tvdb-movie-blacklist"));
-            KeyValueStore.expiredCheck(1, caches.get("tvdb-legacy-mapping"));
+            KeyValueStore.expiredCheck(30, caches.get("tvdb-blacklist"));
+            KeyValueStore.expiredCheck(30, caches.get("tmdb-series-blacklist"));
+            KeyValueStore.expiredCheck(30, caches.get("tmdb-blacklist"));
+            KeyValueStore.expiredCheck(30, caches.get("tvdb-movie-blacklist"));
+            KeyValueStore.expiredCheck(3, caches.get("tvdb-legacy-mapping"));
 
             List<Library> libraries = new ArrayList<>();
             ImdbLibraryMetadata metadata = null;

src/main/java/updatetool/imdb/ImdbPipeline.java

@@ -96,7 +96,7 @@ public ImdbPipeline(ImdbLibraryMetadata metadata, ExecutorService service, Map<S
         this.configuration = configuration;
         this.dataset = dataset;
 
-        var tmdbResolver = configuration.resolveTmdb() ? new TmdbToImdbResolvement(new TmdbApiV3(configuration.tmdbApiKey,caches.get("tmdb-series"), caches.get("tmdb"), caches.get("tmdb-series-blacklist"), caches.get("tmdb-blacklist"))) : resolveDefault;
+        var tmdbResolver = configuration.resolveTmdb() ? new TmdbToImdbResolvement(new TmdbApiV3(configuration.tmdbApiKey, caches.get("tmdb-series"), caches.get("tmdb"), caches.get("tmdb-series-blacklist"), caches.get("tmdb-blacklist"))) : resolveDefault;
         var tvdbResolver = configuration.resolveTvdb() ? new TvdbToImdbResolvement(configuration.isTvdbV4 
                 ? new TvdbApiV4(configuration.tvdbApiKey, caches.get("tvdb"), caches.get("tvdb-blacklist"), caches.get("tvdb-movie"), caches.get("tvdb-movie-blacklist"), caches.get("tvdb-legacy-mapping"))
                 : new TvdbApiV3(configuration.tvdbApiKey, caches.get("tvdb"), caches.get("tvdb-blacklist"), caches.get("tvdb-movie"), caches.get("tvdb-movie-blacklist"))) 

src/main/java/updatetool/imdb/resolvement/NewPlexAgentToImdbResolvement.java

@@ -39,7 +39,6 @@ public boolean resolve(ImdbMetadataResult toResolve) {
             return true;
         } else if(candidate.startsWith("tmdb")) {
             if(fallbackTmdb == null) {
-                Logger.warn("TMDB id associated with new plex {} agent guid but no TMDB resolvement enabled. Ignoring item: ({}, {}, {})", toResolve.type, candidate, toResolve.guid, toResolve.title);
                 return false;
             }
 
@@ -54,7 +53,6 @@ public boolean resolve(ImdbMetadataResult toResolve) {
             return success;
         } else if(candidate.startsWith("tvdb")) {
             if(fallbackTvdb == null) {
-                Logger.warn("TVDB id associated with new plex {} agent guid but no TVDB resolvement enabled. Ignoring item: ({}, {}, {})", toResolve.type, candidate, toResolve.guid, toResolve.title);
                 return false;
             }
 

src/main/resources/VERSION

@@ -1 +1 @@
-1.5.3
+1.5.4