diff --git a/CHANGELOG.md b/CHANGELOG.md index 59cdb5f..ed6e0aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.6.6 +- Fixed broken ImdbScraper due to changes on IMDBs website. +- Updated the IMDB resolvement process to now factor in both TVDB/TMDB instead of just choosing one of them. This is in preparation to hopefully soon support IMDB lookup for items that only have a TMDB ID from the new Plex agent. + ## 1.6.5 - Updated ImdbScraper to handle new IMDB web design. The scraper will now work again instead of throwing tons of `appears to not be allowed to be rated by anyone` messages. - Mitigation added to automatically reset the set of scraper blacklisted items for older versions once on start-up, so you don't need to wait for 30 days until the scraper picks up those possibly wrongly blacklisted items again for processing. diff --git a/VERSION b/VERSION index 49ebdd6..83d1a5e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.6.5 \ No newline at end of file +1.6.6 \ No newline at end of file diff --git a/build.gradle b/build.gradle index 96079ca..0106f13 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ plugins { id 'com.github.spotbugs' version '2.0.1' } -version = '1.6.5' +version = '1.6.6' sourceCompatibility = '11' new File(projectDir, "VERSION").text = version; diff --git a/src/main/java/updatetool/Mitigations.java b/src/main/java/updatetool/Mitigations.java index 4663a41..85a7661 100644 --- a/src/main/java/updatetool/Mitigations.java +++ b/src/main/java/updatetool/Mitigations.java @@ -16,9 +16,27 @@ public static void executeMitigations() { executeTypoSwitchCacheResetMitigation(); executeCacheParameterWrongOrderMitigation(); executeCacheResetForImdbScraperUpdateMitigation(); + executeNewAgentMappingFormatReset(); MITIGATIONS.dump(); } + private static void executeNewAgentMappingFormatReset() { + String KEY = "executeNewAgentMappingFormatReset"; + + if(MITIGATIONS.lookup(KEY) != null) + return; + + Logger.info("One time mitigation executed: Reset new-agent-mapping.json for new storage format."); + Logger.info("This mitigation will only be executed once."); + + var newAgentMapping = KeyValueStore.of(Main.PWD.resolve("new-agent-mapping.json")); + newAgentMapping.reset(); + newAgentMapping.dump(); + + Logger.info("Mitigation completed!"); + MITIGATIONS.cache(KEY, ""); + } + private static void executeCacheResetForImdbScraperUpdateMitigation() { String KEY = "executeCacheResetForImdbScraperUpdateMitigation"; diff --git a/src/main/java/updatetool/common/DatabaseSupport.java b/src/main/java/updatetool/common/DatabaseSupport.java index 3a25574..a14d6a0 100644 --- a/src/main/java/updatetool/common/DatabaseSupport.java +++ b/src/main/java/updatetool/common/DatabaseSupport.java @@ -15,6 +15,10 @@ public DatabaseSupport(SqliteDatabaseProvider provider) { this.provider = provider; } + public enum NewAgentSeriesType { + SERIES, SEASON, EPISODE; + } + public enum LibraryType { MOVIE(1), SERIES(2); diff --git a/src/main/java/updatetool/common/HttpRunner.java b/src/main/java/updatetool/common/HttpRunner.java index ba7bd8f..720dfb1 100644 --- a/src/main/java/updatetool/common/HttpRunner.java +++ b/src/main/java/updatetool/common/HttpRunner.java @@ -6,6 +6,7 @@ import java.util.Objects; import java.util.Optional; import org.tinylog.Logger; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; public class HttpRunner { private final int maxTries; @@ -35,6 +36,14 @@ public static RunnerResult ofFailure(T result) { } } + @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE") + private static String trim(Object o) { + if (o == null) return null; + String s = o.toString(); + if(s == null) return s; + return s.trim(); + } + public static class HttpCodeHandler { private final Map> handlers = new HashMap<>(); private final Handler defaultHandler; @@ -49,7 +58,7 @@ private HttpCodeHandler(Map> handlers, Handler { - Logger.error("{} : Unhandled HTTP Error [response={} | payload={}]", identifier, body, body.body()); + Logger.error("{} : Unhandled HTTP Error [response={} | payload={}]", identifier, trim(body), trim(body.body())); return RunnerResult.ofFailure(result); } : defaultHandler; diff --git a/src/main/java/updatetool/common/externalapis/TvdbApiV4.java b/src/main/java/updatetool/common/externalapis/TvdbApiV4.java index 8e59d4b..f7db3ac 100644 --- a/src/main/java/updatetool/common/externalapis/TvdbApiV4.java +++ b/src/main/java/updatetool/common/externalapis/TvdbApiV4.java @@ -18,6 +18,7 @@ import net.minidev.json.JSONArray; import net.minidev.json.JSONObject; import updatetool.common.DatabaseSupport.LibraryType; +import updatetool.common.DatabaseSupport.NewAgentSeriesType; import updatetool.common.HttpRunner; import updatetool.common.KeyValueStore; import updatetool.common.Utility; @@ -274,7 +275,7 @@ public void resolveImdbIdForItem(ImdbMetadataResult result) { if(parts.length == 3) { resolveLegacyLookup(parts, result); } else { - runner.run(result.type == LibraryType.MOVIE ? () -> queryForMovie(result.extractedId) : result.hasEpisodeAgentFlag ? () -> queryForEpisode(result.extractedId) : () -> queryForSeries(result.extractedId), result); + runner.run(result.type == LibraryType.MOVIE ? () -> queryForMovie(result.extractedId) : result.seriesType == NewAgentSeriesType.EPISODE ? () -> queryForEpisode(result.extractedId) : () -> queryForSeries(result.extractedId), result); } } diff --git a/src/main/java/updatetool/imdb/ImdbDatabaseSupport.java b/src/main/java/updatetool/imdb/ImdbDatabaseSupport.java index a47795f..37fd185 100644 --- a/src/main/java/updatetool/imdb/ImdbDatabaseSupport.java +++ b/src/main/java/updatetool/imdb/ImdbDatabaseSupport.java @@ -26,6 +26,7 @@ import updatetool.Globals; import updatetool.Main; import updatetool.common.DatabaseSupport.LibraryType; +import updatetool.common.DatabaseSupport.NewAgentSeriesType; import updatetool.common.KeyValueStore; import updatetool.common.SqliteDatabaseProvider; import updatetool.common.Utility; @@ -36,6 +37,61 @@ public class ImdbDatabaseSupport { private final KeyValueStore newAgentMapping; private final ImdbPipelineConfiguration config; + public static class ImdbMetadataResult { + //Id will be resolved in the pipeline and not here + public String imdbId, extractedId; + public String title, hash; + public Integer id, libraryId, index; + public String extraData, guid; + public Double rating, audienceRating; + public boolean resolved; + public LibraryType type; + public NewAgentSeriesType seriesType; + + public ImdbMetadataResult() {}; + + private ImdbMetadataResult(ResultSet rs, LibraryType type) throws SQLException { + this.type = type; + id = rs.getInt(1); + libraryId = rs.getInt(2); + guid = rs.getString(3); + title = rs.getString(4); + extraData = rs.getString(5); + hash = rs.getString(6); + rating = (Double) rs.getObject(7); + audienceRating = (Double) rs.getObject(8); + index = (Integer) rs.getObject(9); + seriesType = guid.startsWith("plex://episode") ? NewAgentSeriesType.EPISODE + : guid.startsWith("plex://season") ? NewAgentSeriesType.SEASON + : guid.startsWith("plex://show") ? NewAgentSeriesType.SERIES : null; + } + + @Override + public int hashCode() { + return Objects.hashCode(imdbId); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ImdbMetadataResult other = (ImdbMetadataResult) obj; + return Objects.equals(id, other.id); + } + + @Override + public String toString() { + return "ImdbMetadataResult [imdbId=" + imdbId + ", extractedId=" + extractedId + ", title=" + title + + ", hash=" + hash + ", id=" + id + ", libraryId=" + libraryId + ", index=" + index + ", extraData=" + + extraData + ", guid=" + guid + ", rating=" + rating + ", audienceRating=" + audienceRating + + ", resolved=" + resolved + ", type=" + type + ", seriesType=" + seriesType + "]"; + } + } + public ImdbDatabaseSupport(SqliteDatabaseProvider provider, KeyValueStore newAgentMapping, ImdbPipelineConfiguration config) { this.provider = provider; this.newAgentMapping = newAgentMapping; @@ -100,68 +156,16 @@ private void testPlexSqliteBinaryVersion() { } } - public static class ImdbMetadataResult { - //Id will be resolved in the pipeline and not here - public String imdbId, extractedId; - public String title, hash; - public Integer id, libraryId; - public String extraData, guid; - public Double rating, audienceRating; - public boolean resolved; - public LibraryType type; - public boolean hasEpisodeAgentFlag; - - public ImdbMetadataResult() {}; - - private ImdbMetadataResult(ResultSet rs, LibraryType type) throws SQLException { - this.type = type; - id = rs.getInt(1); - libraryId = rs.getInt(2); - guid = rs.getString(3); - title = rs.getString(4); - extraData = rs.getString(5); - hash = rs.getString(6); - rating = (Double) rs.getObject(7); - audienceRating = (Double) rs.getObject(8); - hasEpisodeAgentFlag = guid.startsWith("plex://episode"); - } - - @Override - public int hashCode() { - return Objects.hashCode(imdbId); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - ImdbMetadataResult other = (ImdbMetadataResult) obj; - return Objects.equals(id, other.id); - } - - @Override - public String toString() { - return "ImdbMetadataResult [imdbId=" + imdbId + ", extractedId=" + extractedId + ", title=" + title - + ", hash=" + hash + ", id=" + id + ", libraryId=" + libraryId + ", extraData=" + extraData - + ", guid=" + guid + ", rating=" + rating + ", audienceRating=" + audienceRating + ", resolved=" - + resolved + ", type=" + type + "]"; - } - } - public List requestEntries(long libraryId, LibraryType type) { - return requestMetadata("SELECT id, library_section_id, guid, title, extra_data, hash, rating, audience_rating from metadata_items WHERE media_item_count = 1 AND library_section_id = " + libraryId, type); + return requestMetadata("SELECT id, library_section_id, guid, title, extra_data, hash, rating, audience_rating, \"index\" from metadata_items WHERE media_item_count = 1 AND library_section_id = " + libraryId, type); } public List requestTvSeriesRoot(long libraryId) { - return requestMetadata("SELECT id, library_section_id, guid, title, extra_data, hash, rating, audience_rating from metadata_items WHERE media_item_count = 0 AND parent_id IS NULL AND library_section_id = " + libraryId, LibraryType.SERIES); + return requestMetadata("SELECT id, library_section_id, guid, title, extra_data, hash, rating, audience_rating, \"index\" from metadata_items WHERE media_item_count = 0 AND parent_id IS NULL AND library_section_id = " + libraryId, LibraryType.SERIES); } public List requestTvSeasonRoot(long libraryId) { - return requestMetadata("SELECT id, library_section_id, guid, title, extra_data, hash, rating, audience_rating from metadata_items WHERE media_item_count = 0 AND parent_id NOT NULL AND library_section_id = " + libraryId, LibraryType.SERIES); + return requestMetadata("SELECT id, library_section_id, guid, title, extra_data, hash, rating, audience_rating, \"index\" from metadata_items WHERE media_item_count = 0 AND parent_id NOT NULL AND library_section_id = " + libraryId, LibraryType.SERIES); } private List requestMetadata(String query, LibraryType type) { @@ -193,21 +197,21 @@ private boolean updateNewAgentMetadataMapping(ImdbMetadataResult m) throws SQLEx return false; String v = newAgentMapping.lookup(m.guid); - if(v != null && v.startsWith("imdb://")) + if(v != null && v.contains("imdb://")) return false; - String result = null; + StringBuilder sb = new StringBuilder(); try(var handle = provider.queryFor("SELECT t.tag FROM taggings tg LEFT JOIN tags t ON tg.tag_id = t.id AND t.tag_type = 314 WHERE tg.metadata_item_id = " + m.id + " AND t.tag NOT NULL ORDER BY t.tag ASC")) { while(handle.result().next()) { - String id = handle.result().getString(1); - if(result == null || !result.startsWith("imdb://")) - result = id; + sb.append(handle.result().getString(1)).append("|"); } } - + + if(sb.length() > 0) { sb.deleteCharAt(sb.length()-1); } + String result = sb.toString(); boolean returnV = false; - if(result != null) { + if(!result.trim().isBlank()) { returnV = newAgentMapping.cache(m.guid, result); if(returnV) { Logger.info("Associated and cached {} with new movie/TV show agent guid {} ({}).", result, m.guid, m.title); diff --git a/src/main/java/updatetool/imdb/ImdbDockerImplementation.java b/src/main/java/updatetool/imdb/ImdbDockerImplementation.java index d1934f8..4f76afa 100644 --- a/src/main/java/updatetool/imdb/ImdbDockerImplementation.java +++ b/src/main/java/updatetool/imdb/ImdbDockerImplementation.java @@ -186,7 +186,6 @@ public void bootstrap(Map args) throws Exception { } Logger.info("Capabilities: " + capabilities.toString()); - var dbLocation = getDatabaseLocation(plexdata, overrideDatabaseLocation).toAbsolutePath().toString(); var config = new ImdbPipelineConfiguration(apikeyTmdb, apiauthTvdb, plexdata.resolve("Metadata/Movies"), dbLocation, executeUpdatesOverPlexSqliteVersion, capabilities); job = new ImdbBatchJob(Main.EXECUTOR, config, plexdata, caches, capabilities); diff --git a/src/main/java/updatetool/imdb/ImdbPipeline.java b/src/main/java/updatetool/imdb/ImdbPipeline.java index 89f82df..3bbee37 100644 --- a/src/main/java/updatetool/imdb/ImdbPipeline.java +++ b/src/main/java/updatetool/imdb/ImdbPipeline.java @@ -170,6 +170,9 @@ public void analyseDatabase(ImdbJob job) throws Exception { resolverTasks.stream().forEach(CompletableFuture::join); Logger.info("Progress printing watchdog has been stopped. Cancelation status: {}", handle.cancel(true)); + Logger.info("Save point: Persisting caches to keep queried look-up data in case of crashes or hang-ups."); + caches.forEach(KeyValueStore::dump); + int resolvedSize = resolved.size(); int itemsSize = items.size(); diff --git a/src/main/java/updatetool/imdb/ImdbRatingDatasetFactory.java b/src/main/java/updatetool/imdb/ImdbRatingDatasetFactory.java index a6d9a03..b099d25 100644 --- a/src/main/java/updatetool/imdb/ImdbRatingDatasetFactory.java +++ b/src/main/java/updatetool/imdb/ImdbRatingDatasetFactory.java @@ -64,7 +64,7 @@ public void ensureAvailability() { this.rating = scrapedRating; } } catch (Exception e) { - Logger.error(e.getClass().getSimpleName() + " exception encountered @ Screen Scraping"); + Logger.error(e.getClass().getSimpleName() + " exception encountered @ Screen Scraping [imdb={}]", imdbId); Logger.error("Please contact the maintainer of the application with the stacktrace below if you think this is unwanted behavior."); Logger.error("========================================"); Logger.error(e); diff --git a/src/main/java/updatetool/imdb/ImdbScraper.java b/src/main/java/updatetool/imdb/ImdbScraper.java index 169d402..1dc2153 100644 --- a/src/main/java/updatetool/imdb/ImdbScraper.java +++ b/src/main/java/updatetool/imdb/ImdbScraper.java @@ -16,6 +16,13 @@ import updatetool.common.Capabilities; import updatetool.common.KeyValueStore; +/* + * Test cases: + * System.out.println(scr.scrapeFallback("tt12850272", "test")); + System.out.println(scr.scrapeFallback("tt13846366", "test")); + System.out.println(scr.scrapeFallback("tt14001894", "test")); + */ + public class ImdbScraper implements Closeable { private static final String RETURN_LONG_BLACKLIST = "BLS_L"; private static final int SCRAPE_EVERY_N_DAYS_IGNORE = 30; @@ -77,12 +84,12 @@ private String scrape(String imdbId) throws Exception { } var doc = Jsoup.parse(response.body()); - var ratingValue = doc.select("span[class*=AggregateRatingButton__RatingScore]"); + var ratingValue = doc.select("div[data-testid*=hero-rating-bar__aggregate-rating__score]"); boolean blacklistShort = true; if(ratingValue.size() == 0) { - var canBeRated = doc.select("div[class*=RatingBar__ButtonContainer]"); - var children = canBeRated.get(0).childNodeSize(); + var canBeRated = doc.select("div[data-testid*=hero-rating-bar__user-rating__unrated]"); + var children = canBeRated.size(); if(children > 0) { if(!ImdbDockerImplementation.checkCapability(Capabilities.IGNORE_SCRAPER_NO_RESULT_LOG)) { @@ -106,7 +113,7 @@ private String scrape(String imdbId) throws Exception { throw new RuntimeException(String.format("Something went wrong with screen scraping the IMDB page for id %s (MORE_THAN_ONE_RESULT). Please contact developer by creating a GitHub issue and add this data: '%s'", imdbId, s)); } - String result = ratingValue.get(0).text(); + String result = ratingValue.get(0).getAllElements().get(1).text().replace(",", "."); try { return result; diff --git a/src/main/java/updatetool/imdb/resolvement/NewPlexAgentToImdbResolvement.java b/src/main/java/updatetool/imdb/resolvement/NewPlexAgentToImdbResolvement.java index af217da..435dca6 100644 --- a/src/main/java/updatetool/imdb/resolvement/NewPlexAgentToImdbResolvement.java +++ b/src/main/java/updatetool/imdb/resolvement/NewPlexAgentToImdbResolvement.java @@ -1,10 +1,12 @@ package updatetool.imdb.resolvement; +import java.util.Arrays; +import java.util.List; import org.tinylog.Logger; import updatetool.api.AgentResolvementStrategy; +import updatetool.common.DatabaseSupport.LibraryType; import updatetool.common.KeyValueStore; import updatetool.common.externalapis.AbstractApi.ApiVersion; -import updatetool.common.DatabaseSupport.LibraryType; import updatetool.imdb.ImdbDatabaseSupport.ImdbMetadataResult; import updatetool.imdb.ImdbUtility; @@ -27,47 +29,87 @@ public NewPlexAgentToImdbResolvement(KeyValueStore cache, TmdbToImdbResolvement @Override public boolean resolve(ImdbMetadataResult toResolve) { - String candidate = cache.lookup(toResolve.guid); + String candidateTmp = cache.lookup(toResolve.guid); - if(candidate == null) { + if(candidateTmp == null) { Logger.error("No external id associated with guid {} ({}).", toResolve.guid, toResolve.title); return false; } - if(candidate.startsWith("imdb")) { - toResolve.imdbId = ImdbUtility.extractId(ImdbUtility.IMDB, candidate); + int isImdb = -1, isTmdb = -1, isTvdb = -1; + List candidates = Arrays.asList(candidateTmp.split("\\|")); + for(int i = 0; i < candidates.size(); i++) { + if(candidates.get(i).startsWith("imdb://")) { isImdb = i; }; + if(candidates.get(i).startsWith("tmdb://")) { isTmdb = i; }; + if(candidates.get(i).startsWith("tvdb://")) { isTvdb = i; }; + } + + // IMDB ID is present + if(isImdb > -1) { + toResolve.imdbId = ImdbUtility.extractId(ImdbUtility.IMDB, candidates.get(isImdb)); return true; - } else if(candidate.startsWith("tmdb")) { - if(fallbackTmdb == null) { - return false; + } + + if(isImdb + isTmdb + isTvdb == -3) { + Logger.warn("Unhandled external ID in ({} => {}) = ({}). Please contact the author of the tool if you want to diagnose this further.", toResolve.type, toResolve.title, candidateTmp); + return false; + } + + boolean success = false; + if(toResolve.type == LibraryType.MOVIE) { + if(isTmdb > -1) { + success = fallbackTmdb(toResolve, candidates.get(isTmdb)); + if(success) return true; } - //TODO: TMDB v3 API is incapable of resolving this at the moment - if(toResolve.type == LibraryType.SERIES && fallbackTmdb.getVersion() == ApiVersion.TMDB_V3) - return false; - - String oldGuid = toResolve.guid; - toResolve.guid = candidate; - boolean success = fallbackTmdb.resolve(toResolve); - toResolve.guid = oldGuid; - return success; - } else if(candidate.startsWith("tvdb")) { - if(fallbackTvdb == null) { - return false; + if(isTvdb > -1) { + success = fallbackTvdb(toResolve, candidates.get(isTvdb)); + if(success) return true; } - - //TODO: TVDB v3 API is incapable of resolving this at the moment - if(toResolve.type == LibraryType.SERIES && fallbackTvdb.getVersion() == ApiVersion.TVDB_V3) - return false; - - String oldGuid = toResolve.guid; - toResolve.guid = candidate; - boolean success = fallbackTvdb.resolve(toResolve); - toResolve.guid = oldGuid; - return success; } else { - Logger.warn("Unhandled external {} id ({}). Please contact the author of the tool.", toResolve.type, candidate); + if(isTvdb > -1) { + success = fallbackTvdb(toResolve, candidates.get(isTvdb)); + if(success) return true; + } + + if(isTmdb > -1) { + success = fallbackTmdb(toResolve, candidates.get(isTmdb)); + if(success) return true; + } + } + + return success; + } + + private boolean fallbackTmdb(ImdbMetadataResult toResolve, String candidate) { + if(fallbackTmdb == null) { + return false; + } + + //TODO: TMDB v3 API is incapable of resolving this at the moment + if(toResolve.type == LibraryType.SERIES && fallbackTmdb.getVersion() == ApiVersion.TMDB_V3) + return false; + + String oldGuid = toResolve.guid; + toResolve.guid = candidate; + boolean success = fallbackTmdb.resolve(toResolve); + toResolve.guid = oldGuid; + return success; + } + + private boolean fallbackTvdb(ImdbMetadataResult toResolve, String candidate) { + if(fallbackTvdb == null) { return false; } + + //TODO: TVDB v3 API is incapable of resolving this at the moment + if(toResolve.type == LibraryType.SERIES && fallbackTvdb.getVersion() == ApiVersion.TVDB_V3) + return false; + + String oldGuid = toResolve.guid; + toResolve.guid = candidate; + boolean success = fallbackTvdb.resolve(toResolve); + toResolve.guid = oldGuid; + return success; } } diff --git a/src/main/resources/VERSION b/src/main/resources/VERSION index 49ebdd6..83d1a5e 100644 --- a/src/main/resources/VERSION +++ b/src/main/resources/VERSION @@ -1 +1 @@ -1.6.5 \ No newline at end of file +1.6.6 \ No newline at end of file