Skip to content

Commit

Permalink
Merge pull request #122 from Galaxypedia-Wiki/single-api-call
Browse files Browse the repository at this point in the history
  • Loading branch information
@smallketchup82
smallketchup82 authored Sep 5, 2024
2 parents 0cf4ec2 + c7cf87b commit c5f53b1
Show file tree
Hide file tree
Showing 2 changed files with 61 additions and 25 deletions.
64 changes: 48 additions & 16 deletions ketchupbot-framework/API/MwClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Text;
using System.Web;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;

namespace ketchupbot_framework.API;

Expand Down Expand Up @@ -91,42 +92,73 @@ public async Task<bool> IsLoggedIn()
return data?.error == null;
}

/// <summary>
/// Get the content of a single article
/// </summary>
/// <param name="title"></param>
/// <returns></returns>
/// <exception cref="InvalidOperationException"></exception>
public async Task<string> GetArticle(string title)
{
return (await GetArticles([title])).FirstOrDefault().Value ??
throw new InvalidOperationException("Failed to fetch article");
}

/// <summary>
/// Get the content of multiple articles
/// </summary>
/// <param name="titles"></param>
/// <returns>An array of page contents. Or an empty array if no pages were found</returns>
/// <exception cref="InvalidOperationException"></exception>
public async Task<Dictionary<string, string>> GetArticles(string[] titles)
{
using HttpResponseMessage response = await Client.GetAsync(
$"{_baseUrl}?action=query&format=json&prop=revisions&titles={HttpUtility.UrlEncode(title)}&rvslots=*&rvprop=content&formatversion=2");
$"{_baseUrl}?action=query&format=json&prop=revisions&titles={string.Join("|", titles.Select(HttpUtility.UrlEncode))}&rvslots=*&rvprop=content&formatversion=2");

response.EnsureSuccessStatusCode();

string jsonResponse = await response.Content.ReadAsStringAsync();

dynamic? data = JsonConvert.DeserializeObject<dynamic>(jsonResponse);
dynamic data = JsonConvert.DeserializeObject<dynamic>(jsonResponse) ?? throw new InvalidOperationException("Failed to deserialize response");

return ExtractPageContent(data) ?? throw new InvalidOperationException("Failed to fetch article");
JArray? pages = data.query?.pages;

if (pages == null || pages.Count == 0)
return [];

Dictionary<string, string> articles = new();

foreach (dynamic page in pages)
{
string? content = ExtractPageContent(page);
if (!string.IsNullOrEmpty(content) && page.title != null)
articles.Add(page.title.ToString(), content);
}

return articles;
}

private static string? ExtractPageContent(dynamic? data)
private static string? ExtractPageContent(dynamic page)
{
if (data == null)
if (page == null)
return null;

if (data.query?.pages == null || data.query.pages.Count <= 0)
if (page.revisions == null || page.revisions.Count <= 0)
return null;

dynamic? page = data.query.pages[0];

if (page?.revisions == null || page?.revisions.Count <= 0)
return null;

dynamic? revision = page?.revisions[0];
dynamic? revision = page.revisions[0];
return revision?.slots?.main?.content;
}

public async Task EditArticle(string title, string newContent, string summary, bool? dryRun = false)
/// <summary>
/// Edit an article on the wiki with the provided content
/// </summary>
/// <param name="title">The title of the page to edit</param>
/// <param name="newContent">The new content of the page</param>
/// <param name="summary">The edit summary</param>
/// <exception cref="InvalidOperationException"></exception>
public async Task EditArticle(string title, string newContent, string summary)
{
// If dry run is enabled, don't actually make the edit. Mock success instead.
if (dryRun == true) return;

if (await IsLoggedIn() == false)
throw new InvalidOperationException("Not logged in");

Expand Down
22 changes: 13 additions & 9 deletions ketchupbot-framework/ShipUpdater.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,9 @@ public async Task UpdateAllShips(Dictionary<string, Dictionary<string, string>>?
}

/// <summary>
///
/// Update multiple ships using the provided data.
/// </summary>
/// <param name="ships"></param>
/// <param name="ships">A list of ships to update</param>
/// <param name="shipDatas"></param>
/// <param name="threads"></param>
public async Task MassUpdateShips(List<string> ships, Dictionary<string, Dictionary<string, string>>? shipDatas = null, int threads = -1)
Expand All @@ -41,6 +41,8 @@ public async Task MassUpdateShips(List<string> ships, Dictionary<string, Diction
shipDatas ??= await apiManager.GetShipsData();
ArgumentNullException.ThrowIfNull(shipDatas);

Dictionary<string, string> articles = await bot.GetArticles(ships.ToArray());

await Parallel.ForEachAsync(ships, new ParallelOptions {
MaxDegreeOfParallelism = threads
}, async (ship, _) =>
Expand All @@ -51,7 +53,7 @@ public async Task MassUpdateShips(List<string> ships, Dictionary<string, Diction
var updateStart = Stopwatch.StartNew();
#endif
Log.Information("{Identifier} Updating ship...", GetShipIdentifier(ship));
await UpdateShip(ship, shipDatas.GetValueOrDefault(ship));
await UpdateShip(ship, shipDatas.GetValueOrDefault(ship), articles.GetValueOrDefault(ship));
#if DEBUG
updateStart.Stop();
Log.Information("{ShipIdentifier)} Updated ship in {UpdateStartElapsedMilliseconds}ms", GetShipIdentifier(ship), updateStart.ElapsedMilliseconds);
Expand All @@ -78,7 +80,8 @@ public async Task MassUpdateShips(List<string> ships, Dictionary<string, Diction
/// </summary>
/// <param name="ship">The name of the ship to update</param>
/// <param name="data">Supply a <see cref="Dictionary{TKey,TValue}"/> to use for updating. If left null, it will be fetched for you, but this is very bandwidth intensive for mass updating. It is better to grab it beforehand, filter the data for the specific <see cref="Dictionary{TKey,TValue}"/> needed, and pass that to the functions.</param>
private async Task UpdateShip(string ship, Dictionary<string, string>? data = null)
/// <param name="shipArticle">Provide a string to use as an article. If left null, it will be fetched based on <paramref name="ship"/></param>
private async Task UpdateShip(string ship, Dictionary<string, string>? data = null, string? shipArticle = null)
{
ship = GetShipName(ship);

Expand All @@ -104,22 +107,22 @@ private async Task UpdateShip(string ship, Dictionary<string, string>? data = nu
var fetchArticleStart = Stopwatch.StartNew();
#endif

string article = await bot.GetArticle(ship); // Throws exception if article does not exist
shipArticle ??= await bot.GetArticle(ship); // Throws exception if article does not exist

#if DEBUG
fetchArticleStart.Stop();
Log.Debug("{Identifier} Fetched article in {1}ms", GetShipIdentifier(ship), fetchArticleStart.ElapsedMilliseconds);
#endif
#endregion

if (IGNORE_FLAG_REGEX().IsMatch(article.ToLower())) throw new InvalidOperationException("Found ignore flag in article");
if (IGNORE_FLAG_REGEX().IsMatch(shipArticle.ToLower())) throw new InvalidOperationException("Found ignore flag in article");

#region Infobox Parsing Logic
#if DEBUG
var parsingInfoboxStart = Stopwatch.StartNew();
#endif

Dictionary<string, string> parsedInfobox = WikiParser.ParseInfobox(WikiParser.ExtractInfobox(article));
Dictionary<string, string> parsedInfobox = WikiParser.ParseInfobox(WikiParser.ExtractInfobox(shipArticle));

#if DEBUG
parsingInfoboxStart.Stop();
Expand Down Expand Up @@ -174,7 +177,7 @@ private async Task UpdateShip(string ship, Dictionary<string, string>? data = nu
var wikitextConstructionStart = Stopwatch.StartNew();
#endif

string newWikitext = WikiParser.ReplaceInfobox(article, WikiParser.ObjectToWikitext(sanitizedData.Item1));
string newWikitext = WikiParser.ReplaceInfobox(shipArticle, WikiParser.ObjectToWikitext(sanitizedData.Item1));

#if DEBUG
wikitextConstructionStart.Stop();
Expand All @@ -200,7 +203,8 @@ private async Task UpdateShip(string ship, Dictionary<string, string>? data = nu
editSummary.AppendLine("Removed parameters: " + string.Join(", ", sanitizedData.Item2));
}

await bot.EditArticle(ship, newWikitext, editSummary.ToString(), dryRun);
if (!dryRun)
await bot.EditArticle(ship, newWikitext, editSummary.ToString());

#if DEBUG
articleEditStart.Stop();
Expand Down

0 comments on commit c5f53b1

Please sign in to comment.