diff --git a/Jobs/JobScheduler.cs b/Jobs/JobScheduler.cs index e1151f7..1cae1cb 100644 --- a/Jobs/JobScheduler.cs +++ b/Jobs/JobScheduler.cs @@ -4,6 +4,7 @@ using Hangfire.States; using TNRD.Zeepkist.GTR.Backend.Levels.Jobs; using TNRD.Zeepkist.GTR.Backend.Levels.Points.Jobs; +using TNRD.Zeepkist.GTR.Backend.Levels.Requests.Jobs; using TNRD.Zeepkist.GTR.Backend.PersonalBests.Jobs; using TNRD.Zeepkist.GTR.Backend.Users.Points.Jobs; using TNRD.Zeepkist.GTR.Backend.WorldRecords.Jobs; @@ -41,11 +42,13 @@ public void ScheduleRecurringJobs() #if DEBUG ScheduleRecurringJob(Cron.Never()); ScheduleRecurringJob(Cron.Never()); + ScheduleRecurringJob(Cron.Never()); ScheduleRecurringJob(Cron.Never()); ScheduleRecurringJob(Cron.Never()); ScheduleRecurringJob(Cron.Never()); ScheduleRecurringJob(Cron.Never()); #else + ScheduleRecurringJob(Cron.MinuteInterval(5)); ScheduleRecurringJob(Cron.Never()); ScheduleRecurringJob(Cron.Never()); ScheduleRecurringJob(Cron.Never()); diff --git a/Levels/Items/LevelItemsService.cs b/Levels/Items/LevelItemsService.cs index 9dabef5..23a0b5e 100644 --- a/Levels/Items/LevelItemsService.cs +++ b/Levels/Items/LevelItemsService.cs @@ -11,6 +11,7 @@ namespace TNRD.Zeepkist.GTR.Backend.Levels.Items; public interface ILevelItemsService { IEnumerable GetAll(); + IEnumerable GetForPublishedFileDetails(PublishedFileDetails publishedFileDetails); bool ExistsForLevel(int levelId); bool Exists(PublishedFileDetails publishedFileDetails, ZeepLevel zeepLevel, string hash); @@ -19,6 +20,8 @@ void Create( WorkshopLevel workshopLevel, ZeepLevel zeepLevel, Level level); + + void MarkDeleted(LevelItem levelItem); } public class LevelItemsService : ILevelItemsService @@ -45,6 +48,16 @@ public IEnumerable GetAll() return _repository.GetAll(); } + public IEnumerable GetForPublishedFileDetails(PublishedFileDetails publishedFileDetails) + { + if (ulong.TryParse(publishedFileDetails.PublishedFileId, out ulong publishedFileId)) + { + return _repository.GetAll(item => item.Deleted == false && item.WorkshopId == publishedFileId); + } + + return Enumerable.Empty(); + } + public bool ExistsForLevel(int levelId) { return _repository.Exists(x => x.IdLevel == levelId); @@ -128,4 +141,10 @@ public void Create( _repository.Insert(item); } + + public void MarkDeleted(LevelItem levelItem) + { + levelItem.Deleted = true; + _repository.Update(levelItem); + } } diff --git a/Levels/Jobs/DownloadResult.cs b/Levels/Jobs/DownloadResult.cs new file mode 100644 index 0000000..f6462db --- /dev/null +++ b/Levels/Jobs/DownloadResult.cs @@ -0,0 +1,7 @@ +using FluentResults; +using TNRD.Zeepkist.GTR.Backend.Steam.Resources; +using TNRD.Zeepkist.GTR.Backend.Workshop; + +namespace TNRD.Zeepkist.GTR.Backend.Levels.Jobs; + +public record DownloadResult(IEnumerable PublishedFileDetails, Result Result); \ No newline at end of file diff --git a/Levels/Jobs/FullWorkshopScanJob.cs b/Levels/Jobs/FullWorkshopScanJob.cs index 3f11dd5..580342a 100644 --- a/Levels/Jobs/FullWorkshopScanJob.cs +++ b/Levels/Jobs/FullWorkshopScanJob.cs @@ -1,43 +1,20 @@ using JetBrains.Annotations; -using Microsoft.Extensions.Options; -using TNRD.Zeepkist.GTR.Backend.Hashing; -using TNRD.Zeepkist.GTR.Backend.Levels.Items; -using TNRD.Zeepkist.GTR.Backend.Levels.Metadata; -using TNRD.Zeepkist.GTR.Backend.Steam; -using TNRD.Zeepkist.GTR.Backend.Steam.Resources; using TNRD.Zeepkist.GTR.Backend.Workshop; -using TNRD.Zeepkist.GTR.Backend.Zeeplevel; namespace TNRD.Zeepkist.GTR.Backend.Levels.Jobs; public class FullWorkshopScanJob : WorkshopScanJob { - private static bool IsRunning = false; - - private readonly IPublishedFileServiceApi _publishedFileServiceApi; - private readonly SteamOptions _steamOptions; + public static bool IsRunning { get; private set; } public FullWorkshopScanJob( ILogger logger, - IHashService hashService, - ILevelService levelService, - ILevelMetadataService levelMetadataService, - ILevelItemsService levelItemsService, - IWorkshopService workshopService, - IZeeplevelService zeeplevelService, - IPublishedFileServiceApi publishedFileServiceApi, - IOptions steamOptions) - : base( - logger, - hashService, - levelService, - levelMetadataService, - levelItemsService, - workshopService, - zeeplevelService) + WorkshopLister workshopLister, + WorkshopDownloader workshopDownloader, + WorkshopProcessor workshopProcessor, + IWorkshopService workshopService) + : base(logger, workshopLister, workshopDownloader, workshopProcessor, workshopService) { - _publishedFileServiceApi = publishedFileServiceApi; - _steamOptions = steamOptions.Value; } [UsedImplicitly] @@ -51,51 +28,8 @@ public async Task ExecuteAsync() IsRunning = true; Logger.LogInformation("Starting full workshop scan"); - - string cursor = "*"; - int attempts = 0; - - while (true) - { - Logger.LogInformation("Scanning page {Cursor}", cursor); - QueryFilesResult result - = await _publishedFileServiceApi.QueryFiles(_steamOptions.ApiKey, cursor); - - string currentCursor = cursor; - cursor = result.Response.NextCursor; - - if (cursor == currentCursor && result.Response.PublishedFileDetails == null) - { - Logger.LogInformation("Reached end of listing {Cursor}", cursor); - break; // No more pages - } - - try - { - await ProcessPage(result); - attempts = 0; - } - catch (Exception e) - { - Logger.LogError(e, "Failed to process page {Cursor}", cursor); - if (attempts >= 3) - { - Logger.LogError("Failed to process page {Cursor} after 3 attempts", cursor); - break; - } - - Logger.LogInformation("Changing cursor back from {Current} to {Previous}", cursor, currentCursor); - cursor = currentCursor; - int delay = 5 * ++attempts; - Logger.LogInformation("Waiting {Delay} minutes", delay); - await Task.Delay(TimeSpan.FromMinutes(delay)); - continue; - } - - if (cursor == currentCursor) - break; - } - + await Run(WorkshopLister.QueryType.Normal); + Logger.LogInformation("Finished full workshop scan"); IsRunning = false; } } diff --git a/Levels/Jobs/PartialWorkshopScanJob.cs b/Levels/Jobs/PartialWorkshopScanJob.cs index 3f63a50..6eb6222 100644 --- a/Levels/Jobs/PartialWorkshopScanJob.cs +++ b/Levels/Jobs/PartialWorkshopScanJob.cs @@ -1,77 +1,44 @@ using JetBrains.Annotations; -using Microsoft.Extensions.Options; -using TNRD.Zeepkist.GTR.Backend.Hashing; -using TNRD.Zeepkist.GTR.Backend.Levels.Items; -using TNRD.Zeepkist.GTR.Backend.Levels.Metadata; -using TNRD.Zeepkist.GTR.Backend.Steam; -using TNRD.Zeepkist.GTR.Backend.Steam.Resources; using TNRD.Zeepkist.GTR.Backend.Workshop; -using TNRD.Zeepkist.GTR.Backend.Zeeplevel; namespace TNRD.Zeepkist.GTR.Backend.Levels.Jobs; public class PartialWorkshopScanJob : WorkshopScanJob { - private readonly IPublishedFileServiceApi _publishedFileServiceApi; - private readonly SteamOptions _steamOptions; - public PartialWorkshopScanJob( ILogger logger, - IHashService hashService, - ILevelService levelService, - ILevelMetadataService levelMetadataService, - ILevelItemsService levelItemsService, - IWorkshopService workshopService, - IZeeplevelService zeeplevelService, - IPublishedFileServiceApi publishedFileServiceApi, - IOptions steamOptions) - : base( - logger, - hashService, - levelService, - levelMetadataService, - levelItemsService, - workshopService, - zeeplevelService) + WorkshopLister workshopLister, + WorkshopDownloader workshopDownloader, + WorkshopProcessor workshopProcessor, + IWorkshopService workshopService) + : base(logger, workshopLister, workshopDownloader, workshopProcessor, workshopService) { - _publishedFileServiceApi = publishedFileServiceApi; - _steamOptions = steamOptions.Value; } [UsedImplicitly] public async Task ExecuteAsync() { + if (FullWorkshopScanJob.IsRunning) + { + Logger.LogInformation("Skipping partial workshop scan because full workshop scan is running"); + return; + } + await ScanByPublicationDate(); await ScanByUpdatedDate(); } private async Task ScanByPublicationDate() { - string cursor = "*"; - - for (int i = 0; i < 5; i++) - { - QueryFilesResult result - = await _publishedFileServiceApi.QueryFilesByPublicationDate(_steamOptions.ApiKey, cursor); - - cursor = result.Response.NextCursor; - - await ProcessPage(result); - } + Logger.LogInformation("Starting partial workshop scan by publication date"); + await Run(WorkshopLister.QueryType.ByPublicationDate, 5); + Logger.LogInformation("Finished partial workshop scan by publication date"); } - + private async Task ScanByUpdatedDate() { - string cursor = "*"; - - for (int i = 0; i < 5; i++) - { - QueryFilesResult result - = await _publishedFileServiceApi.QueryFilesByUpdatedDate(_steamOptions.ApiKey, cursor); - - cursor = result.Response.NextCursor; - - await ProcessPage(result); - } + Logger.LogInformation("Starting partial workshop scan by updated date"); + await Run(WorkshopLister.QueryType.ByUpdatedDate, 5); + Logger.LogInformation("Finished partial workshop scan by updated date"); } } diff --git a/Levels/Jobs/WorkshopDownloader.cs b/Levels/Jobs/WorkshopDownloader.cs new file mode 100644 index 0000000..6aef13f --- /dev/null +++ b/Levels/Jobs/WorkshopDownloader.cs @@ -0,0 +1,65 @@ +using FluentResults; +using TNRD.Zeepkist.GTR.Backend.Steam.Resources; +using TNRD.Zeepkist.GTR.Backend.Workshop; + +namespace TNRD.Zeepkist.GTR.Backend.Levels.Jobs; + +public class WorkshopDownloader +{ + private const int ItemsPerChunk = 10; + private const int MaxConcurrency = 25; + + private readonly ILogger _logger; + private readonly IWorkshopService _workshopService; + + public WorkshopDownloader(ILogger logger, IWorkshopService workshopService) + { + _logger = logger; + _workshopService = workshopService; + } + + public async Task Download(IEnumerable publishedFileDetails) + { + List chunks = publishedFileDetails.Chunk(ItemsPerChunk).ToList(); + SemaphoreSlim semaphore = new(MaxConcurrency, MaxConcurrency); + List> tasks = new(); + + for (int i = 0; i < chunks.Count; i++) + { + PublishedFileDetails[] chunk = chunks[i]; + int index = i; + tasks.Add(DownloadWorkshopItems(chunk, semaphore, index)); + } + + _logger.LogInformation("Waiting for all ({Amount}) tasks to complete (this may take a while)", chunks.Count); + DownloadResult[] results = await Task.WhenAll(tasks); + _logger.LogInformation("All tasks completed"); + return results; + } + + private async Task DownloadWorkshopItems( + IEnumerable publishedFileDetails, + SemaphoreSlim semaphore, + int index) + { + await semaphore.WaitAsync(); + try + { + _logger.LogInformation("Starting workshop download {Index}", index); + Result result = + await _workshopService.DownloadWorkshopItems( + publishedFileDetails.Select(x => ulong.Parse(x.PublishedFileId))); + _logger.LogInformation("Finished workshop download {Index}", index); + return new DownloadResult(publishedFileDetails, result); + } + catch (Exception e) + { + _logger.LogError(e, "Failed to download workshop items"); + return new DownloadResult(publishedFileDetails, Result.Fail(new ExceptionalError(e))); + } + finally + { + semaphore.Release(); + } + } +} \ No newline at end of file diff --git a/Levels/Jobs/WorkshopLister.cs b/Levels/Jobs/WorkshopLister.cs new file mode 100644 index 0000000..fd73a4a --- /dev/null +++ b/Levels/Jobs/WorkshopLister.cs @@ -0,0 +1,112 @@ +using FluentResults; +using Microsoft.Extensions.Options; +using Refit; +using TNRD.Zeepkist.GTR.Backend.Steam; +using TNRD.Zeepkist.GTR.Backend.Steam.Resources; + +namespace TNRD.Zeepkist.GTR.Backend.Levels.Jobs; + +public class WorkshopLister +{ + public enum QueryType + { + Normal, + ByPublicationDate, + ByUpdatedDate + } + + private ILogger _logger; + private IPublishedFileServiceApi _publishedFileServiceApi; + private SteamOptions _steamOptions; + + public WorkshopLister(ILogger logger, + IPublishedFileServiceApi publishedFileServiceApi, + IOptions steamOptions) + { + _logger = logger; + _publishedFileServiceApi = publishedFileServiceApi; + _steamOptions = steamOptions.Value; + } + + public async Task>> List(decimal publishedFileId) + { + QueryFilesResult result = + await _publishedFileServiceApi.GetDetails(_steamOptions.ApiKey, publishedFileId); + + if (result.Response.PublishedFileDetails == null) + { + return Result.Fail("Unable to parse published file details"); + } + + return Result.Ok(result.Response.PublishedFileDetails.ToList()); + } + + public async Task>> List(QueryType queryType, + int pageLimit = -1) + { + string cursor = "*"; + int attempts = 0; + int currentPage = 0; + + List publishedFileIds = new(); + + while (currentPage < pageLimit || pageLimit == -1) + { + _logger.LogInformation("Scanning page {Cursor}", cursor); + + QueryFilesResult result = await QueryFilesResult(queryType, cursor); + + string currentCursor = cursor; + cursor = result.Response.NextCursor; + + if (cursor == currentCursor && result.Response.PublishedFileDetails == null) + { + _logger.LogInformation("Reached end of listing {Cursor}", cursor); + break; // No more pages + } + + try + { + publishedFileIds.AddRange(result.Response.PublishedFileDetails); + attempts = 0; + currentPage++; + } + catch (Exception e) + { + if (attempts >= 3) + { + _logger.LogError("Failed to process page {Cursor} after 3 attempts", cursor); + return Result.Fail($"Failed to process page {cursor} after 3 attempts"); + } + + _logger.LogError(e, "Failed to process page {Cursor}", cursor); + cursor = currentCursor; + int delay = 5 * ++attempts; + _logger.LogInformation("Waiting {Delay} minutes", delay); + await Task.Delay(TimeSpan.FromMinutes(delay)); + continue; + } + + if (cursor == currentCursor) + break; + } + + return Result.Ok(publishedFileIds); + } + + private async Task QueryFilesResult(QueryType queryType, string cursor) + { + QueryFilesResult result = queryType switch + { + QueryType.Normal => await _publishedFileServiceApi.QueryFiles(_steamOptions.ApiKey, cursor, + numPerPage: 100), + QueryType.ByPublicationDate => await _publishedFileServiceApi.QueryFilesByPublicationDate( + _steamOptions.ApiKey, cursor, numPerPage: 100), + QueryType.ByUpdatedDate => await _publishedFileServiceApi.QueryFilesByUpdatedDate(_steamOptions.ApiKey, + cursor, numPerPage: 100), + _ => throw new ArgumentOutOfRangeException(nameof(queryType), queryType, null) + }; + + return result; + } +} \ No newline at end of file diff --git a/Levels/Jobs/WorkshopProcessor.cs b/Levels/Jobs/WorkshopProcessor.cs new file mode 100644 index 0000000..84a93ac --- /dev/null +++ b/Levels/Jobs/WorkshopProcessor.cs @@ -0,0 +1,202 @@ +using TNRD.Zeepkist.GTR.Backend.Hashing; +using TNRD.Zeepkist.GTR.Backend.Levels.Items; +using TNRD.Zeepkist.GTR.Backend.Levels.Metadata; +using TNRD.Zeepkist.GTR.Backend.Steam.Resources; +using TNRD.Zeepkist.GTR.Backend.Workshop; +using TNRD.Zeepkist.GTR.Backend.Zeeplevel; +using TNRD.Zeepkist.GTR.Backend.Zeeplevel.Resources; +using TNRD.Zeepkist.GTR.Database.Data.Entities; + +namespace TNRD.Zeepkist.GTR.Backend.Levels.Jobs; + +public class WorkshopProcessor +{ + private const int MaxConcurrency = 10; + + private readonly ILogger _logger; + private readonly IServiceProvider _provider; + + public WorkshopProcessor( + ILogger logger, + IServiceProvider provider) + { + _logger = logger; + _provider = provider; + } + + public async Task Process(DownloadResult[] downloadResults) + { + List tasks = new(); + SemaphoreSlim semaphore = new(MaxConcurrency, MaxConcurrency); + + foreach (DownloadResult downloadResult in downloadResults) + { + if (downloadResult.Result.IsFailed) + { + _logger.LogError("Failed to download workshop items: {Result}", downloadResult.Result.ToString()); + continue; + } + + tasks.Add(CreateTask(downloadResult)); + } + + await Task.WhenAll(tasks); + } + + private Task CreateTask(DownloadResult downloadResult) + { + return Task.Run(() => + { + IServiceScope scope = _provider.CreateScope(); + + try + { + WorkshopProcess workshopProcess = scope.ServiceProvider.GetRequiredService(); + + Dictionary workshopItems = + downloadResult.Result.Value.Items.ToDictionary(x => x.PublishedFileId); + foreach (PublishedFileDetails publishedFileDetails in downloadResult.PublishedFileDetails) + { + workshopProcess.Execute(publishedFileDetails, workshopItems); + } + } + catch (Exception e) + { + _logger.LogError(e, "Failed to process workshop items"); + } + finally + { + scope.Dispose(); + } + }); + } + + public class WorkshopProcess + { + private readonly ILogger _logger; + private readonly ILevelService _levelService; + private readonly ILevelMetadataService _levelMetadataService; + private readonly ILevelItemsService _levelItemsService; + private readonly IHashService _hashService; + private readonly IZeeplevelService _zeeplevelService; + + public WorkshopProcess( + ILogger logger, + ILevelService levelService, + ILevelMetadataService levelMetadataService, + ILevelItemsService levelItemsService, + IHashService hashService, + IZeeplevelService zeeplevelService) + { + _logger = logger; + _levelService = levelService; + _levelMetadataService = levelMetadataService; + _levelItemsService = levelItemsService; + _hashService = hashService; + _zeeplevelService = zeeplevelService; + } + + public void Execute( + PublishedFileDetails publishedFileDetails, + Dictionary workshopItems) + { + _logger.LogInformation("Processing published file details {PublishedFileId}", + publishedFileDetails.PublishedFileId); + + DeleteMissingLevels(publishedFileDetails, workshopItems); + + ulong publishedFileId = ulong.Parse(publishedFileDetails.PublishedFileId); + + if (!workshopItems.TryGetValue(publishedFileId, out WorkshopItem? workshopItem)) + { + _logger.LogWarning( + "PublishedFileId ({PublishedFileId}) has not been downloaded from workshop", + publishedFileId); + + return; + } + + foreach (WorkshopLevel level in workshopItem.Levels) + { + ProcessWorkshopLevel(publishedFileId, publishedFileDetails, level); + } + } + + private void DeleteMissingLevels(PublishedFileDetails publishedFileDetails, + Dictionary workshopItems) + { + _logger.LogInformation("Deleting missing levels for {PublishedFileId}", + publishedFileDetails.PublishedFileId); + + IEnumerable existingLevelItems = + _levelItemsService.GetForPublishedFileDetails(publishedFileDetails); + List zeepLevels = new(); + + foreach (KeyValuePair kvp in workshopItems) + { + foreach (WorkshopLevel workshopLevel in kvp.Value.Levels) + { + ZeepLevel? zeepLevel = _zeeplevelService.Parse(workshopLevel.ZeeplevelPath); + if (zeepLevel != null) + { + zeepLevels.Add(zeepLevel); + } + } + } + + foreach (LevelItem levelItem in existingLevelItems) + { + if (zeepLevels.Any(x => x.UniqueId == levelItem.FileUid)) + continue; + + _logger.LogInformation("Marking level item {Id} for {PublishedFileId} as deleted", + levelItem.Id, + publishedFileDetails.PublishedFileId); + + _levelItemsService.MarkDeleted(levelItem); + } + } + + private void ProcessWorkshopLevel( + ulong publishedFileId, + PublishedFileDetails publishedFileDetails, + WorkshopLevel workshopLevel) + { + ZeepLevel? zeepLevel = _zeeplevelService.Parse(workshopLevel.ZeeplevelPath); + if (zeepLevel == null) + { + _logger.LogWarning( + "Unable to parse zeeplevel from workshop item {PublishedFileId}", + publishedFileId); + + return; + } + + string hash = _hashService.Hash(zeepLevel); + + if (!_levelService.TryGetByHash(hash, out Level? level)) + { + _logger.LogInformation("Creating level for {PublishedFileId};{Uid}", + publishedFileId, + zeepLevel.UniqueId); + level = _levelService.Create(hash); + } + + if (!_levelItemsService.Exists(publishedFileDetails, zeepLevel, hash)) + { + _logger.LogInformation("Creating level item for {PublishedFileId};{Uid}", + publishedFileId, + zeepLevel.UniqueId); + _levelItemsService.Create(publishedFileDetails, workshopLevel, zeepLevel, level); + } + + if (!_levelMetadataService.Exists(hash)) + { + _logger.LogInformation("Creating level metadata for {PublishedFileId};{Uid}", + publishedFileId, + zeepLevel.UniqueId); + _levelMetadataService.Create(zeepLevel, hash); + } + } + } +} \ No newline at end of file diff --git a/Levels/Jobs/WorkshopScanJob.cs b/Levels/Jobs/WorkshopScanJob.cs index 6dba45d..7ef0d52 100644 --- a/Levels/Jobs/WorkshopScanJob.cs +++ b/Levels/Jobs/WorkshopScanJob.cs @@ -1,122 +1,59 @@ using FluentResults; -using TNRD.Zeepkist.GTR.Backend.Hashing; -using TNRD.Zeepkist.GTR.Backend.Levels.Items; -using TNRD.Zeepkist.GTR.Backend.Levels.Metadata; using TNRD.Zeepkist.GTR.Backend.Steam.Resources; using TNRD.Zeepkist.GTR.Backend.Workshop; -using TNRD.Zeepkist.GTR.Backend.Zeeplevel; -using TNRD.Zeepkist.GTR.Backend.Zeeplevel.Resources; -using TNRD.Zeepkist.GTR.Database.Data.Entities; namespace TNRD.Zeepkist.GTR.Backend.Levels.Jobs; public abstract class WorkshopScanJob { private readonly ILogger _logger; - private readonly IHashService _hashService; - private readonly ILevelService _levelService; - private readonly ILevelMetadataService _levelMetadataService; - private readonly ILevelItemsService _levelItemsService; + private readonly WorkshopLister _workshopLister; + private readonly WorkshopDownloader _workshopDownloader; + private readonly WorkshopProcessor _workshopProcessor; private readonly IWorkshopService _workshopService; - private readonly IZeeplevelService _zeeplevelService; - public ILogger Logger => _logger; - - protected WorkshopScanJob( - ILogger logger, - IHashService hashService, - ILevelService levelService, - ILevelMetadataService levelMetadataService, - ILevelItemsService levelItemsService, - IWorkshopService workshopService, - IZeeplevelService zeeplevelService) + protected WorkshopScanJob(ILogger logger, + WorkshopLister workshopLister, + WorkshopDownloader workshopDownloader, + WorkshopProcessor workshopProcessor, + IWorkshopService workshopService) { _logger = logger; - _hashService = hashService; - _levelService = levelService; - _levelMetadataService = levelMetadataService; - _levelItemsService = levelItemsService; + _workshopLister = workshopLister; + _workshopDownloader = workshopDownloader; + _workshopProcessor = workshopProcessor; _workshopService = workshopService; - _zeeplevelService = zeeplevelService; } - protected async Task ProcessPage(QueryFilesResult result) - { - List publishedFileIds = result.Response.PublishedFileDetails - .Select(x => ulong.Parse(x.PublishedFileId)) - .ToList(); - - Result downloadResult = await _workshopService.DownloadWorkshopItems(publishedFileIds); - - if (downloadResult.IsFailed) - { - _logger.LogWarning("Failed to download workshop items"); - _workshopService.RemoveAllDownloads(); - return; - } - - Dictionary workshopItems = downloadResult.Value.Items.ToDictionary(x => x.PublishedFileId); - foreach (PublishedFileDetails publishedFileDetails in result.Response.PublishedFileDetails) - { - ProcessPublishedFileDetails(publishedFileDetails, workshopItems); - } - - _workshopService.RemoveDownloads(downloadResult.Value); - } - - private void ProcessPublishedFileDetails( - PublishedFileDetails publishedFileDetails, - Dictionary workshopItems) - { - ulong publishedFileId = ulong.Parse(publishedFileDetails.PublishedFileId); - - // TODO: get level items based on workshop id and check what needs to be marked as deleted - - if (!workshopItems.TryGetValue(publishedFileId, out WorkshopItem? workshopItem)) - { - _logger.LogWarning( - "PublishedFileId ({PublishedFileId}) has not been downloaded from workshop", - publishedFileId); - - return; - } - - foreach (WorkshopLevel level in workshopItem.Levels) - { - ProcessWorkshopLevel(publishedFileId, publishedFileDetails, level); - } - } + public ILogger Logger => _logger; - private void ProcessWorkshopLevel( - ulong publishedFileId, - PublishedFileDetails publishedFileDetails, - WorkshopLevel workshopLevel) + protected async Task Run(WorkshopLister.QueryType queryType, int pageLimit = -1) { - ZeepLevel? zeepLevel = _zeeplevelService.Parse(workshopLevel.ZeeplevelPath); - if (zeepLevel == null) + try { - _logger.LogWarning( - "Unable to parse zeeplevel from workshop item {PublishedFileId}", - publishedFileId); + Result> publishedFileResults = await _workshopLister.List(queryType, pageLimit); - return; - } + if (publishedFileResults.IsFailed) + { + _logger.LogWarning("Failed to get published file ids"); + return; + } - string hash = _hashService.Hash(zeepLevel); + DownloadResult[] downloadResults = await _workshopDownloader.Download(publishedFileResults.Value); - if (!_levelService.TryGetByHash(hash, out Level? level)) - { - level = _levelService.Create(hash); - } + await _workshopProcessor.Process(downloadResults); - if (!_levelItemsService.Exists(publishedFileDetails, zeepLevel, hash)) - { - _levelItemsService.Create(publishedFileDetails, workshopLevel, zeepLevel, level); + foreach (DownloadResult downloadResult in downloadResults) + { + if (downloadResult.Result.IsSuccess) + { + _workshopService.RemoveDownloads(downloadResult.Result.Value); + } + } } - - if (!_levelMetadataService.Exists(hash)) + catch (Exception e) { - _levelMetadataService.Create(zeepLevel, hash); + _logger.LogError(e, "Failed to run job"); } } } diff --git a/Levels/Requests/Jobs/ProcessLevelRequestsJob.cs b/Levels/Requests/Jobs/ProcessLevelRequestsJob.cs new file mode 100644 index 0000000..411f5b0 --- /dev/null +++ b/Levels/Requests/Jobs/ProcessLevelRequestsJob.cs @@ -0,0 +1,70 @@ +using FluentResults; +using JetBrains.Annotations; +using TNRD.Zeepkist.GTR.Backend.Levels.Jobs; +using TNRD.Zeepkist.GTR.Backend.Steam.Resources; +using TNRD.Zeepkist.GTR.Backend.Workshop; +using TNRD.Zeepkist.GTR.Database.Data.Entities; + +namespace TNRD.Zeepkist.GTR.Backend.Levels.Requests.Jobs; + +public class ProcessLevelRequestsJob +{ + private readonly ILevelRequestsService _service; + private readonly WorkshopLister _workshopLister; + private readonly WorkshopDownloader _workshopDownloader; + private readonly WorkshopProcessor _workshopProcessor; + private readonly ILogger _logger; + private readonly IWorkshopService _workshopService; + + public ProcessLevelRequestsJob(ILevelRequestsService service, + WorkshopLister workshopLister, + WorkshopDownloader workshopDownloader, + WorkshopProcessor workshopProcessor, + ILogger logger, + IWorkshopService workshopService) + { + _service = service; + _workshopLister = workshopLister; + _workshopDownloader = workshopDownloader; + _workshopProcessor = workshopProcessor; + _logger = logger; + _workshopService = workshopService; + } + + [UsedImplicitly] + public async Task ExecuteAsync() + { + List requests = _service.GetRequests().ToList(); + + foreach (LevelRequest request in requests) + { + try + { + Result> + publishedFileResults = await _workshopLister.List(request.WorkshopId); + if (publishedFileResults.IsFailed) + { + _logger.LogWarning("Failed to get published file ids"); + return; + } + + DownloadResult[] downloadResults = await _workshopDownloader.Download(publishedFileResults.Value); + + await _workshopProcessor.Process(downloadResults); + + foreach (DownloadResult downloadResult in downloadResults) + { + if (downloadResult.Result.IsSuccess) + { + _workshopService.RemoveDownloads(downloadResult.Result.Value); + } + } + } + catch (Exception e) + { + _logger.LogError(e, "Failed to process level request"); + _service.Delete(request); + } + } + } +} \ No newline at end of file diff --git a/Levels/Requests/LevelRequestsController.cs b/Levels/Requests/LevelRequestsController.cs index 01fc5e9..d0104d0 100644 --- a/Levels/Requests/LevelRequestsController.cs +++ b/Levels/Requests/LevelRequestsController.cs @@ -1,9 +1,23 @@ using Microsoft.AspNetCore.Mvc; +using TNRD.Zeepkist.GTR.Backend.Levels.Requests.Resources; namespace TNRD.Zeepkist.GTR.Backend.Levels.Requests; [ApiController] -[Route("[controller]")] +[Route("level/requests/")] public class LevelRequestsController : ControllerBase { + private readonly ILevelRequestsService _service; + + public LevelRequestsController(ILevelRequestsService service) + { + _service = service; + } + + [HttpPost("submit")] + public IActionResult Submit([FromBody] LevelRequestResource resource) + { + _service.Add(resource.WorkshopId); + return Ok(); + } } diff --git a/Levels/Requests/LevelRequestsService.cs b/Levels/Requests/LevelRequestsService.cs index e1a38bf..4d58b85 100644 --- a/Levels/Requests/LevelRequestsService.cs +++ b/Levels/Requests/LevelRequestsService.cs @@ -4,6 +4,7 @@ namespace TNRD.Zeepkist.GTR.Backend.Levels.Requests; public interface ILevelRequestsService { + void Add(ulong workshopId); IEnumerable GetRequests(); void Delete(LevelRequest request); } @@ -17,6 +18,18 @@ public LevelRequestsService(ILevelRequestsRepository repository) _repository = repository; } + public void Add(ulong workshopId) + { + LevelRequest? existing = _repository.GetSingle(x => x.WorkshopId == workshopId); + if (existing == null) + { + _repository.Insert(new LevelRequest() + { + WorkshopId = workshopId + }); + } + } + public IEnumerable GetRequests() { return _repository.GetAll(); diff --git a/Levels/Requests/Resources/LevelRequestResource.cs b/Levels/Requests/Resources/LevelRequestResource.cs new file mode 100644 index 0000000..6712c13 --- /dev/null +++ b/Levels/Requests/Resources/LevelRequestResource.cs @@ -0,0 +1,6 @@ +namespace TNRD.Zeepkist.GTR.Backend.Levels.Requests.Resources; + +public class LevelRequestResource +{ + public ulong WorkshopId { get; set; } +} \ No newline at end of file diff --git a/Program.cs b/Program.cs index 5538786..d619cee 100644 --- a/Program.cs +++ b/Program.cs @@ -23,6 +23,7 @@ using TNRD.Zeepkist.GTR.Backend.Jwt; using TNRD.Zeepkist.GTR.Backend.Levels; using TNRD.Zeepkist.GTR.Backend.Levels.Items; +using TNRD.Zeepkist.GTR.Backend.Levels.Jobs; using TNRD.Zeepkist.GTR.Backend.Levels.Metadata; using TNRD.Zeepkist.GTR.Backend.Levels.Points; using TNRD.Zeepkist.GTR.Backend.Levels.Requests; @@ -207,6 +208,11 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); + builder.Services.AddTransient(); builder.Services.Configure(builder.Configuration.GetSection(AuthenticationOptions.Key)); diff --git a/Steam/IPublishedFileServiceApi.cs b/Steam/IPublishedFileServiceApi.cs index 9e11900..ee7838c 100644 --- a/Steam/IPublishedFileServiceApi.cs +++ b/Steam/IPublishedFileServiceApi.cs @@ -33,4 +33,9 @@ Task QueryFilesByUpdatedDate( [AliasAs("appid")] uint appId = 1440670, [AliasAs("return_metadata")] bool returnMetadata = true, [AliasAs("return_details")] bool returnDetails = true); + + [Get("/GetDetails/v1/")] + Task GetDetails( + string key, + [AliasAs("publishedfileids[0]")] decimal publishedFileId); } diff --git a/Workshop/WorkshopService.cs b/Workshop/WorkshopService.cs index e8e2fd1..44ebad8 100644 --- a/Workshop/WorkshopService.cs +++ b/Workshop/WorkshopService.cs @@ -119,7 +119,18 @@ public void RemoveDownloads(WorkshopDownloads downloads) public void RemoveAllDownloads() { - throw new NotImplementedException(); + string[] directories = Directory.GetDirectories(_options.MountPath, "*", SearchOption.TopDirectoryOnly); + foreach (string directory in directories) + { + try + { + Directory.Delete(directory, true); + } + catch (Exception e) + { + _logger.LogError(e, "Failed to delete directory: {Directory}", directory); + } + } } private Task RunProcess(string guid, IEnumerable parameters) diff --git a/Zeeplevel/Resources/ZeepBlock.cs b/Zeeplevel/Resources/ZeepBlock.cs index 21bd97d..5488f7a 100644 --- a/Zeeplevel/Resources/ZeepBlock.cs +++ b/Zeeplevel/Resources/ZeepBlock.cs @@ -1,9 +1,12 @@ -using System.Numerics; +using System.Globalization; +using System.Numerics; namespace TNRD.Zeepkist.GTR.Backend.Zeeplevel.Resources; public class ZeepBlock { + private static readonly CultureInfo Culture = new CultureInfo("en-US"); + public ZeepBlock() { Paints = new List(); @@ -20,6 +23,6 @@ public ZeepBlock() public override string ToString() { return - $"Id: {Id}, Position: {Position}, Euler: {Euler}, Scale: {Scale}, Paints: {string.Join(", ", Paints)}, Options: {string.Join(", ", Options)}"; + $"Id: {Id.ToString(Culture)}, Position: {Position.ToString(string.Empty, Culture)}, Euler: {Euler.ToString(string.Empty, Culture)}, Scale: {Scale.ToString(string.Empty, Culture)}, Paints: {string.Join(", ", Paints.Select(x => x.ToString(Culture)))}, Options: {string.Join(", ", Options.Select(x => x.ToString(Culture)))}"; } } diff --git a/Zeeplevel/ZeeplevelService.cs b/Zeeplevel/ZeeplevelService.cs index d835743..6500fc3 100644 --- a/Zeeplevel/ZeeplevelService.cs +++ b/Zeeplevel/ZeeplevelService.cs @@ -1,4 +1,5 @@ -using System.Numerics; +using System.Globalization; +using System.Numerics; using TNRD.Zeepkist.GTR.Backend.Zeeplevel.Resources; namespace TNRD.Zeepkist.GTR.Backend.Zeeplevel; @@ -12,6 +13,7 @@ public interface IZeeplevelService public class ZeeplevelService : IZeeplevelService { private readonly ILogger _logger; + private readonly CultureInfo _culture = new("en-US"); public ZeeplevelService(ILogger logger) { @@ -76,18 +78,18 @@ private bool ParseCameraLine(string line, ZeepLevel level) return false; level.CameraPosition = new Vector3( - float.Parse(splits[0]), - float.Parse(splits[1]), - float.Parse(splits[2])); + float.Parse(splits[0], NumberStyles.Any, _culture), + float.Parse(splits[1], NumberStyles.Any, _culture), + float.Parse(splits[2], NumberStyles.Any, _culture)); level.CameraEuler = new Vector3( - float.Parse(splits[3]), - float.Parse(splits[4]), - float.Parse(splits[5])); + float.Parse(splits[3], NumberStyles.Any, _culture), + float.Parse(splits[4], NumberStyles.Any, _culture), + float.Parse(splits[5], NumberStyles.Any, _culture)); level.CameraRotation = new Vector2( - float.Parse(splits[6]), - float.Parse(splits[7])); + float.Parse(splits[6], NumberStyles.Any, _culture), + float.Parse(splits[7], NumberStyles.Any, _culture)); return true; } @@ -117,12 +119,12 @@ private bool ParseValidationLine(string line, ZeepLevel level) level.IsValidated = false; } - level.GoldTime = float.Parse(splits[1]); - level.SilverTime = float.Parse(splits[2]); - level.BronzeTime = float.Parse(splits[3]); + level.GoldTime = float.Parse(splits[1], NumberStyles.Any, _culture); + level.SilverTime = float.Parse(splits[2], NumberStyles.Any, _culture); + level.BronzeTime = float.Parse(splits[3], NumberStyles.Any, _culture); - level.Skybox = int.Parse(splits[4]); - level.Ground = int.Parse(splits[5]); + level.Skybox = int.Parse(splits[4], NumberStyles.Any, _culture); + level.Ground = int.Parse(splits[5], NumberStyles.Any, _culture); return true; } @@ -139,6 +141,9 @@ private bool ParseBlocks(string[] lines, ZeepLevel level) foreach (string line in lines) { + if (string.IsNullOrWhiteSpace(line)) + continue; + try { string[] splits = line.Split(','); @@ -147,26 +152,26 @@ private bool ParseBlocks(string[] lines, ZeepLevel level) ZeepBlock block = new(); - block.Id = int.Parse(splits[0]); + block.Id = int.Parse(splits[0], NumberStyles.Any, _culture); block.Position = new Vector3( - float.Parse(splits[1]), - float.Parse(splits[2]), - float.Parse(splits[3])); + float.Parse(splits[1], NumberStyles.Any, _culture), + float.Parse(splits[2], NumberStyles.Any, _culture), + float.Parse(splits[3], NumberStyles.Any, _culture)); block.Euler = new Vector3( - float.Parse(splits[4]), - float.Parse(splits[5]), - float.Parse(splits[6])); + float.Parse(splits[4], NumberStyles.Any, _culture), + float.Parse(splits[5], NumberStyles.Any, _culture), + float.Parse(splits[6], NumberStyles.Any, _culture)); block.Scale = new Vector3( - float.Parse(splits[7]), - float.Parse(splits[8]), - float.Parse(splits[9])); + float.Parse(splits[7], NumberStyles.Any, _culture), + float.Parse(splits[8], NumberStyles.Any, _culture), + float.Parse(splits[9], NumberStyles.Any, _culture)); - splits[10..27].ToList().ForEach(x => block.Paints.Add(int.Parse(x))); + splits[10..27].ToList().ForEach(x => block.Paints.Add(int.Parse(x, NumberStyles.Any, _culture))); - splits[27..38].ToList().ForEach(x => block.Options.Add(float.Parse(x))); + splits[27..38].ToList().ForEach(x => block.Options.Add(float.Parse(x, NumberStyles.Any, _culture))); blocks.Add(block); } @@ -180,4 +185,4 @@ private bool ParseBlocks(string[] lines, ZeepLevel level) level.Blocks = blocks; return true; } -} +} \ No newline at end of file