-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fixed up the full and partial workshop jobs Added level request processing job Updated zeeplevel parsing
- Loading branch information
1 parent
f681014
commit 0262585
Showing
18 changed files
with
629 additions
and
250 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| using FluentResults; | ||
| using TNRD.Zeepkist.GTR.Backend.Steam.Resources; | ||
| using TNRD.Zeepkist.GTR.Backend.Workshop; | ||
|
|
||
| namespace TNRD.Zeepkist.GTR.Backend.Levels.Jobs; | ||
|
|
||
| public record DownloadResult(IEnumerable<PublishedFileDetails> PublishedFileDetails, Result<WorkshopDownloads> Result); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,77 +1,44 @@ | ||
| using JetBrains.Annotations; | ||
| using Microsoft.Extensions.Options; | ||
| using TNRD.Zeepkist.GTR.Backend.Hashing; | ||
| using TNRD.Zeepkist.GTR.Backend.Levels.Items; | ||
| using TNRD.Zeepkist.GTR.Backend.Levels.Metadata; | ||
| using TNRD.Zeepkist.GTR.Backend.Steam; | ||
| using TNRD.Zeepkist.GTR.Backend.Steam.Resources; | ||
| using TNRD.Zeepkist.GTR.Backend.Workshop; | ||
| using TNRD.Zeepkist.GTR.Backend.Zeeplevel; | ||
|
|
||
| namespace TNRD.Zeepkist.GTR.Backend.Levels.Jobs; | ||
|
|
||
| public class PartialWorkshopScanJob : WorkshopScanJob | ||
| { | ||
| private readonly IPublishedFileServiceApi _publishedFileServiceApi; | ||
| private readonly SteamOptions _steamOptions; | ||
|
|
||
| public PartialWorkshopScanJob( | ||
| ILogger<PartialWorkshopScanJob> logger, | ||
| IHashService hashService, | ||
| ILevelService levelService, | ||
| ILevelMetadataService levelMetadataService, | ||
| ILevelItemsService levelItemsService, | ||
| IWorkshopService workshopService, | ||
| IZeeplevelService zeeplevelService, | ||
| IPublishedFileServiceApi publishedFileServiceApi, | ||
| IOptions<SteamOptions> steamOptions) | ||
| : base( | ||
| logger, | ||
| hashService, | ||
| levelService, | ||
| levelMetadataService, | ||
| levelItemsService, | ||
| workshopService, | ||
| zeeplevelService) | ||
| WorkshopLister workshopLister, | ||
| WorkshopDownloader workshopDownloader, | ||
| WorkshopProcessor workshopProcessor, | ||
| IWorkshopService workshopService) | ||
| : base(logger, workshopLister, workshopDownloader, workshopProcessor, workshopService) | ||
| { | ||
| _publishedFileServiceApi = publishedFileServiceApi; | ||
| _steamOptions = steamOptions.Value; | ||
| } | ||
|
|
||
| [UsedImplicitly] | ||
| public async Task ExecuteAsync() | ||
| { | ||
| if (FullWorkshopScanJob.IsRunning) | ||
| { | ||
| Logger.LogInformation("Skipping partial workshop scan because full workshop scan is running"); | ||
| return; | ||
| } | ||
|
|
||
| await ScanByPublicationDate(); | ||
| await ScanByUpdatedDate(); | ||
| } | ||
|
|
||
| private async Task ScanByPublicationDate() | ||
| { | ||
| string cursor = "*"; | ||
|
|
||
| for (int i = 0; i < 5; i++) | ||
| { | ||
| QueryFilesResult result | ||
| = await _publishedFileServiceApi.QueryFilesByPublicationDate(_steamOptions.ApiKey, cursor); | ||
|
|
||
| cursor = result.Response.NextCursor; | ||
|
|
||
| await ProcessPage(result); | ||
| } | ||
| Logger.LogInformation("Starting partial workshop scan by publication date"); | ||
| await Run(WorkshopLister.QueryType.ByPublicationDate, 5); | ||
| Logger.LogInformation("Finished partial workshop scan by publication date"); | ||
| } | ||
|
|
||
| private async Task ScanByUpdatedDate() | ||
| { | ||
| string cursor = "*"; | ||
|
|
||
| for (int i = 0; i < 5; i++) | ||
| { | ||
| QueryFilesResult result | ||
| = await _publishedFileServiceApi.QueryFilesByUpdatedDate(_steamOptions.ApiKey, cursor); | ||
|
|
||
| cursor = result.Response.NextCursor; | ||
|
|
||
| await ProcessPage(result); | ||
| } | ||
| Logger.LogInformation("Starting partial workshop scan by updated date"); | ||
| await Run(WorkshopLister.QueryType.ByUpdatedDate, 5); | ||
| Logger.LogInformation("Finished partial workshop scan by updated date"); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| using FluentResults; | ||
| using TNRD.Zeepkist.GTR.Backend.Steam.Resources; | ||
| using TNRD.Zeepkist.GTR.Backend.Workshop; | ||
|
|
||
| namespace TNRD.Zeepkist.GTR.Backend.Levels.Jobs; | ||
|
|
||
| public class WorkshopDownloader | ||
| { | ||
| private const int ItemsPerChunk = 10; | ||
| private const int MaxConcurrency = 25; | ||
|
|
||
| private readonly ILogger<WorkshopDownloader> _logger; | ||
| private readonly IWorkshopService _workshopService; | ||
|
|
||
| public WorkshopDownloader(ILogger<WorkshopDownloader> logger, IWorkshopService workshopService) | ||
| { | ||
| _logger = logger; | ||
| _workshopService = workshopService; | ||
| } | ||
|
|
||
| public async Task<DownloadResult[]> Download(IEnumerable<PublishedFileDetails> publishedFileDetails) | ||
| { | ||
| List<PublishedFileDetails[]> chunks = publishedFileDetails.Chunk(ItemsPerChunk).ToList(); | ||
| SemaphoreSlim semaphore = new(MaxConcurrency, MaxConcurrency); | ||
| List<Task<DownloadResult>> tasks = new(); | ||
|
|
||
| for (int i = 0; i < chunks.Count; i++) | ||
| { | ||
| PublishedFileDetails[] chunk = chunks[i]; | ||
| int index = i; | ||
| tasks.Add(DownloadWorkshopItems(chunk, semaphore, index)); | ||
| } | ||
|
|
||
| _logger.LogInformation("Waiting for all ({Amount}) tasks to complete (this may take a while)", chunks.Count); | ||
| DownloadResult[] results = await Task.WhenAll(tasks); | ||
| _logger.LogInformation("All tasks completed"); | ||
| return results; | ||
| } | ||
|
|
||
| private async Task<DownloadResult> DownloadWorkshopItems( | ||
| IEnumerable<PublishedFileDetails> publishedFileDetails, | ||
| SemaphoreSlim semaphore, | ||
| int index) | ||
| { | ||
| await semaphore.WaitAsync(); | ||
| try | ||
| { | ||
| _logger.LogInformation("Starting workshop download {Index}", index); | ||
| Result<WorkshopDownloads> result = | ||
| await _workshopService.DownloadWorkshopItems( | ||
| publishedFileDetails.Select(x => ulong.Parse(x.PublishedFileId))); | ||
| _logger.LogInformation("Finished workshop download {Index}", index); | ||
| return new DownloadResult(publishedFileDetails, result); | ||
| } | ||
| catch (Exception e) | ||
| { | ||
| _logger.LogError(e, "Failed to download workshop items"); | ||
| return new DownloadResult(publishedFileDetails, Result.Fail(new ExceptionalError(e))); | ||
| } | ||
| finally | ||
| { | ||
| semaphore.Release(); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.