Skip to content

Commit

Permalink
indexer_scheduler limiter usage change
Browse files Browse the repository at this point in the history
  • Loading branch information
brianolson committed Dec 17, 2024
1 parent 13a6d4a commit 62ddc08
Showing 1 changed file with 9 additions and 4 deletions.
13 changes: 9 additions & 4 deletions indexer/crawler.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,10 +145,15 @@ func (c *CrawlDispatcher) mainLoop() {
pdsID := crawlJob.act.PDS
limiter := c.repoFetcher.GetOrCreateLimiterLazy(pdsID)
now := time.Now()
res := limiter.ReserveN(now, 1)
wouldDelay := res.DelayFrom(now)
res.Cancel()
crawlJob.eligibleTime = now.Add(wouldDelay)
avail := limiter.TokensAt(now)
if avail > 1 {
crawlJob.eligibleTime = now
} else {
rate := float64(limiter.Limit())
need := 1 - avail
wouldDelay := time.Duration(float64(time.Second) * need / rate)
crawlJob.eligibleTime = now.Add(wouldDelay)
}
// put crawl job on heap sorted by eligible time
c.enheapJob(crawlJob)
c.dequeueJob(crawlJob)
Expand Down

0 comments on commit 62ddc08

Please sign in to comment.