From 31e3e752961fdadbfb6b6afdc4978c9a1e3b1632 Mon Sep 17 00:00:00 2001 From: Jonas Date: Thu, 9 Jan 2025 22:45:15 +0100 Subject: [PATCH] wip: improves contributions loading logic (#324) --- app/Livewire/Monitors/ContributionsView.php | 97 ++++--- app/Models/Contribution.php | 26 +- app/Models/Monitor.php | 2 + app/Traits/ContributionCollector.php | 251 +++++++++++++----- ...025_01_09_200408_create_authors_table.php} | 4 +- ..._09_200409_create_contributions_table.php} | 16 +- ...00410_create_contribution_author_table.php | 26 ++ .../monitors/contributions-view.blade.php | 64 ++++- 8 files changed, 368 insertions(+), 118 deletions(-) rename database/migrations/{2024_11_30_232408_create_authors_table.php => 2025_01_09_200408_create_authors_table.php} (80%) rename database/migrations/{2024_11_30_232409_create_contributions_table.php => 2025_01_09_200409_create_contributions_table.php} (58%) create mode 100644 database/migrations/2025_01_09_200410_create_contribution_author_table.php diff --git a/app/Livewire/Monitors/ContributionsView.php b/app/Livewire/Monitors/ContributionsView.php index ed2a56c..3a1df1f 100644 --- a/app/Livewire/Monitors/ContributionsView.php +++ b/app/Livewire/Monitors/ContributionsView.php @@ -2,62 +2,95 @@ namespace App\Livewire\Monitors; -use App\Models\Contribution; use App\Models\Monitor; -use App\Traits\ContributionCollector; -use Carbon\Carbon; -use Exception; +use App\Models\Contribution; +use App\Models\Author; use Livewire\Component; +use Log; +use Exception; +use Livewire\Attributes\On; class ContributionsView extends Component { - use ContributionCollector; - public Monitor $monitor; public $contributions = []; - public $contributionId; + public $nextRootCursor = null; + public $nextCursor = null; + public bool $hasMoreRepositories = false; + public bool $hasMoreCommits = false; + public bool $loading = false; + public ?string $currentRepositoryName = null; + public ?string $error = null; - public $commit; - public function mount(Monitor $monitor, $contributionId = null) + public function mount(Monitor $monitor) { $this->monitor = $monitor; - $this->contributionId = $contributionId; - $this->loadContributions(); - $this-> commit = $this->calculateContributions(); + $this->loadNext(); } - public function loadContributions() + public function loadNext() { try { - $this->contributions = $this->collect_contributions($this->monitor); + $this->loading = true; + $this->error = null; + + $result = $this->monitor->collect_contributions($this->nextRootCursor, $this->nextCursor); + + if (!$result) { + throw new Exception('No response from API server'); + } + + Log::info('API Response:', [ + 'hasMoreCommits' => $result['has_more_commits'] ?? false, + 'hasMoreRepositories' => $result['has_more_repositories'] ?? false, + 'nextRootCursor' => $result['next_root_cursor'] ?? null, + 'nextCursor' => $result['next_cursor'] ?? null, + 'currentRepo' => $result['current_repository_name'] ?? 'unknown', + 'contributionsCount' => count($result['contributions'] ?? []) + ]); + + if (!empty($result['contributions'])) { + $contributionIds = collect($result['contributions'])->pluck('id'); + $loadedContributions = Contribution::with('authors') + ->whereIn('id', $contributionIds) + ->orderBy('committed_date', 'desc') + ->get(); + + $this->contributions = array_merge($this->contributions, $loadedContributions->all()); + } + + $this->nextRootCursor = $result['next_root_cursor']; + $this->nextCursor = $result['next_cursor']; + $this->hasMoreRepositories = $result['has_more_repositories'] ?? false; + $this->hasMoreCommits = $result['has_more_commits'] ?? false; + $this->currentRepositoryName = $result['current_repository_name']; + } catch (Exception $e) { - $this->addError('contributions', $e->getMessage()); + $this->error = "Error loading contributions. Please try again."; + $this->hasMoreCommits = false; + $this->hasMoreRepositories = false; + } finally { + $this->loading = false; } } - - public function calculateContributions() + public function loadMore() { - $commits = Contribution::selectRaw('authors.name, COUNT(contributions.id) as commit_count') - ->leftJoin('authors', 'contributions.author_id', '=', 'authors.id') - ->where('committed_date', '>=', Carbon::now()->subDays(14)) - ->where('committed_date', '<', Carbon::now()) - ->groupBy('authors.name') - ->get(); - - return $commits; + if (!$this->loading) { + if ($this->hasMoreCommits) { + $this->loadNext(); + } else if ($this->hasMoreRepositories) { + $this->nextCursor = null; + $this->loadNext(); + } + } } - public function placeholder() + public function retry() { - return <<<'HTML' -
-
-
- HTML; + $this->loadNext(); } - public function render() { return view('livewire.monitors.contributions-view'); diff --git a/app/Models/Contribution.php b/app/Models/Contribution.php index ae480c3..dcf767c 100644 --- a/app/Models/Contribution.php +++ b/app/Models/Contribution.php @@ -3,27 +3,41 @@ namespace App\Models; use Illuminate\Database\Eloquent\Model; -use Illuminate\Database\Eloquent\Relations\BelongsTo; +use Illuminate\Database\Eloquent\Relations\BelongsToMany; class Contribution extends Model { + public $incrementing = false; + protected $keyType = 'string'; + protected $fillable = [ + 'id', 'commit_url', 'message_headline', 'message_body', 'additions', 'deletions', 'changed_files', - 'committed_date', - 'author_id', + 'committed_date' ]; protected $casts = [ - 'committed_date' => 'datetime', + 'committed_date' => 'datetime' ]; - public function author(): BelongsTo + public function authors(): BelongsToMany + { + return $this->belongsToMany(Author::class, 'contribution_author'); + } + + protected static function booted() { - return $this->belongsTo(Author::class); + static::creating(function ($contribution) { + if (!$contribution->id) { + if (preg_match('/\/commit\/([a-f0-9]{40})$/', $contribution->commit_url, $matches)) { + $contribution->id = $matches[1]; + } + } + }); } } diff --git a/app/Models/Monitor.php b/app/Models/Monitor.php index eb5a589..a526d7f 100644 --- a/app/Models/Monitor.php +++ b/app/Models/Monitor.php @@ -2,6 +2,7 @@ namespace App\Models; +use App\Traits\ContributionCollector; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\BelongsToMany; @@ -66,6 +67,7 @@ class Monitor extends Model { use HasFactory; + use ContributionCollector; protected $fillable = [ "project_url", diff --git a/app/Traits/ContributionCollector.php b/app/Traits/ContributionCollector.php index a022ae6..238f831 100644 --- a/app/Traits/ContributionCollector.php +++ b/app/Traits/ContributionCollector.php @@ -11,91 +11,222 @@ trait ContributionCollector { - /** - * @throws Exception - */ - public function collect_contributions(Monitor $monitor) + private $collectedContributions = []; + + public function collect_contributions(?string $rootContinueAfter = null, ?string $continueAfter = null) { - $url = $monitor->type == 'ORGANIZATION' - ? $_ENV['APP_SERVICE_URL'] . '/v1/github/orgs/' . $monitor->organization_name . '/projects/' . $monitor->project_identification . '/repositories/contributions?rootPageSize=25&pageSize=50' - : $_ENV['APP_SERVICE_URL'] . '/v1/github/users/' . $monitor->login_name . '/projects/' . $monitor->project_identification . '/repositories/contributions?rootPageSize=25&pageSize=50'; + try { + $baseUrl = $this->type == 'ORGANIZATION' + ? $_ENV['APP_SERVICE_URL'] . '/v1/github/orgs/' . $this->organization_name . '/projects/' . $this->project_identification . '/repositories/contributions' + : $_ENV['APP_SERVICE_URL'] . '/v1/github/users/' . $this->login_name . '/projects/' . $this->project_identification . '/repositories/contributions'; - Log::info('Fetching contributions from URL:', ['url' => $url]); + // Build URL with both pagination parameters + $url = $baseUrl . '?rootPageSize=1&pageSize=50'; + if ($rootContinueAfter) { + $url .= '&rootContinueAfter=' . $rootContinueAfter; + } + if ($continueAfter) { + $url .= '&continueAfter=' . $continueAfter; + } - try { - // reset commits -> prevent duplicates - Contribution::truncate(); + Log::info('Fetching contributions:', [ + 'url' => $url, + 'rootContinueAfter' => $rootContinueAfter, + 'continueAfter' => $continueAfter + ]); - $response = Http::withHeaders([ + $response = Http::timeout(15)->withHeaders([ 'content-type' => 'application/json', 'Accept' => 'text/plain', - 'Authorization' => 'Bearer ' . $monitor->pat_token + 'Authorization' => 'Bearer ' . $this->pat_token ])->get($url); - Log::debug('API Response:', [ - 'status' => $response->status(), - 'body' => $response->body() + if (!$response->successful()) { + Log::error('Failed to fetch contributions:', [ + 'status' => $response->status(), + 'body' => $response->body() + ]); + return [ + 'contributions' => [], + 'next_root_cursor' => null, + 'next_cursor' => null, + 'has_more_repositories' => false, + 'has_more_commits' => false, + 'current_repository_name' => null + ]; + } + + $responseData = $response->json(); + Log::info('Raw API Response:', [ + 'response' => $responseData ]); + + $data = $responseData['data'] ?? null; + + if (!$data) { + Log::error('No data in response:', [ + 'response' => $responseData + ]); + return [ + 'contributions' => [], + 'next_root_cursor' => null, + 'next_cursor' => null, + 'has_more_repositories' => false, + 'has_more_commits' => false, + 'current_repository_name' => null + ]; + } + + $repositories = $data['organization']['projectV2']['repositories']['nodes'] ?? []; + $repoPageInfo = $data['organization']['projectV2']['repositories']['pageInfo'] ?? null; + + if (empty($repositories)) { + return [ + 'contributions' => [], + 'next_root_cursor' => null, + 'next_cursor' => null, + 'has_more_repositories' => false, + 'has_more_commits' => false, + 'current_repository_name' => null + ]; + } - if ($response->successful()) { - $repositories = $response->json()['data']['organization']['projectV2']['repositories']['nodes'] ?? []; - $contributions = []; + $repo = $repositories[0]; + $currentRepoName = $repo['name'] ?? 'unknown'; + $contributions = []; + + if (!isset($repo['defaultBranchRef']) || !isset($repo['defaultBranchRef']['target']['history']['edges'])) { + // If repository has no commits, move to next repository + return [ + 'contributions' => [], + 'next_root_cursor' => $repoPageInfo['endCursor'], + 'next_cursor' => null, + 'has_more_repositories' => $repoPageInfo['hasNextPage'], + 'has_more_commits' => false, + 'current_repository_name' => $currentRepoName + ]; + } - foreach ($repositories as $repo) { - $commits = $repo['defaultBranchRef']['target']['history']['edges'] ?? []; + $commits = $repo['defaultBranchRef']['target']['history']['edges']; + $commitPageInfo = $repo['defaultBranchRef']['target']['history']['pageInfo']; - foreach ($commits as $commitData) { - $commitNode = $commitData['node'] ?? null; - if ($commitNode) { - $authors = $commitNode['authors']['nodes'] ?? []; + foreach ($commits as $commitData) { + $commitNode = $commitData['node'] ?? null; + if (!$commitNode) { + Log::warning('Empty commit node'); + continue; + } - Log::debug('Authors for commit:', [ - 'commit_url' => $commitNode['commitUrl'], - 'authors' => $authors - ]); + $authors = $commitNode['authors']['nodes'] ?? []; + if (empty($authors)) { + Log::warning('No authors for commit', [ + 'commit' => $commitNode['oid'] ?? 'unknown' + ]); + continue; + } + + try { + $commitHash = $commitNode['oid'] ?? null; - $contribution = Contribution::create([ - 'commit_url' => $commitNode['commitUrl'], - 'message_headline' => strip_tags($commitNode['messageHeadlineHTML']), - 'message_body' => strip_tags($commitNode['messageBodyHTML']), - 'additions' => $commitNode['additions'], - 'deletions' => $commitNode['deletions'], - 'changed_files' => $commitNode['changedFilesIfAvailable'], - 'committed_date' => $commitNode['committedDate'], - 'author_id' => $authors ? $this->getOrCreateAuthor($authors[0]) : null, + if (!$commitHash) { + Log::error('No commit hash (oid) found:', [ + 'commit' => $commitNode + ]); + continue; + } + + Log::info('Creating contribution:', [ + 'hash' => $commitHash, + 'headline' => $commitNode['messageHeadlineHTML'] ?? '', + 'authors_count' => count($authors) + ]); + + $contribution = Contribution::firstOrCreate( + ['id' => $commitHash], + [ + 'commit_url' => $commitNode['commitUrl'], + 'message_headline' => strip_tags($commitNode['messageHeadlineHTML']), + 'message_body' => strip_tags($commitNode['messageBodyHTML']), + 'additions' => $commitNode['additions'], + 'deletions' => $commitNode['deletions'], + 'changed_files' => $commitNode['changedFilesIfAvailable'], + 'committed_date' => $commitNode['committedDate'] + ] + ); + + foreach ($authors as $authorData) { + preg_match('/\/u\/(\d+)/', $authorData['avatarUrl'], $matches); + $githubUserId = $matches[1] ?? null; + + if (!$githubUserId) { + Log::error('Failed to extract GitHub user ID:', [ + 'avatar_url' => $authorData['avatarUrl'] ]); + continue; + } + + Log::info('Creating author:', [ + 'id' => $githubUserId, + 'name' => $authorData['name'] + ]); - $contribution->authors = $authors; - $contributions[] = $contribution; + if ($githubUserId) { + $author = Author::updateOrCreate( + ['id' => $githubUserId], + [ + 'name' => $authorData['name'], + 'email' => $authorData['email'], + 'avatar_url' => $authorData['avatarUrl'] + ] + ); + + $contribution->authors()->syncWithoutDetaching([$githubUserId]); } } + + $contributions[] = $contribution; + Log::info('Contribution created successfully', [ + 'hash' => $commitHash, + 'total_contributions' => count($contributions) + ]); + } catch (Exception $e) { + Log::error('Failed to create contribution:', [ + 'error' => $e->getMessage(), + 'commit' => $commitNode + ]); + continue; } + } - return $contributions; - } else { - Log::error('Error fetching contributions:', [ - 'status' => $response->status(), - 'body' => $response->body() - ]); - throw new Exception("Fehler beim Abrufen der Beiträge: " . $response->body()); + // Determine if we should continue with commits or move to next repository + $hasMoreCommits = $commitPageInfo['hasNextPage'] ?? false; + $nextCursor = $commitPageInfo['endCursor'] ?? null; + $hasMoreRepositories = $repoPageInfo['hasNextPage'] ?? false; + + // If no more commits in current repository, move to next repository + if (!$hasMoreCommits) { + $nextCursor = null; + if ($hasMoreRepositories) { + $nextRootCursor = $repoPageInfo['endCursor']; + } } + + return [ + 'contributions' => $contributions, + 'next_root_cursor' => !$hasMoreCommits && $hasMoreRepositories ? $repoPageInfo['endCursor'] : null, + 'next_cursor' => $hasMoreCommits ? $nextCursor : null, + 'has_more_repositories' => $hasMoreRepositories, + 'has_more_commits' => $hasMoreCommits, + 'current_repository_name' => $currentRepoName + ]; + } catch (Exception $e) { - Log::error('Exception in collect_contributions:', [ - 'message' => $e->getMessage(), + Log::error('Failed to collect contributions:', [ + 'error' => $e->getMessage(), 'trace' => $e->getTraceAsString() ]); + throw $e; } } - - private function getOrCreateAuthor(array $authorData) - { - return Author::firstOrCreate( - ['email' => $authorData['email']], - [ - 'name' => $authorData['name'], - 'avatar_url' => $authorData['avatarUrl'], - ] - )->id; - } } diff --git a/database/migrations/2024_11_30_232408_create_authors_table.php b/database/migrations/2025_01_09_200408_create_authors_table.php similarity index 80% rename from database/migrations/2024_11_30_232408_create_authors_table.php rename to database/migrations/2025_01_09_200408_create_authors_table.php index 7f3127a..98f1d0f 100644 --- a/database/migrations/2024_11_30_232408_create_authors_table.php +++ b/database/migrations/2025_01_09_200408_create_authors_table.php @@ -9,10 +9,10 @@ public function up(): void { Schema::create('authors', function (Blueprint $table) { - $table->id(); + $table->unsignedBigInteger('id')->primary(); // GitHub user ID $table->string('name'); $table->string('email')->unique(); - $table->string('avatar_url')->nullable(); + $table->string('avatar_url'); $table->timestamps(); }); } diff --git a/database/migrations/2024_11_30_232409_create_contributions_table.php b/database/migrations/2025_01_09_200409_create_contributions_table.php similarity index 58% rename from database/migrations/2024_11_30_232409_create_contributions_table.php rename to database/migrations/2025_01_09_200409_create_contributions_table.php index a0549fa..c2ffbc6 100644 --- a/database/migrations/2024_11_30_232409_create_contributions_table.php +++ b/database/migrations/2025_01_09_200409_create_contributions_table.php @@ -9,19 +9,21 @@ public function up(): void { Schema::create('contributions', function (Blueprint $table) { - $table->id(); - $table->foreignId('author_id')->constrained()->cascadeOnDelete(); + $table->string('id')->primary(); // GitHub commit hash $table->string('commit_url'); - $table->text('message_headline'); + $table->string('message_headline'); $table->text('message_body')->nullable(); - $table->integer('additions')->default(0); - $table->integer('deletions')->default(0); - $table->integer('changed_files')->default(0); - $table->timestamp('committed_date')->nullable(); + $table->integer('additions'); + $table->integer('deletions'); + $table->integer('changed_files'); + $table->timestamp('committed_date'); $table->timestamps(); }); } + /** + * Reverse the migrations. + */ public function down(): void { Schema::dropIfExists('contributions'); diff --git a/database/migrations/2025_01_09_200410_create_contribution_author_table.php b/database/migrations/2025_01_09_200410_create_contribution_author_table.php new file mode 100644 index 0000000..98b70de --- /dev/null +++ b/database/migrations/2025_01_09_200410_create_contribution_author_table.php @@ -0,0 +1,26 @@ +string('contribution_id'); + $table->unsignedBigInteger('author_id'); + $table->timestamps(); + + $table->foreign('contribution_id')->references('id')->on('contributions')->onDelete('cascade'); + $table->foreign('author_id')->references('id')->on('authors')->onDelete('cascade'); + $table->primary(['contribution_id', 'author_id']); + }); + } + + public function down(): void + { + Schema::dropIfExists('contribution_author'); + } +}; diff --git a/resources/views/livewire/monitors/contributions-view.blade.php b/resources/views/livewire/monitors/contributions-view.blade.php index 894dbea..28a58e4 100644 --- a/resources/views/livewire/monitors/contributions-view.blade.php +++ b/resources/views/livewire/monitors/contributions-view.blade.php @@ -1,22 +1,51 @@
- + {{ strtoupper($monitor->type == 'USER' ? $monitor->login_name : $monitor->organization_name) }} / {{ strtoupper($monitor->title) }}
+ + @if($error) +
+

{{ $error }}

+ + Retry Loading + +
+ @endif + + + + -
+
@forelse($contributions as $contribution) -
+

{{ $contribution->message_headline }}

-

{{ $contribution->message_body }}

+

{{ $contribution->message_body }}

{{ is_string($contribution->committed_date) ? \Carbon\Carbon::parse($contribution->committed_date)->format('M d, Y H:i') : $contribution->committed_date->format('M d, Y H:i') }} @@ -29,24 +58,37 @@ {{ $contribution->changed_files }} changed files
- @if(isset($contribution->authors) && is_array($contribution->authors) && count($contribution->authors) > 0) + @if($contribution->authors->isNotEmpty()) @foreach($contribution->authors as $author) - - {{ $author['name'] }} + + {{ $author->name }} @endforeach @else - Keine Autoren verfügbar + No authors available @endif
@empty -
- Keine Beiträge gefunden +
+ No contributions found
@endforelse
+ + + @if($hasMoreRepositories || $hasMoreCommits) +
+ + @if($hasMoreCommits) + Load more commits from {{ $currentRepositoryName }} + @else + Load next repository + @endif + +
+ @endif