-
Notifications
You must be signed in to change notification settings - Fork 0
172 lines (147 loc) · 7.09 KB
/
scrape.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
name: Daily Scrape
on:
workflow_dispatch:
schedule:
- cron: "30 20 * * *"
issues:
types:
- opened
jobs:
scrape:
runs-on: ubuntu-latest
permissions:
contents: write
issues: write
steps:
- name: Check to proceed
id: check_to_proceed
shell: pwsh
run: |
$proceed = "false"
$chartSyncRequest = "false"
if ("${{ github.event_name }}" -eq "workflow_dispatch") {
$proceed = "true"
} elseif ("${{ github.event_name }}" -eq "schedule") {
$proceed = "true"
} elseif ("${{ github.event_name }}" -eq "issues" -and "${{ github.event.action }}" -eq "opened") {
if (${{ toJson(github.event.issue.body) }} -eq "### Issue Type\n\nSync Request") {
$proceed = "true"
$chartSyncRequest = "true"
}
}
"proceed=$proceed" >> $env:GITHUB_OUTPUT
"chartSyncRequest=$chartSyncRequest" >> $env:GITHUB_OUTPUT
- name: Checkout code
if: steps.check_to_proceed.outputs.proceed == 'true'
uses: actions/checkout@v4
- name: Setup .NET
if: steps.check_to_proceed.outputs.proceed == 'true'
uses: actions/setup-dotnet@v4
with:
dotnet-version: '8.x'
- name: Restore NuGet packages
if: steps.check_to_proceed.outputs.proceed == 'true'
shell: bash
run: |
dotnet restore
- name: Build solution
if: steps.check_to_proceed.outputs.proceed == 'true'
shell: bash
run: |
dotnet build -c Release
- name: Install Playwright
if: steps.check_to_proceed.outputs.proceed == 'true'
shell: pwsh
run: |
$playwright = Get-ChildItem -File Microsoft.Playwright.dll -Path . -Recurse
$installer = "$($playwright[0].Directory.FullName)/playwright.ps1"
& "$installer" install
- name: Run Melon app - Melon Top 100
if: steps.check_to_proceed.outputs.proceed == 'true'
shell: pwsh
run: |
$date = (Get-Date).ToUniversalTime().AddHours(9).ToString("yyyyMMdd")
# $result = dotnet run --project ./samples/MelonChart.ConsoleApp/ -- -c Top100 --json | ConvertFrom-Json
$result = dotnet run --project ./samples/MelonChart.ConsoleApp/ -- -c Top100 --json
mkdir -p ./data
pushd ./data
# $result | ConvertTo-Json -Depth 100 | Out-File -FilePath "top100-$date.json" -Force
$result | Out-File -FilePath "top100-$date.json" -Force
popd
- name: Run Spotify app - Melon Top 100
if: steps.check_to_proceed.outputs.proceed == 'true'
shell: pwsh
run: |
$appsettings = Get-Content -Path ./samples/SpotifyPlaylist.ConsoleApp/appsettings.Development.sample.json | ConvertFrom-Json
$appsettings.Azure.APIM.BaseUrl = "${{ secrets.APIM_BASE_URL }}"
$appsettings.Azure.APIM.SubscriptionKey = "${{ secrets.APIM_SUBSCRIPTION_KEY }}"
$appsettings | ConvertTo-Json -Depth 100 | Out-File -FilePath ./samples/SpotifyPlaylist.ConsoleApp/appsettings.Development.json -Force
$date = (Get-Date).ToUniversalTime().AddHours(9).ToString("yyyyMMdd")
# $result = dotnet run --project ./samples/SpotifyPlaylist.ConsoleApp/ -- -t melon -s "../../data/top100-$date.json" --json | ConvertFrom-Json
$result = dotnet run --project ./samples/SpotifyPlaylist.ConsoleApp/ -- -t melon -s "../../data/top100-$date.json" --json
mkdir -p ./data
pushd ./data
# $result[$($result.indexOf("{"))..$($result.Length-1)] | ConvertTo-Json -Depth 100 | Out-File -FilePath "spotify100-$date.json" -Force
$result[$($result.indexOf("{"))..$($result.Length-1)] | Out-File -FilePath "spotify100-$date.json" -Force
popd
- name: Run Spotify app - Billboard Hot 100
if: steps.check_to_proceed.outputs.proceed == 'true'
shell: pwsh
run: |
$appsettings = Get-Content -Path ./samples/SpotifyPlaylist.ConsoleApp/appsettings.Development.sample.json | ConvertFrom-Json
$appsettings.Azure.APIM.BaseUrl = "${{ secrets.APIM_BASE_URL }}"
$appsettings.Azure.APIM.SubscriptionKey = "${{ secrets.APIM_SUBSCRIPTION_KEY }}"
$appsettings | ConvertTo-Json -Depth 100 | Out-File -FilePath ./samples/SpotifyPlaylist.ConsoleApp/appsettings.Development.json -Force
$date = (Get-Date).ToUniversalTime().AddHours(9).ToString("yyyyMMdd")
# $result = dotnet run --project ./samples/SpotifyPlaylist.ConsoleApp/ -- -t spotify -s ${{ secrets.SPOTIFY_BILLBOARD_HOT100_PLAYLIST_ID }} --json | ConvertFrom-Json
$result = dotnet run --project ./samples/SpotifyPlaylist.ConsoleApp/ -- -t spotify -s ${{ secrets.SPOTIFY_BILLBOARD_HOT100_PLAYLIST_ID }} --json
mkdir -p ./data
pushd ./data
# $result[$($result.indexOf("{"))..$($result.Length-1)] | ConvertTo-Json -Depth 100 | Out-File -FilePath "billboard100-$date.json" -Force
$result[$($result.indexOf("{"))..$($result.Length-1)] | Out-File -FilePath "billboard100-$date.json" -Force
popd
- name: Upload data
if: steps.check_to_proceed.outputs.proceed == 'true'
uses: stefanzweifel/git-auto-commit-action@v5
with:
commit_message: "Update data"
branch: "main"
commit_user_name: "GitHub Actions"
commit_user_email: "scraper+github-actions[bot]@users.noreply.github.com"
commit_author: "GitHub Actions <scraper+github-actions[bot]@users.noreply.github.com>"
- name: Check missing tracks
id: missing_tracks
if: steps.check_to_proceed.outputs.proceed == 'true'
shell: pwsh
run: |
$date = (Get-Date).ToUniversalTime().AddHours(9).ToString("yyyyMMdd")
$collection = Get-Content "./data/spotify100-$date.json" | ConvertFrom-Json
$missingTracks = ($collection.missingTracks.Count -gt 0).ToString().ToLowerInvariant()
"exist=$missingTracks" >> $env:GITHUB_OUTPUT
- name: Issue missing tracks
if: |
steps.check_to_proceed.outputs.proceed == 'true' &&
steps.missing_tracks.outputs.exist == 'true'
shell: pwsh
env:
GH_TOKEN: ${{ github.token }}
run: |
$date = (Get-Date).ToUniversalTime().AddHours(9).ToString("yyyyMMdd")
$collection = Get-Content "./data/spotify100-$date.json" | ConvertFrom-Json
$tracks = $collection.missingTracks | ConvertTo-Json -Depth 100
gh issue create `
--title "Missing tracks in Spotify playlist" `
--body "The following tracks are missing in the Spotify playlist:`r`n```````jsonr`n$tracks`r`n```````r`n" `
--label missing-tracks
- name: Close sync request
if: |
steps.check_to_proceed.outputs.proceed == 'true' &&
steps.check_to_proceed.outputs.chartSyncRequest == 'true'
shell: pwsh
env:
GH_TOKEN: ${{ github.token }}
run: |
$issueNumber = ${{ github.event.issue.number }}
gh issue close $issueNumber `
--reason completed `
--comment "The chart has been synced up-to-date."