gitlyx / github.v
479 lines · 439 sloc · 14.39 KB · f01d3f77ed35a21882e0e1e97310a13b0a9dfd8b
Raw
1// Copyright (c) 2020-2021 Alexander Medvednikov. All rights reserved.
2// Use of this source code is governed by a GPL license that can be found in the LICENSE file.
3module main
4
5import veb
6import x.json2 as json
7import net.http
8import time
9import git
10// import veb.auth as oauth
11import veb.oauth
12
13struct GitHubUser {
14 username string @[json: 'login']
15 name string
16 email string
17 avatar string @[json: 'avatar_url']
18}
19
20struct GitHubIssueAuthor {
21 login string
22}
23
24struct GitHubPullRequestRef {
25 url string
26}
27
28struct GitHubPullRequestBranch {
29 ref_name string @[json: 'ref']
30 sha string
31}
32
33struct GitHubLabel {
34 name string
35 color string
36 description string
37}
38
39struct GitHubRepoInfo {
40 description string
41}
42
43struct GitHubContributor {
44 login string
45 avatar_url string
46 type_ string @[json: 'type']
47 html_url string
48 id int
49}
50
51struct GitHubIssue {
52 number int
53 title string
54 body string
55 state string
56 created_at string
57 user GitHubIssueAuthor
58 pull_request GitHubPullRequestRef
59 labels []GitHubLabel
60}
61
62struct GitHubPullRequest {
63 number int
64 title string
65 body string
66 state string
67 created_at string
68 user GitHubIssueAuthor
69 head GitHubPullRequestBranch
70 base GitHubPullRequestBranch
71}
72
73fn parse_github_timestamp(s string) int {
74 if s == '' {
75 return int(time.now().unix())
76 }
77 t := time.parse_iso8601(s) or { return int(time.now().unix()) }
78 return int(t.unix())
79}
80
81fn parse_github_owner_repo(clone_url string) ?(string, string) {
82 mut s := clone_url.trim_space()
83 for prefix in ['https://', 'http://', 'git@'] {
84 if s.starts_with(prefix) {
85 s = s[prefix.len..]
86 break
87 }
88 }
89 s = s.trim_string_left('github.com')
90 s = s.trim_left(':/')
91 s = s.trim_string_right('.git')
92 s = s.trim('/')
93 parts := s.split('/')
94 if parts.len < 2 || parts[0] == '' || parts[1] == '' {
95 return none
96 }
97 return parts[0], parts[1]
98}
99
100// Returns the local user id for a GitHub login, creating an unregistered
101// "shadow" user (no password, no email, just the username and GitHub avatar)
102// when one does not yet exist.
103fn (mut app App) find_or_create_github_shadow_user(github_login string) !int {
104 if u := app.get_user_by_username(github_login) {
105 return u.id
106 }
107 user := User{
108 username: github_login
109 github_username: github_login
110 is_github: true
111 is_registered: false
112 avatar: 'https://github.com/${github_login}.png'
113 created_at: time.now()
114 }
115 app.add_user(user)!
116 created := app.get_user_by_username(github_login) or {
117 return error('shadow user not found after insert: ${github_login}')
118 }
119 return created.id
120}
121
122// fetch_github_repo_description returns the GitHub description for a repo, or
123// an empty string if it cannot be retrieved.
124fn fetch_github_repo_description(clone_url string) string {
125 owner, name := parse_github_owner_repo(clone_url) or {
126 eprintln('[github-info] cannot parse github url: ${clone_url}')
127 return ''
128 }
129 url := 'https://api.github.com/repos/${owner}/${name}'
130 eprintln('[github-info] GET ${url}')
131 mut req := http.new_request(.get, url, '')
132 req.add_header(.user_agent, 'gitly')
133 req.add_header(.accept, 'application/vnd.github+json')
134 resp := req.do() or {
135 eprintln('[github-info] request failed: ${err}')
136 return ''
137 }
138 if resp.status_code != 200 {
139 eprintln('[github-info] non-200 status ${resp.status_code}: ${resp.body#[..200]}')
140 return ''
141 }
142 info := json.decode[GitHubRepoInfo](resp.body) or {
143 eprintln('[github-info] cannot decode response: ${err}')
144 return ''
145 }
146 return info.description
147}
148
149fn (mut app App) import_github_contributors(repo_id int, clone_url string) ! {
150 eprintln('[github-contrib] starting for repo_id=${repo_id} clone_url=${clone_url}')
151 owner, name := parse_github_owner_repo(clone_url) or {
152 return error('cannot parse github url: ${clone_url}')
153 }
154 mut page := 1
155 mut total := 0
156 for page <= 10 {
157 url := 'https://api.github.com/repos/${owner}/${name}/contributors?per_page=100&page=${page}'
158 eprintln('[github-contrib] GET ${url}')
159 mut req := http.new_request(.get, url, '')
160 req.add_header(.user_agent, 'gitly')
161 req.add_header(.accept, 'application/vnd.github+json')
162 resp := req.do() or { return error('github api request failed: ${err}') }
163 if resp.status_code != 200 {
164 return error('github api ${resp.status_code}: ${resp.body}')
165 }
166 contributors := json.decode[[]GitHubContributor](resp.body) or {
167 return error('cannot decode github contributors: ${err}')
168 }
169 if contributors.len == 0 {
170 break
171 }
172 for c in contributors {
173 if c.login == '' || c.type_ == 'Bot' {
174 continue
175 }
176 user_id := app.find_or_create_github_shadow_contributor(c.login, c.avatar_url) or {
177 eprintln('[github-contrib] cannot resolve @${c.login}: ${err}')
178 continue
179 }
180 app.add_contributor(user_id, repo_id) or {
181 eprintln('[github-contrib] cannot link @${c.login}: ${err}')
182 continue
183 }
184 total++
185 }
186 if contributors.len < 100 {
187 break
188 }
189 page++
190 }
191 app.update_repo_contributor_count(repo_id) or {
192 eprintln('[github-contrib] cannot update contributor count: ${err}')
193 }
194 eprintln('[github-contrib] done: imported ${total} contributors into repo ${repo_id}')
195}
196
197fn (mut app App) import_github_pull_requests(repo Repo, owner_user_id int) ! {
198 eprintln('[github-pr] starting for repo_id=${repo.id} clone_url=${repo.clone_url} owner_user_id=${owner_user_id}')
199 owner, name := parse_github_owner_repo(repo.clone_url) or {
200 return error('cannot parse github url: ${repo.clone_url}')
201 }
202 defer {
203 app.sync_repo_open_pr_count(repo.id) or {
204 eprintln('[github-pr] cannot sync open PR count: ${err}')
205 }
206 }
207 mut page := 1
208 mut imported := 0
209 mut fetched := 0
210 for page <= 100 {
211 url := 'https://api.github.com/repos/${owner}/${name}/pulls?state=open&per_page=100&page=${page}'
212 eprintln('[github-pr] GET ${url}')
213 mut req := http.new_request(.get, url, '')
214 req.add_header(.user_agent, 'gitly')
215 req.add_header(.accept, 'application/vnd.github+json')
216 resp := req.do() or {
217 eprintln('[github-pr] ERROR: request failed: ${err}')
218 return error('github api request failed: ${err}')
219 }
220 eprintln('[github-pr] page=${page} status=${resp.status_code} body_len=${resp.body.len}')
221 if resp.status_code != 200 {
222 eprintln('[github-pr] ERROR body: ${resp.body}')
223 return error('github api ${resp.status_code}: ${resp.body}')
224 }
225 prs := json.decode[[]GitHubPullRequest](resp.body) or {
226 eprintln('[github-pr] ERROR: cannot decode response: ${err}')
227 eprintln('[github-pr] response body was: ${resp.body#[..1000]}')
228 return error('cannot decode github pull requests: ${err}')
229 }
230 eprintln('[github-pr] decoded ${prs.len} pull requests on page ${page}')
231 if prs.len == 0 {
232 break
233 }
234 for gh_pr in prs {
235 if gh_pr.number <= 0 {
236 continue
237 }
238 head_branch := 'pr/${gh_pr.number}'
239 refspec := '+refs/pull/${gh_pr.number}/head:refs/heads/${head_branch}'
240 fetch_result := git.Git.fetch_ref(repo.git_dir, 'origin', refspec)
241 if fetch_result.exit_code != 0 {
242 eprintln('[github-pr] cannot fetch PR #${gh_pr.number}: ${fetch_result.output}')
243 continue
244 }
245 fetched++
246
247 base_branch := gh_pr.base.ref_name
248 if base_branch == '' || !is_safe_ref(base_branch) {
249 eprintln('[github-pr] skipping PR #${gh_pr.number}: invalid base branch "${base_branch}"')
250 continue
251 }
252 if app.pull_request_exists_for_head(repo.id, head_branch) {
253 continue
254 }
255 mut author_id := owner_user_id
256 if gh_pr.user.login != '' {
257 author_id = app.find_or_create_github_shadow_user(gh_pr.user.login) or {
258 eprintln('[github-pr] cannot resolve author @${gh_pr.user.login}: ${err}')
259 owner_user_id
260 }
261 }
262 created_at := parse_github_timestamp(gh_pr.created_at)
263 title := if gh_pr.title != '' { gh_pr.title } else { 'Pull request #${gh_pr.number}' }
264 app.add_imported_pull_request(repo.id, author_id, title, gh_pr.body, head_branch,
265 base_branch, created_at) or {
266 eprintln('[github-pr] ERROR inserting PR #${gh_pr.number}: ${err}')
267 continue
268 }
269 app.increment_repo_open_prs(repo.id) or {
270 eprintln('[github-pr] cannot bump PR count: ${err}')
271 }
272 imported++
273 }
274 if prs.len < 100 {
275 break
276 }
277 page++
278 }
279 eprintln('[github-pr] done: fetched ${fetched} PR refs, imported ${imported} pull requests into repo ${repo.id}')
280}
281
282// find_or_create_github_shadow_contributor is like find_or_create_github_shadow_user
283// but also stores the GitHub avatar URL when given.
284fn (mut app App) find_or_create_github_shadow_contributor(github_login string, avatar_url string) !int {
285 if u := app.get_user_by_username(github_login) {
286 return u.id
287 }
288 avatar := if avatar_url != '' { avatar_url } else { 'https://github.com/${github_login}.png' }
289 user := User{
290 username: github_login
291 github_username: github_login
292 is_github: true
293 is_registered: false
294 avatar: avatar
295 created_at: time.now()
296 }
297 app.add_user(user)!
298 created := app.get_user_by_username(github_login) or {
299 return error('shadow user not found after insert: ${github_login}')
300 }
301 return created.id
302}
303
304fn (mut app App) import_github_issues(repo_id int, clone_url string, owner_user_id int) ! {
305 eprintln('[github-import] starting for repo_id=${repo_id} clone_url=${clone_url} owner_user_id=${owner_user_id}')
306 owner, name := parse_github_owner_repo(clone_url) or {
307 eprintln('[github-import] ERROR: cannot parse github url: ${clone_url}')
308 return error('cannot parse github url: ${clone_url}')
309 }
310 eprintln('[github-import] parsed owner=${owner} name=${name}')
311 mut page := 1
312 mut total := 0
313 for page <= 100 {
314 url := 'https://api.github.com/repos/${owner}/${name}/issues?state=open&per_page=100&page=${page}'
315 eprintln('[github-import] GET ${url}')
316 mut req := http.new_request(.get, url, '')
317 req.add_header(.user_agent, 'gitly')
318 req.add_header(.accept, 'application/vnd.github+json')
319 resp := req.do() or {
320 eprintln('[github-import] ERROR: request failed: ${err}')
321 return error('github api request failed: ${err}')
322 }
323 eprintln('[github-import] page=${page} status=${resp.status_code} body_len=${resp.body.len}')
324 if resp.status_code != 200 {
325 eprintln('[github-import] ERROR body: ${resp.body}')
326 return error('github api ${resp.status_code}: ${resp.body}')
327 }
328 issues := json.decode[[]GitHubIssue](resp.body) or {
329 eprintln('[github-import] ERROR: cannot decode response: ${err}')
330 eprintln('[github-import] response body was: ${resp.body#[..1000]}')
331 return error('cannot decode github issues: ${err}')
332 }
333 eprintln('[github-import] decoded ${issues.len} issues on page ${page}')
334 if issues.len == 0 {
335 break
336 }
337 for gi in issues {
338 // GitHub returns PRs in the issues endpoint; skip them.
339 if gi.pull_request.url != '' {
340 eprintln('[github-import] skipping PR #${gi.number}')
341 continue
342 }
343 mut author_id := owner_user_id
344 if gi.user.login != '' {
345 author_id = app.find_or_create_github_shadow_user(gi.user.login) or {
346 eprintln('[github-import] cannot resolve author @${gi.user.login}: ${err}')
347 owner_user_id
348 }
349 }
350 created_at := parse_github_timestamp(gi.created_at)
351 issue_id := app.add_imported_issue_returning_id(repo_id, author_id, gi.title, gi.body,
352 created_at) or {
353 eprintln('[github-import] ERROR inserting issue #${gi.number}: ${err}')
354 continue
355 }
356 app.increment_repo_issues(repo_id) or {
357 eprintln('[github-import] cannot bump issue count: ${err}')
358 }
359 for gl in gi.labels {
360 if gl.name == '' {
361 continue
362 }
363 color := if gl.color == '' { 'cccccc' } else { gl.color }
364 label_id := app.find_or_create_label(repo_id, gl.name, color) or {
365 eprintln('[github-import] cannot create label ${gl.name}: ${err}')
366 continue
367 }
368 if label_id == 0 {
369 continue
370 }
371 app.add_issue_label(issue_id, label_id) or {
372 eprintln('[github-import] cannot link label ${gl.name} to issue #${gi.number}: ${err}')
373 }
374 }
375 total++
376 }
377 if issues.len < 100 {
378 break
379 }
380 page++
381 }
382 eprintln('[github-import] done: imported ${total} issues into repo ${repo_id}')
383}
384
385@['/oauth']
386pub fn (mut app App) handle_oauth() veb.Result {
387 code := ctx.query['code']
388 state := ctx.query['state']
389
390 if code == '' {
391 app.add_security_log(user_id: ctx.user.id, kind: .empty_oauth_code) or {
392 app.info(err.str())
393 }
394 app.info('Code is empty')
395
396 return ctx.redirect_to_index()
397 }
398
399 csrf := ctx.get_cookie('csrf') or { return ctx.redirect_to_index() }
400 if csrf != state || csrf == '' {
401 app.add_security_log(
402 user_id: ctx.user.id
403 kind: .wrong_oauth_state
404 arg1: 'csrf=${csrf}'
405 arg2: 'state=${state}'
406 ) or { app.info(err.str()) }
407
408 return ctx.redirect_to_index()
409 }
410
411 oauth_request := oauth.Request{
412 client_id: app.settings.oauth_client_id
413 client_secret: app.settings.oauth_client_secret
414 code: code
415 state: csrf
416 }
417
418 js := json.encode(oauth_request)
419 access_response := http.post_json('https://github.com/login/oauth/access_token', js) or {
420 app.info(err.msg())
421
422 return ctx.redirect_to_index()
423 }
424
425 mut token := access_response.body.find_between('access_token=', '&')
426 mut request := http.new_request(.get, 'https://api.github.com/user', '')
427 request.add_header(.authorization, 'token ${token}')
428
429 user_response := request.do() or {
430 app.info(err.msg())
431
432 return ctx.redirect_to_index()
433 }
434
435 if user_response.status_code != 200 {
436 app.info(user_response.status_code.str())
437 app.info(user_response.body)
438 return ctx.text('Received ${user_response.status_code} error while attempting to contact GitHub')
439 }
440
441 github_user := json.decode[GitHubUser](user_response.body) or { return ctx.redirect_to_index() }
442
443 if github_user.email.trim_space().len == 0 {
444 app.add_security_log(
445 user_id: ctx.user.id
446 kind: .empty_oauth_email
447 arg1: user_response.body
448 ) or { app.info(err.str()) }
449 app.info('Email is empty')
450 }
451
452 mut user := app.get_user_by_github_username(github_user.username) or { User{} }
453
454 if !user.is_github {
455 // Register a new user via github
456 app.add_security_log(
457 user_id: user.id
458 kind: .registered_via_github
459 arg1: user_response.body
460 ) or { app.info(err.str()) }
461
462 app.register_user(github_user.username, '', '', [github_user.email], true, false) or {
463 app.info(err.msg())
464 }
465
466 user = app.get_user_by_github_username(github_user.username) or {
467 return ctx.redirect_to_index()
468 }
469
470 app.update_user_avatar(user.id, github_user.avatar) or { app.info(err.msg()) }
471 }
472
473 app.auth_user(mut ctx, user, ctx.ip()) or { app.info(err.msg()) }
474 app.add_security_log(user_id: user.id, kind: .logged_in_via_github, arg1: user_response.body) or {
475 app.info(err.str())
476 }
477
478 return ctx.redirect_to_index()
479}
480