| 1 | // Copyright (c) 2020-2021 Alexander Medvednikov. All rights reserved. |
| 2 | // Use of this source code is governed by a GPL license that can be found in the LICENSE file. |
| 3 | module main |
| 4 | |
| 5 | import veb |
| 6 | import json |
| 7 | import net.http |
| 8 | import time |
| 9 | // import veb.auth as oauth |
| 10 | import veb.oauth |
| 11 | |
| 12 | struct GitHubUser { |
| 13 | username string @[json: 'login'] |
| 14 | name string |
| 15 | email string |
| 16 | avatar string @[json: 'avatar_url'] |
| 17 | } |
| 18 | |
| 19 | struct GitHubIssueAuthor { |
| 20 | login string |
| 21 | } |
| 22 | |
| 23 | struct GitHubPullRequestRef { |
| 24 | url string |
| 25 | } |
| 26 | |
| 27 | struct GitHubLabel { |
| 28 | name string |
| 29 | color string |
| 30 | description string |
| 31 | } |
| 32 | |
| 33 | struct GitHubRepoInfo { |
| 34 | description string |
| 35 | } |
| 36 | |
| 37 | struct GitHubContributor { |
| 38 | login string |
| 39 | avatar_url string |
| 40 | type_ string @[json: 'type'] |
| 41 | html_url string |
| 42 | id int |
| 43 | } |
| 44 | |
| 45 | struct GitHubIssue { |
| 46 | number int |
| 47 | title string |
| 48 | body string |
| 49 | state string |
| 50 | created_at string |
| 51 | user GitHubIssueAuthor |
| 52 | pull_request GitHubPullRequestRef |
| 53 | labels []GitHubLabel |
| 54 | } |
| 55 | |
| 56 | fn parse_github_timestamp(s string) int { |
| 57 | if s == '' { |
| 58 | return int(time.now().unix()) |
| 59 | } |
| 60 | t := time.parse_iso8601(s) or { return int(time.now().unix()) } |
| 61 | return int(t.unix()) |
| 62 | } |
| 63 | |
| 64 | fn parse_github_owner_repo(clone_url string) ?(string, string) { |
| 65 | mut s := clone_url.trim_space() |
| 66 | for prefix in ['https://', 'http://', 'git@'] { |
| 67 | if s.starts_with(prefix) { |
| 68 | s = s[prefix.len..] |
| 69 | break |
| 70 | } |
| 71 | } |
| 72 | s = s.trim_string_left('github.com') |
| 73 | s = s.trim_left(':/') |
| 74 | s = s.trim_string_right('.git') |
| 75 | s = s.trim('/') |
| 76 | parts := s.split('/') |
| 77 | if parts.len < 2 || parts[0] == '' || parts[1] == '' { |
| 78 | return none |
| 79 | } |
| 80 | return parts[0], parts[1] |
| 81 | } |
| 82 | |
| 83 | // Returns the local user id for a GitHub login, creating an unregistered |
| 84 | // "shadow" user (no password, no email, just the username and GitHub avatar) |
| 85 | // when one does not yet exist. |
| 86 | fn (mut app App) find_or_create_github_shadow_user(github_login string) !int { |
| 87 | if u := app.get_user_by_username(github_login) { |
| 88 | return u.id |
| 89 | } |
| 90 | user := User{ |
| 91 | username: github_login |
| 92 | github_username: github_login |
| 93 | is_github: true |
| 94 | is_registered: false |
| 95 | avatar: 'https://github.com/${github_login}.png' |
| 96 | created_at: time.now() |
| 97 | } |
| 98 | app.add_user(user)! |
| 99 | created := app.get_user_by_username(github_login) or { |
| 100 | return error('shadow user not found after insert: ${github_login}') |
| 101 | } |
| 102 | return created.id |
| 103 | } |
| 104 | |
| 105 | // fetch_github_repo_description returns the GitHub description for a repo, or |
| 106 | // an empty string if it cannot be retrieved. |
| 107 | fn fetch_github_repo_description(clone_url string) string { |
| 108 | owner, name := parse_github_owner_repo(clone_url) or { |
| 109 | eprintln('[github-info] cannot parse github url: ${clone_url}') |
| 110 | return '' |
| 111 | } |
| 112 | url := 'https://api.github.com/repos/${owner}/${name}' |
| 113 | eprintln('[github-info] GET ${url}') |
| 114 | mut req := http.new_request(.get, url, '') |
| 115 | req.add_header(.user_agent, 'gitly') |
| 116 | req.add_header(.accept, 'application/vnd.github+json') |
| 117 | resp := req.do() or { |
| 118 | eprintln('[github-info] request failed: ${err}') |
| 119 | return '' |
| 120 | } |
| 121 | if resp.status_code != 200 { |
| 122 | eprintln('[github-info] non-200 status ${resp.status_code}: ${resp.body#[..200]}') |
| 123 | return '' |
| 124 | } |
| 125 | info := json.decode(GitHubRepoInfo, resp.body) or { |
| 126 | eprintln('[github-info] cannot decode response: ${err}') |
| 127 | return '' |
| 128 | } |
| 129 | return info.description |
| 130 | } |
| 131 | |
| 132 | fn (mut app App) import_github_contributors(repo_id int, clone_url string) ! { |
| 133 | eprintln('[github-contrib] starting for repo_id=${repo_id} clone_url=${clone_url}') |
| 134 | owner, name := parse_github_owner_repo(clone_url) or { |
| 135 | return error('cannot parse github url: ${clone_url}') |
| 136 | } |
| 137 | mut page := 1 |
| 138 | mut total := 0 |
| 139 | for page <= 10 { |
| 140 | url := 'https://api.github.com/repos/${owner}/${name}/contributors?per_page=100&page=${page}' |
| 141 | eprintln('[github-contrib] GET ${url}') |
| 142 | mut req := http.new_request(.get, url, '') |
| 143 | req.add_header(.user_agent, 'gitly') |
| 144 | req.add_header(.accept, 'application/vnd.github+json') |
| 145 | resp := req.do() or { return error('github api request failed: ${err}') } |
| 146 | if resp.status_code != 200 { |
| 147 | return error('github api ${resp.status_code}: ${resp.body}') |
| 148 | } |
| 149 | contributors := json.decode([]GitHubContributor, resp.body) or { |
| 150 | return error('cannot decode github contributors: ${err}') |
| 151 | } |
| 152 | if contributors.len == 0 { |
| 153 | break |
| 154 | } |
| 155 | for c in contributors { |
| 156 | if c.login == '' || c.type_ == 'Bot' { |
| 157 | continue |
| 158 | } |
| 159 | user_id := app.find_or_create_github_shadow_contributor(c.login, c.avatar_url) or { |
| 160 | eprintln('[github-contrib] cannot resolve @${c.login}: ${err}') |
| 161 | continue |
| 162 | } |
| 163 | app.add_contributor(user_id, repo_id) or { |
| 164 | eprintln('[github-contrib] cannot link @${c.login}: ${err}') |
| 165 | continue |
| 166 | } |
| 167 | total++ |
| 168 | } |
| 169 | if contributors.len < 100 { |
| 170 | break |
| 171 | } |
| 172 | page++ |
| 173 | } |
| 174 | app.update_repo_contributor_count(repo_id) or { |
| 175 | eprintln('[github-contrib] cannot update contributor count: ${err}') |
| 176 | } |
| 177 | eprintln('[github-contrib] done: imported ${total} contributors into repo ${repo_id}') |
| 178 | } |
| 179 | |
| 180 | // find_or_create_github_shadow_contributor is like find_or_create_github_shadow_user |
| 181 | // but also stores the GitHub avatar URL when given. |
| 182 | fn (mut app App) find_or_create_github_shadow_contributor(github_login string, avatar_url string) !int { |
| 183 | if u := app.get_user_by_username(github_login) { |
| 184 | return u.id |
| 185 | } |
| 186 | avatar := if avatar_url != '' { avatar_url } else { 'https://github.com/${github_login}.png' } |
| 187 | user := User{ |
| 188 | username: github_login |
| 189 | github_username: github_login |
| 190 | is_github: true |
| 191 | is_registered: false |
| 192 | avatar: avatar |
| 193 | created_at: time.now() |
| 194 | } |
| 195 | app.add_user(user)! |
| 196 | created := app.get_user_by_username(github_login) or { |
| 197 | return error('shadow user not found after insert: ${github_login}') |
| 198 | } |
| 199 | return created.id |
| 200 | } |
| 201 | |
| 202 | fn (mut app App) import_github_issues(repo_id int, clone_url string, owner_user_id int) ! { |
| 203 | eprintln('[github-import] starting for repo_id=${repo_id} clone_url=${clone_url} owner_user_id=${owner_user_id}') |
| 204 | owner, name := parse_github_owner_repo(clone_url) or { |
| 205 | eprintln('[github-import] ERROR: cannot parse github url: ${clone_url}') |
| 206 | return error('cannot parse github url: ${clone_url}') |
| 207 | } |
| 208 | eprintln('[github-import] parsed owner=${owner} name=${name}') |
| 209 | mut page := 1 |
| 210 | mut total := 0 |
| 211 | for page <= 100 { |
| 212 | url := 'https://api.github.com/repos/${owner}/${name}/issues?state=open&per_page=100&page=${page}' |
| 213 | eprintln('[github-import] GET ${url}') |
| 214 | mut req := http.new_request(.get, url, '') |
| 215 | req.add_header(.user_agent, 'gitly') |
| 216 | req.add_header(.accept, 'application/vnd.github+json') |
| 217 | resp := req.do() or { |
| 218 | eprintln('[github-import] ERROR: request failed: ${err}') |
| 219 | return error('github api request failed: ${err}') |
| 220 | } |
| 221 | eprintln('[github-import] page=${page} status=${resp.status_code} body_len=${resp.body.len}') |
| 222 | if resp.status_code != 200 { |
| 223 | eprintln('[github-import] ERROR body: ${resp.body}') |
| 224 | return error('github api ${resp.status_code}: ${resp.body}') |
| 225 | } |
| 226 | issues := json.decode([]GitHubIssue, resp.body) or { |
| 227 | eprintln('[github-import] ERROR: cannot decode response: ${err}') |
| 228 | eprintln('[github-import] response body was: ${resp.body#[..1000]}') |
| 229 | return error('cannot decode github issues: ${err}') |
| 230 | } |
| 231 | eprintln('[github-import] decoded ${issues.len} issues on page ${page}') |
| 232 | if issues.len == 0 { |
| 233 | break |
| 234 | } |
| 235 | for gi in issues { |
| 236 | // GitHub returns PRs in the issues endpoint; skip them. |
| 237 | if gi.pull_request.url != '' { |
| 238 | eprintln('[github-import] skipping PR #${gi.number}') |
| 239 | continue |
| 240 | } |
| 241 | mut author_id := owner_user_id |
| 242 | if gi.user.login != '' { |
| 243 | author_id = app.find_or_create_github_shadow_user(gi.user.login) or { |
| 244 | eprintln('[github-import] cannot resolve author @${gi.user.login}: ${err}') |
| 245 | owner_user_id |
| 246 | } |
| 247 | } |
| 248 | created_at := parse_github_timestamp(gi.created_at) |
| 249 | issue_id := app.add_imported_issue_returning_id(repo_id, author_id, gi.title, gi.body, |
| 250 | created_at) or { |
| 251 | eprintln('[github-import] ERROR inserting issue #${gi.number}: ${err}') |
| 252 | continue |
| 253 | } |
| 254 | app.increment_repo_issues(repo_id) or { |
| 255 | eprintln('[github-import] cannot bump issue count: ${err}') |
| 256 | } |
| 257 | for gl in gi.labels { |
| 258 | if gl.name == '' { |
| 259 | continue |
| 260 | } |
| 261 | color := if gl.color == '' { 'cccccc' } else { gl.color } |
| 262 | label_id := app.find_or_create_label(repo_id, gl.name, color) or { |
| 263 | eprintln('[github-import] cannot create label ${gl.name}: ${err}') |
| 264 | continue |
| 265 | } |
| 266 | if label_id == 0 { |
| 267 | continue |
| 268 | } |
| 269 | app.add_issue_label(issue_id, label_id) or { |
| 270 | eprintln('[github-import] cannot link label ${gl.name} to issue #${gi.number}: ${err}') |
| 271 | } |
| 272 | } |
| 273 | total++ |
| 274 | } |
| 275 | if issues.len < 100 { |
| 276 | break |
| 277 | } |
| 278 | page++ |
| 279 | } |
| 280 | eprintln('[github-import] done: imported ${total} issues into repo ${repo_id}') |
| 281 | } |
| 282 | |
| 283 | @['/oauth'] |
| 284 | pub fn (mut app App) handle_oauth() veb.Result { |
| 285 | code := ctx.query['code'] |
| 286 | state := ctx.query['state'] |
| 287 | |
| 288 | if code == '' { |
| 289 | app.add_security_log(user_id: ctx.user.id, kind: .empty_oauth_code) or { |
| 290 | app.info(err.str()) |
| 291 | } |
| 292 | app.info('Code is empty') |
| 293 | |
| 294 | return ctx.redirect_to_index() |
| 295 | } |
| 296 | |
| 297 | csrf := ctx.get_cookie('csrf') or { return ctx.redirect_to_index() } |
| 298 | if csrf != state || csrf == '' { |
| 299 | app.add_security_log( |
| 300 | user_id: ctx.user.id |
| 301 | kind: .wrong_oauth_state |
| 302 | arg1: 'csrf=${csrf}' |
| 303 | arg2: 'state=${state}' |
| 304 | ) or { app.info(err.str()) } |
| 305 | |
| 306 | return ctx.redirect_to_index() |
| 307 | } |
| 308 | |
| 309 | oauth_request := oauth.Request{ |
| 310 | client_id: app.settings.oauth_client_id |
| 311 | client_secret: app.settings.oauth_client_secret |
| 312 | code: code |
| 313 | state: csrf |
| 314 | } |
| 315 | |
| 316 | js := json.encode(oauth_request) |
| 317 | access_response := http.post_json('https://github.com/login/oauth/access_token', js) or { |
| 318 | app.info(err.msg()) |
| 319 | |
| 320 | return ctx.redirect_to_index() |
| 321 | } |
| 322 | |
| 323 | mut token := access_response.body.find_between('access_token=', '&') |
| 324 | mut request := http.new_request(.get, 'https://api.github.com/user', '') |
| 325 | request.add_header(.authorization, 'token ${token}') |
| 326 | |
| 327 | user_response := request.do() or { |
| 328 | app.info(err.msg()) |
| 329 | |
| 330 | return ctx.redirect_to_index() |
| 331 | } |
| 332 | |
| 333 | if user_response.status_code != 200 { |
| 334 | app.info(user_response.status_code.str()) |
| 335 | app.info(user_response.body) |
| 336 | return ctx.text('Received ${user_response.status_code} error while attempting to contact GitHub') |
| 337 | } |
| 338 | |
| 339 | github_user := json.decode(GitHubUser, user_response.body) or { return ctx.redirect_to_index() } |
| 340 | |
| 341 | if github_user.email.trim_space().len == 0 { |
| 342 | app.add_security_log( |
| 343 | user_id: ctx.user.id |
| 344 | kind: .empty_oauth_email |
| 345 | arg1: user_response.body |
| 346 | ) or { app.info(err.str()) } |
| 347 | app.info('Email is empty') |
| 348 | } |
| 349 | |
| 350 | mut user := app.get_user_by_github_username(github_user.username) or { User{} } |
| 351 | |
| 352 | if !user.is_github { |
| 353 | // Register a new user via github |
| 354 | app.add_security_log( |
| 355 | user_id: user.id |
| 356 | kind: .registered_via_github |
| 357 | arg1: user_response.body |
| 358 | ) or { app.info(err.str()) } |
| 359 | |
| 360 | app.register_user(github_user.username, '', '', [github_user.email], true, false) or { |
| 361 | app.info(err.msg()) |
| 362 | } |
| 363 | |
| 364 | user = app.get_user_by_github_username(github_user.username) or { |
| 365 | return ctx.redirect_to_index() |
| 366 | } |
| 367 | |
| 368 | app.update_user_avatar(user.id, github_user.avatar) or { app.info(err.msg()) } |
| 369 | } |
| 370 | |
| 371 | app.auth_user(mut ctx, user, ctx.ip()) or { app.info(err.msg()) } |
| 372 | app.add_security_log(user_id: user.id, kind: .logged_in_via_github, arg1: user_response.body) or { |
| 373 | app.info(err.str()) |
| 374 | } |
| 375 | |
| 376 | return ctx.redirect_to_index() |
| 377 | } |
| 378 | |