diff --git a/codespeed/admin.py b/codespeed/admin.py index 678cf3fd..702f909d 100644 --- a/codespeed/admin.py +++ b/codespeed/admin.py @@ -1,8 +1,9 @@ from django import forms -from django.contrib import admin +from django.contrib import admin, messages from codespeed.models import (Project, Revision, Executable, Benchmark, Branch, - Result, Environment, Report) + Result, Environment, Report, GitHubAccount, + PRRun, PRRunner) class ProjectForm(forms.ModelForm): @@ -109,3 +110,70 @@ class ReportAdmin(admin.ModelAdmin): list_filter = ('environment', 'executable') ordering = ['-revision'] actions = [recalculate_report] + + +@admin.register(GitHubAccount) +class GitHubAccountAdmin(admin.ModelAdmin): + """Read-only: these rows mirror GitHub and are rewritten at each sign-in.""" + list_display = ('login', 'name', 'github_id', 'user', 'verified_at') + search_fields = ('login', 'name') + readonly_fields = ('user', 'github_id', 'login', 'name', 'avatar_url', + 'verified_at') + + def has_add_permission(self, request): + return False + + +@admin.register(PRRunner) +class PRRunnerAdmin(admin.ModelAdmin): + list_display = ('environment', 'enabled', 'backend', 'has_token', + 'last_seen') + list_filter = ('enabled', 'backend') + filter_horizontal = ('executables',) + readonly_fields = ('last_seen',) + actions = ['issue_token'] + fieldsets = ( + (None, {'fields': ('environment', 'executables', 'enabled', 'notes', + 'last_seen')}), + ('How the machine is told about work', { + 'fields': ('backend', 'workflow_repo', 'workflow_file', + 'workflow_ref'), + }), + ) + + @admin.display(boolean=True, description='Token issued') + def has_token(self, obj): + return bool(obj.token_hash) + + def save_model(self, request, obj, form, change): + token = None if obj.token_hash else obj.new_token() + super().save_model(request, obj, form, change) + if token: + self._show_token(request, obj, token) + + @admin.action(description='Issue a new token (the old one stops working)') + def issue_token(self, request, queryset): + for runner in queryset: + token = runner.new_token() + runner.save(update_fields=['token_hash']) + self._show_token(request, runner, token) + + def _show_token(self, request, runner, token): + # Only the hash is kept, so this message is the one chance to copy it + messages.warning( + request, 'Token for %s: %s (copy it now; it is not stored and ' + 'cannot be shown again)' % (runner, token)) + + +@admin.register(PRRun) +class PRRunAdmin(admin.ModelAdmin): + """An audit trail; runs are made from the PR Runner tab, not here.""" + list_display = ('id', 'pr_number', 'head_sha', 'runner', 'executable', + 'status', 'requested_by_login', 'created_at') + list_filter = ('status', 'runner', 'project') + search_fields = ('pr_number', 'head_sha', 'requested_by_login') + raw_id_fields = ('requested_by',) + date_hierarchy = 'created_at' + + def has_add_permission(self, request): + return False diff --git a/codespeed/auth.py b/codespeed/auth.py index 487a0098..0efbb794 100644 --- a/codespeed/auth.py +++ b/codespeed/auth.py @@ -24,6 +24,20 @@ def is_authenticated(request): return False +def may_post_results(user): + """Whether a signed-in user may post results. + + Any local account may. People who signed in with GitHub get an + account for the PR runner. + """ + from .models import GitHubAccount + + pk = getattr(user, 'pk', None) + if isinstance(pk, int) and GitHubAccount.objects.filter(user_id=pk).exists(): + return user.has_perm('codespeed.add_result') + return True + + def basic_auth_required(realm='default'): def _helper(func): @wraps(func) @@ -33,7 +47,9 @@ def _decorator(request, *args, **kwargs): if settings.ALLOW_ANONYMOUS_POST: logger.debug('allowing anonymous post') allowed = True - elif hasattr(request, 'user') and is_authenticated(request=request): + elif (hasattr(request, 'user') and + is_authenticated(request=request) and + may_post_results(request.user)): allowed = True elif 'HTTP_AUTHORIZATION' in request.META: logger.debug('checking for http authorization header') @@ -44,7 +60,8 @@ def _decorator(request, *args, **kwargs): if authmeth.lower() == 'basic': username, password = decode_basic_auth(auth) user = authenticate(username=username, password=password) - if user is not None and user.is_active: + if (user is not None and user.is_active and + may_post_results(user)): logger.info( 'Authentication succeeded for {}'.format(username)) login(request, user) diff --git a/codespeed/commits/github.py b/codespeed/commits/github.py index 4d9cc42a..1403ead2 100644 --- a/codespeed/commits/github.py +++ b/codespeed/commits/github.py @@ -48,9 +48,11 @@ def fetch_json(url): try: json_obj = json.load(urlopen(request)) except IOError as e: - logger.exception("Unable to load %s: %s", - url, e, exc_info=True) - raise e + # Rate limits, and such things. CommitLogError lets a + # result upload go through without the commit's metadata rather + # than fail with a 500. + logger.warning("Unable to load %s: %s", url, e) + raise CommitLogError("Unable to load %s: %s" % (url, e)) from e if "message" in json_obj and \ json_obj["message"] in ("Not Found", "Server Error",): diff --git a/codespeed/github_api.py b/codespeed/github_api.py new file mode 100644 index 00000000..a7d44977 --- /dev/null +++ b/codespeed/github_api.py @@ -0,0 +1,81 @@ +"""A small GitHub REST client for sign-in and the PR runner. + +It is kept apart from ``codespeed.commits.github``, which caches every +response for a month. That suits immutable commits but is wrong for team +memberships and pull requests, which may change. +""" +import json +import logging +import re +from urllib.error import HTTPError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +logger = logging.getLogger(__name__) + +API_ROOT = 'https://api.github.com' +API_VERSION = '2022-11-28' +USER_AGENT = 'codespeed' +TIMEOUT = 10 + +_REPO_RE = re.compile( + r'^https?://github\.com/(?P[\w.-]+)/(?P[\w.-]+?)(\.git)?/?$' +) + + +class GitHubError(Exception): + """A GitHub call failed. status is None when GitHub was unreachable.""" + + def __init__(self, status, message): + super().__init__('GitHub API error (%s): %s' % (status, message)) + self.status = status + self.message = message + + +def send(request): + """Send a prepared request and return the decoded JSON body, or None.""" + try: + with urlopen(request, timeout=TIMEOUT) as response: + body = response.read() + except HTTPError as e: + try: + message = json.loads(e.read()).get('message', e.reason) + except (ValueError, AttributeError): + message = e.reason + raise GitHubError(e.code, message) from None + except OSError as e: # URLError, timeouts, connection resets + raise GitHubError(None, str(e)) from None + if not body: + return None + try: + return json.loads(body) + except ValueError: + raise GitHubError(None, 'response was not JSON') from None + + +def api(method, path, token=None, data=None, params=None): + """Call https://api.github.com and return the decoded JSON.""" + if not path.startswith('/'): + raise ValueError('path must start with "/": %r' % path) + url = API_ROOT + path + if params: + url += '?' + urlencode(params) + headers = { + 'Accept': 'application/vnd.github+json', + 'X-GitHub-Api-Version': API_VERSION, + 'User-Agent': USER_AGENT, + } + if token: + headers['Authorization'] = 'Bearer %s' % token + body = None + if data is not None: + body = json.dumps(data).encode() + headers['Content-Type'] = 'application/json' + return send(Request(url, data=body, headers=headers, method=method)) + + +def repo_slug(repo_url): + m = _REPO_RE.match(repo_url or '') + if not m: + return None + return m.group('owner'), m.group('name') diff --git a/codespeed/github_auth.py b/codespeed/github_auth.py new file mode 100644 index 00000000..af5088d7 --- /dev/null +++ b/codespeed/github_auth.py @@ -0,0 +1,208 @@ +"""Sign in with GitHub. + +The OAuth web flow is small enough to do by hand, and doing so keeps the +footprint to one model and three views. Only people who pass the membership +check get an account; everyone else is turned away before a user row exists. + +The access token is used for that one check and then dropped. It is never +stored, so there is nothing to leak or to revoke. +""" +import logging +import secrets +from datetime import timedelta +from urllib.parse import quote, urlencode +from urllib.request import Request + +from django.conf import settings +from django.contrib.auth import get_user_model, login, logout +from django.http import Http404 +from django.shortcuts import redirect, render +from django.urls import reverse +from django.utils import timezone +from django.utils.crypto import constant_time_compare +from django.utils.http import url_has_allowed_host_and_scheme +from django.views.decorators.http import require_GET, require_POST + +from . import github_api +from .github_api import GitHubError +from .models import GitHubAccount + +logger = logging.getLogger(__name__) + +AUTHORIZE_URL = 'https://github.com/login/oauth/authorize' +TOKEN_URL = 'https://github.com/login/oauth/access_token' +SESSION_KEY = 'github_oauth' +# read:org lets us see the user's membership of teams in a private org +# listing. It grants no write access to anything. +SCOPE = 'read:org' +USERNAME_PREFIX = 'github-' +AUTH_BACKEND = 'django.contrib.auth.backends.ModelBackend' + + +def login_enabled(): + return bool(settings.GITHUB_LOGIN_CLIENT_ID and + settings.GITHUB_LOGIN_CLIENT_SECRET) + + +def is_fresh(account): + """Whether the account's membership check is recent enough to act on.""" + max_age = timedelta(seconds=settings.GITHUB_LOGIN_RECHECK_SECONDS) + return account.verified_at >= timezone.now() - max_age + + +def _safe_next(request, url): + if url and url_has_allowed_host_and_scheme( + url, allowed_hosts={request.get_host()}, + require_https=request.is_secure()): + return url + return reverse('home') + + +def _callback_url(request): + return request.build_absolute_uri(reverse('github-callback')) + + +def _denied(request, message, status): + return render(request, 'codespeed/signin_denied.html', + {'message': message}, status=status) + + +def exchange_code(code, redirect_uri): + """Trade the one-time code from GitHub for a user access token.""" + body = urlencode({ + 'client_id': settings.GITHUB_LOGIN_CLIENT_ID, + 'client_secret': settings.GITHUB_LOGIN_CLIENT_SECRET, + 'code': code, + 'redirect_uri': redirect_uri, + }).encode() + data = github_api.send(Request(TOKEN_URL, data=body, method='POST', headers={ + 'Accept': 'application/json', + 'User-Agent': github_api.USER_AGENT, + })) + token = (data or {}).get('access_token') + if not token: + # GitHub answers 200 with an "error" key for a bad or reused code + raise GitHubError(400, (data or {}).get('error_description', + 'no access token returned')) + return token + + +def is_allowed(github_login, token): + """Whether this GitHub user may sign in. + + Fails closed: with no teams and no users configured, nobody gets in. + """ + allowed_users = {u.lower() for u in settings.GITHUB_LOGIN_ALLOWED_USERS} + if github_login.lower() in allowed_users: + return True + for team in settings.GITHUB_LOGIN_ALLOWED_TEAMS: + org, _, slug = team.partition('/') + path = '/orgs/%s/teams/%s/memberships/%s' % ( + quote(org, safe=''), quote(slug, safe=''), + quote(github_login, safe='')) + try: + membership = github_api.api('GET', path, token=token) + except GitHubError as e: + if e.status == 404: # not a member, or the team is not visible + continue + raise + if membership and membership.get('state') == 'active': + return True + return False + + +def _sign_in_account(profile): + """Return the Django user for a verified GitHub profile, or None. + + Accounts are keyed on GitHub's numeric id, never on the login name: + logins can be renamed and reused, and a GitHub user who happens to be + called "admin" must not land in a local account of the same name. + """ + github_id = int(profile['id']) + account = GitHubAccount.objects.select_related('user').filter( + github_id=github_id).first() + if account is None: + User = get_user_model() + username = '%s%d' % (USERNAME_PREFIX, github_id) + if User.objects.filter(username=username).exists(): + logger.error('Cannot create %s: a local user has that name', + username) + return None + user = User(username=username) + user.set_unusable_password() + user.save() + account = GitHubAccount(user=user, github_id=github_id) + elif not account.user.is_active: + return None + + avatar = profile.get('avatar_url') or '' + account.login = profile['login'] + account.name = (profile.get('name') or '')[:255] + account.avatar_url = avatar if avatar.startswith('https://') else '' + account.verified_at = timezone.now() + account.save() + return account.user + + +@require_GET +def github_login(request): + if not login_enabled(): + raise Http404 + state = secrets.token_urlsafe(32) + request.session[SESSION_KEY] = { + 'state': state, + 'next': _safe_next(request, request.GET.get('next')), + } + return redirect(AUTHORIZE_URL + '?' + urlencode({ + 'client_id': settings.GITHUB_LOGIN_CLIENT_ID, + 'redirect_uri': _callback_url(request), + 'scope': SCOPE, + 'state': state, + 'allow_signup': 'false', + })) + + +@require_GET +def github_callback(request): + if not login_enabled(): + raise Http404 + # The state ties this callback to the browser that started the flow. + # Popping it makes each one usable once. + saved = request.session.pop(SESSION_KEY, None) + state = request.GET.get('state', '') + if not saved or not constant_time_compare(saved['state'], state): + return _denied( + request, 'That sign-in attempt expired. Please try again.', 400) + code = request.GET.get('code') + if request.GET.get('error') or not code: + return _denied(request, 'Sign-in was cancelled on GitHub.', 400) + + try: + token = exchange_code(code, _callback_url(request)) + profile = github_api.api('GET', '/user', token=token) + allowed = is_allowed(profile['login'], token) + except (GitHubError, KeyError, TypeError) as e: + logger.warning('GitHub sign-in failed: %s', e) + return _denied( + request, 'GitHub could not confirm your account just now. ' + 'Please try again in a moment.', 502) + + if not allowed: + logger.info('Refused sign-in for GitHub user %s', profile['login']) + return _denied( + request, 'Signed in to GitHub as %s, but that account is not in ' + 'a team allowed to use this site.' % profile['login'], + 403) + + user = _sign_in_account(profile) + if user is None: + return _denied(request, 'This account has been disabled here.', 403) + login(request, user, backend=AUTH_BACKEND) + logger.info('GitHub user %s signed in', profile['login']) + return redirect(saved['next']) + + +@require_POST +def sign_out(request): + logout(request) + return redirect(_safe_next(request, request.POST.get('next'))) diff --git a/codespeed/migrations/0007_github_sign_in_and_pr_runner.py b/codespeed/migrations/0007_github_sign_in_and_pr_runner.py new file mode 100644 index 00000000..96d5c397 --- /dev/null +++ b/codespeed/migrations/0007_github_sign_in_and_pr_runner.py @@ -0,0 +1,76 @@ +# Generated by Django 6.1.2 on 2026-11-09 20:04 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('codespeed', '0006_alter_benchmark_name_alter_benchmark_unique_together'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='GitHubAccount', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('github_id', models.BigIntegerField(unique=True)), + ('login', models.CharField(max_length=39)), + ('name', models.CharField(blank=True, max_length=255)), + ('avatar_url', models.URLField(blank=True)), + ('verified_at', models.DateTimeField()), + ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='github_account', to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='PRRunner', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('enabled', models.BooleanField(default=True, help_text='Untick to stop offering this machine')), + ('notes', models.CharField(blank=True, help_text='Shown next to the environment when starting a run', max_length=200)), + ('token_hash', models.CharField(blank=True, editable=False, max_length=64, null=True, unique=True)), + ('last_seen', models.DateTimeField(blank=True, editable=False, null=True)), + ('backend', models.CharField(choices=[('agent', 'Agent polls for work'), ('gha', 'Woken through a GitHub Actions workflow')], default='agent', help_text='How the machine learns that a run is waiting', max_length=10)), + ('workflow_repo', models.CharField(blank=True, help_text="GitHub Actions only: 'owner/name' of the repository holding the workflow", max_length=100)), + ('workflow_file', models.CharField(blank=True, default='benchmark-pr.yml', max_length=100)), + ('workflow_ref', models.CharField(blank=True, default='main', max_length=100)), + ('environment', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='pr_runner', to='codespeed.environment')), + ('executables', models.ManyToManyField(help_text='Builds this machine knows how to produce', related_name='pr_runners', to='codespeed.executable')), + ], + options={ + 'verbose_name': 'PR runner', + }, + ), + migrations.CreateModel( + name='PRRun', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('pr_number', models.PositiveIntegerField()), + ('head_sha', models.CharField(max_length=40)), + ('base_sha', models.CharField(blank=True, max_length=40)), + ('base_branch', models.CharField(max_length=32)), + ('benchmarks', models.TextField(blank=True, help_text='Comma-separated names; empty runs the suite')), + ('reuse_base', models.BooleanField(default=True)), + ('requested_by_login', models.CharField(max_length=150)), + ('status', models.CharField(choices=[('queued', 'Queued'), ('running', 'Running'), ('completed', 'Completed'), ('failed', 'Failed'), ('cancelled', 'Cancelled')], db_index=True, default='queued', max_length=10)), + ('message', models.CharField(blank=True, max_length=500)), + ('log_url', models.URLField(blank=True)), + ('comment_url', models.URLField(blank=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('started_at', models.DateTimeField(blank=True, null=True)), + ('heartbeat_at', models.DateTimeField(blank=True, null=True)), + ('finished_at', models.DateTimeField(blank=True, null=True)), + ('executable', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pr_runs', to='codespeed.executable')), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pr_runs', to='codespeed.project')), + ('requested_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='pr_runs', to=settings.AUTH_USER_MODEL)), + ('runner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='runs', to='codespeed.prrunner')), + ], + options={ + 'verbose_name': 'PR run', + 'ordering': ['-id'], + }, + ), + ] diff --git a/codespeed/models.py b/codespeed/models.py index 12a6f4aa..12a35125 100644 --- a/codespeed/models.py +++ b/codespeed/models.py @@ -1,6 +1,8 @@ +import hashlib import logging import os import json +import secrets from django.core.exceptions import ValidationError from django.urls import reverse @@ -593,3 +595,168 @@ def _get_tablecache(self): def _tablecache_is_stale(cached): return any('bench_source' not in row for units in cached for row in units.get('rows', [])) + + +class GitHubAccount(models.Model): + """Links a Django user to the GitHub account they signed in with. + + Keyed on GitHub's numeric id, which survives renames. ``verified_at`` is + the last time the allowed-team check passed; privileged actions require + it to be recent (see ``GITHUB_LOGIN_RECHECK_SECONDS``). + """ + user = models.OneToOneField( + settings.AUTH_USER_MODEL, on_delete=models.CASCADE, + related_name='github_account') + github_id = models.BigIntegerField(unique=True) + login = models.CharField(max_length=39) + name = models.CharField(max_length=255, blank=True) + avatar_url = models.URLField(blank=True) + verified_at = models.DateTimeField() + + def __str__(self): + return self.login + + +class PRRunner(models.Model): + """A benchmark machine that accepts on-demand runs for pull requests. + + One per Environment, because in Codespeed the environment *is* the + machine. The machine authenticates with a token of its own, so a single + runner can be cut off without touching the others. Only the hash of the + token is stored. + """ + AGENT = 'agent' + GITHUB_ACTIONS = 'gha' + BACKENDS = ( + (AGENT, 'Agent polls for work'), + (GITHUB_ACTIONS, 'Woken through a GitHub Actions workflow'), + ) + + environment = models.OneToOneField( + Environment, on_delete=models.CASCADE, related_name='pr_runner') + executables = models.ManyToManyField( + Executable, related_name='pr_runners', + help_text="Builds this machine knows how to produce") + enabled = models.BooleanField( + default=True, help_text="Untick to stop offering this machine") + notes = models.CharField( + max_length=200, blank=True, + help_text="Shown next to the environment when starting a run") + token_hash = models.CharField( + max_length=64, unique=True, null=True, blank=True, editable=False) + last_seen = models.DateTimeField(null=True, blank=True, editable=False) + + backend = models.CharField( + max_length=10, choices=BACKENDS, default=AGENT, + help_text="How the machine learns that a run is waiting") + workflow_repo = models.CharField( + max_length=100, blank=True, + help_text="GitHub Actions only: 'owner/name' of the repository " + "holding the workflow") + workflow_file = models.CharField( + max_length=100, blank=True, default='benchmark-pr.yml') + workflow_ref = models.CharField(max_length=100, blank=True, default='main') + + class Meta: + verbose_name = "PR runner" + + def __str__(self): + return self.environment.name + + @staticmethod + def hash_token(token): + return hashlib.sha256(token.encode()).hexdigest() + + def new_token(self): + """Replace the token and return the new one; this is the only time + it exists in the clear.""" + token = secrets.token_urlsafe(32) + self.token_hash = self.hash_token(token) + return token + + @classmethod + def for_token(cls, token): + if not token: + return None + return cls.objects.filter( + token_hash=cls.hash_token(token), enabled=True + ).select_related('environment').first() + + +class PRRun(models.Model): + """One request to benchmark a pull request on one runner. + + ``head_sha`` is pinned when the request is made, so a later push to the + pull request does not change what runs. ``base_sha``, the merge base + with the target branch, is worked out by the runner from its clone and + reported back before it builds anything. + """ + QUEUED = 'queued' + RUNNING = 'running' + COMPLETED = 'completed' + FAILED = 'failed' + CANCELLED = 'cancelled' + STATUSES = ( + (QUEUED, 'Queued'), + (RUNNING, 'Running'), + (COMPLETED, 'Completed'), + (FAILED, 'Failed'), + (CANCELLED, 'Cancelled'), + ) + ACTIVE = (QUEUED, RUNNING) + + runner = models.ForeignKey( + PRRunner, on_delete=models.CASCADE, related_name='runs') + project = models.ForeignKey( + Project, on_delete=models.CASCADE, related_name='pr_runs') + executable = models.ForeignKey( + Executable, on_delete=models.CASCADE, related_name='pr_runs') + + pr_number = models.PositiveIntegerField() + head_sha = models.CharField(max_length=40) + base_sha = models.CharField(max_length=40, blank=True) + base_branch = models.CharField(max_length=32) + benchmarks = models.TextField( + blank=True, help_text="Comma-separated names; empty runs the suite") + reuse_base = models.BooleanField(default=True) + + requested_by = models.ForeignKey( + settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True, + blank=True, related_name='pr_runs') + # Kept as text too, so the audit trail outlives the user row + requested_by_login = models.CharField(max_length=150) + + status = models.CharField( + max_length=10, choices=STATUSES, default=QUEUED, db_index=True) + message = models.CharField(max_length=500, blank=True) + log_url = models.URLField(blank=True) + comment_url = models.URLField(blank=True) + + created_at = models.DateTimeField(auto_now_add=True) + started_at = models.DateTimeField(null=True, blank=True) + heartbeat_at = models.DateTimeField(null=True, blank=True) + finished_at = models.DateTimeField(null=True, blank=True) + + class Meta: + ordering = ['-id'] + verbose_name = "PR run" + + def __str__(self): + return "#%d %s on %s" % ( + self.pr_number, self.head_sha[:7], self.runner) + + def get_absolute_url(self): + return reverse('pr-run', args=[self.pk]) + + @property + def head_branch(self): + """The Codespeed branch the pull request's results are filed under.""" + return 'pr-%d' % self.pr_number + + @property + def is_active(self): + return self.status in self.ACTIVE + + @property + def benchmark_list(self): + return [b for b in self.benchmarks.split(',') if b] diff --git a/codespeed/pr_runner/__init__.py b/codespeed/pr_runner/__init__.py new file mode 100644 index 00000000..2e292b98 --- /dev/null +++ b/codespeed/pr_runner/__init__.py @@ -0,0 +1,60 @@ +"""Benchmark a pull request on demand. + +A signed-in member queues a run from the "PR Runner" tab. Benchmark machines +pull work from that queue over a small token-authenticated API, upload +results through the ordinary ``/result/add/json/`` endpoint, and report +back; Codespeed then posts a comparison to the pull request. + +Nothing here knows how a machine builds or benchmarks anything. That keeps +the queue usable from a cron job, a systemd service, a CI system or a GitHub +Actions workflow alike. See ``documentation/pr-runner.md``. +""" +from functools import wraps + +from django.conf import settings +from django.http import Http404 + +# Reasons a user may not start a run, for the page to explain +SIGN_IN = 'sign-in' +STALE = 'stale' +FORBIDDEN = 'forbidden' + + +def enabled(): + return bool(settings.PR_RUNNER_ENABLED) + + +def feature_required(view): + """404 unless the site has turned the PR runner on.""" + @wraps(view) + def wrapper(request, *args, **kwargs): + if not enabled(): + raise Http404 + return view(request, *args, **kwargs) + return wrapper + + +def start_refusal(user): + """Why `user` may not start runs, or None when they may. + + Two ways in: the ``codespeed.add_prrun`` permission, for local accounts + an administrator trusts, or a GitHub sign-in whose team check is recent. + Signing in with GitHub already required membership of an allowed team; + the freshness rule bounds how long a removal from that team takes to + bite. + """ + from codespeed.github_auth import is_fresh + from codespeed.models import GitHubAccount + + if not user.is_authenticated: + return SIGN_IN + if not user.is_active: + return FORBIDDEN + if user.has_perm('codespeed.add_prrun'): + return None + account = GitHubAccount.objects.filter(user=user).first() + if account is None: + return FORBIDDEN + if not is_fresh(account): + return STALE + return None diff --git a/codespeed/pr_runner/api.py b/codespeed/pr_runner/api.py new file mode 100644 index 00000000..0a5fed9c --- /dev/null +++ b/codespeed/pr_runner/api.py @@ -0,0 +1,142 @@ +"""The API benchmark machines use to take and report on runs. + +Two calls, both authenticated with the runner's own bearer token: + +``POST /pr-runner/api/claim/`` + Take the next queued run. Answers 200 with the job as JSON, or 204 when + there is nothing to do. ``run_id`` asks for one particular run. + +``POST /pr-runner/api/runs//`` + Report on a run this runner claimed: ``status`` is ``running`` (a + heartbeat, with an optional progress ``message``), ``completed`` or + ``failed``. ``base_sha`` reports the merge base once the runner has + worked it out; the answer then carries ``skip_base``. The answer always + says whether the run was cancelled meanwhile, so the machine can stop + early. + +Results themselves go to ``/result/add/json/`` as they always have. +""" +import json + +from django.core.exceptions import ValidationError +from django.core.validators import URLValidator +from django.http import HttpResponse, JsonResponse +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_POST + +from codespeed.models import PRRun, PRRunner + +from . import feature_required, queue +from .git import SHA_RE + +_validate_url = URLValidator(schemes=['http', 'https']) + + +def _error(status, message): + return JsonResponse({'error': message}, status=status) + + +def _runner(request): + scheme, _, token = request.META.get('HTTP_AUTHORIZATION', '').partition(' ') + if scheme.lower() != 'bearer': + return None + return PRRunner.for_token(token.strip()) + + +def _payload(request): + if request.content_type == 'application/json': + try: + data = json.loads(request.body or b'{}') + except ValueError: + return None + return data if isinstance(data, dict) else None + return request.POST + + +def _run_url(request, run): + return request.build_absolute_uri(run.get_absolute_url()) + + +@csrf_exempt +@require_POST +@feature_required +def claim(request): + runner = _runner(request) + if runner is None: + return _error(401, 'Unknown or disabled runner token.') + data = _payload(request) + if data is None: + return _error(400, 'The body is not a JSON object.') + run_id = data.get('run_id') + if run_id in (None, ''): + run_id = None + else: + try: + run_id = int(run_id) + except (TypeError, ValueError): + return _error(400, 'run_id is not a number.') + + run = queue.claim(runner, run_id) + if run is None: + return HttpResponse(status=204) + return JsonResponse(queue.job(run, _run_url(request, run))) + + +@csrf_exempt +@require_POST +@feature_required +def update(request, run_id): + runner = _runner(request) + if runner is None: + return _error(401, 'Unknown or disabled runner token.') + # A run that belongs to another runner looks the same as no run at all + run = PRRun.objects.select_related( + 'runner__environment', 'project', 'executable' + ).filter(pk=run_id, runner=runner).first() + if run is None: + return _error(404, 'No such run for this runner.') + data = _payload(request) + if data is None: + return _error(400, 'The body is not a JSON object.') + + status = data.get('status') + message = str(data.get('message') or '') + log_url = str(data.get('log_url') or '') + if log_url: + try: + _validate_url(log_url) + except ValidationError: + return _error(400, 'log_url is not an http(s) URL.') + if len(log_url) > 200: + return _error(400, 'log_url is too long.') + if status not in (PRRun.RUNNING, PRRun.COMPLETED, PRRun.FAILED): + return _error(400, 'status must be running, completed or failed.') + base_sha = str(data.get('base_sha') or '') + if base_sha and not SHA_RE.match(base_sha): + return _error(400, 'base_sha is not a full commit id.') + + skip_base = None + if run.status == PRRun.RUNNING: + queue.heartbeat(run, message if status == PRRun.RUNNING else '', + log_url) + if base_sha: + skip_base = queue.record_base(run, base_sha) + if skip_base is None: + return _error(409, 'Run %d already has base %s.' % ( + run.pk, run.base_sha)) + if status == PRRun.COMPLETED: + queue.complete(run, _run_url(request, run)) + elif status == PRRun.FAILED: + queue.fail(run, message) + run.refresh_from_db() + elif run.status != PRRun.CANCELLED: + # Finished, timed out or never claimed: the runner is out of step + return _error(409, 'Run %d is %s.' % (run.pk, run.status)) + + answer = { + 'status': run.status, + 'cancelled': run.status == PRRun.CANCELLED, + } + if skip_base is not None: + answer['skip_base'] = skip_base + return JsonResponse(answer) diff --git a/codespeed/pr_runner/git.py b/codespeed/pr_runner/git.py new file mode 100644 index 00000000..3b171e21 --- /dev/null +++ b/codespeed/pr_runner/git.py @@ -0,0 +1,83 @@ +"""Find a pull request's commit with git alone. + +GitHub publishes every pull request as ``refs/pull//head``, and keeps +``refs/pull//merge`` only while the pull request is open and merges +cleanly. One ``git ls-remote`` therefore tells us whether the pull request +is open and which commit it is at, with no API token and no rate limit. +""" +import os +import re +import subprocess + +from codespeed.github_api import repo_slug +from codespeed.models import Project + +SHA_RE = re.compile(r'^[0-9a-f]{40}$') +TIMEOUT = 30 +_PR_URL_RE = re.compile( + r'^https?://github\.com/(?P[\w.-]+)/(?P[\w.-]+)/pull/(?P\d+)') + + +class PullRequestError(Exception): + """Something about the pull request that the requester should be told.""" + + +def repository_url(project): + """The project's GitHub clone URL, or an error for other projects.""" + slug = None + if project.repo_type == Project.GITHUB: + slug = repo_slug(project.repo_path) + if slug is None: + raise PullRequestError( + '%s is not set up as a GitHub project.' % project.name) + return 'https://github.com/%s/%s' % slug + + +def parse_number(project, text): + """The pull request number in "123", "#123" or a pull request URL.""" + text = (text or '').strip() + m = _PR_URL_RE.match(text) + if m: + if (m.group('owner'), m.group('name')) != repo_slug(project.repo_path): + return None + return int(m.group('n')) + text = text.lstrip('#') + if text.isdigit() and len(text) <= 9 and int(text) > 0: + return int(text) + return None + + +def ls_remote(url, refs): + """``{ref: sha}`` for the refs the remote has among `refs`.""" + env = dict(os.environ, GIT_TERMINAL_PROMPT='0', LC_ALL='C') + try: + done = subprocess.run( + ['git', 'ls-remote', '--', url] + list(refs), + capture_output=True, text=True, timeout=TIMEOUT, env=env) + except FileNotFoundError: + raise PullRequestError('git is not installed on this server.') from None + except subprocess.TimeoutExpired: + raise PullRequestError('GitHub did not answer in time.') from None + if done.returncode: + raise PullRequestError( + 'Could not reach the repository: %s' % done.stderr.strip()[-200:]) + found = {} + for line in done.stdout.splitlines(): + sha, _, ref = line.partition('\t') + if SHA_RE.match(sha) and ref: + found[ref] = sha + return found + + +def pull_request_head(project, number): + """The commit an open pull request is at right now.""" + head = 'refs/pull/%d/head' % number + merge = 'refs/pull/%d/merge' % number + refs = ls_remote(repository_url(project), [head, merge]) + if head not in refs: + raise PullRequestError('There is no pull request #%d.' % number) + if merge not in refs: + raise PullRequestError( + 'Pull request #%d is closed, or does not merge cleanly into %s.' + % (number, project.default_branch)) + return refs[head] diff --git a/codespeed/pr_runner/github.py b/codespeed/pr_runner/github.py new file mode 100644 index 00000000..e4cbca7c --- /dev/null +++ b/codespeed/pr_runner/github.py @@ -0,0 +1,43 @@ +"""The two GitHub API calls the PR runner makes, with the site's token: +commenting on the pull request and, optionally, waking a workflow.""" +from urllib.parse import quote + +from django.conf import settings + +from codespeed.github_api import api, repo_slug +from codespeed.models import Project + +from .git import PullRequestError + + +def repo_path(project): + """``/repos//`` for a project hosted on GitHub.""" + slug = None + if project.repo_type == Project.GITHUB: + slug = repo_slug(project.repo_path) + if slug is None: + raise PullRequestError( + '%s is not set up as a GitHub project.' % project.name) + return '/repos/%s/%s' % slug + + +def post_comment(project, number, body): + """Comment on the pull request and return the comment's URL.""" + data = api('POST', '%s/issues/%d/comments' % (repo_path(project), number), + token=settings.PR_RUNNER_GITHUB_TOKEN, data={'body': body}) + return data.get('html_url', '') + + +def dispatch_workflow(runner, run): + """Wake a GitHub Actions hosted runner for one run. + + The workflow is told only which run to pick up. It fetches the details + through the runner API with its own token, so nothing that matters + travels in workflow inputs, which anyone who can see the run can read. + """ + owner, _, name = runner.workflow_repo.partition('/') + api('POST', '/repos/%s/%s/actions/workflows/%s/dispatches' % ( + quote(owner, safe=''), quote(name, safe=''), + quote(runner.workflow_file, safe='')), + token=settings.PR_RUNNER_GITHUB_TOKEN, + data={'ref': runner.workflow_ref, 'inputs': {'run_id': str(run.pk)}}) diff --git a/codespeed/pr_runner/queue.py b/codespeed/pr_runner/queue.py new file mode 100644 index 00000000..097f3448 --- /dev/null +++ b/codespeed/pr_runner/queue.py @@ -0,0 +1,282 @@ +"""The life of a run: requested, claimed by a runner, reported, finished.""" +import logging +import re +from datetime import timedelta + +from django.conf import settings +from django.db import transaction +from django.utils import timezone + +from codespeed.github_api import GitHubError +from codespeed.models import Benchmark, PRRun, PRRunner, Result + +from . import git, github, report + +logger = logging.getLogger(__name__) + +# A base commit counts as already benchmarked when it has at least this share +# of the results a typical revision has; runs drop the odd benchmark. +BASE_MIN_COVERAGE = 0.8 +# Benchmark names end up in a runner's configuration, so keep them plain +_BENCHMARK_RE = re.compile(r'^[\w.+\-]+$') + + +class RunError(Exception): + """A request that cannot be queued; the text is shown to the requester.""" + + +def requester_name(user): + account = getattr(user, 'github_account', None) + return account.login if account else user.get_username() + + +def request_runs(user, project, pr_number, runners, executable, + benchmarks=(), reuse_base=True): + """Queue one run per runner and return them. + + The pull request's current commit is pinned here, with one + ``git ls-remote`` against GitHub, so the runs stay on that commit however + the pull request moves afterwards. The merge base is left to the runner, + which has a clone to work it out from. + """ + if not runners: + raise RunError('Choose at least one environment.') + if executable.project_id != project.pk: + raise RunError('That executable belongs to another project.') + if not pr_number: + raise RunError('Enter a pull request number.') + + benchmarks = sorted(set(benchmarks)) + for name in benchmarks: + if not _BENCHMARK_RE.match(name): + raise RunError('"%s" is not a usable benchmark name.' % name) + known = set(Benchmark.objects.filter( + name__in=benchmarks).values_list('name', flat=True)) + unknown = [name for name in benchmarks if name not in known] + if unknown: + raise RunError('Unknown benchmark: %s.' % ', '.join(unknown)) + + try: + head_sha = git.pull_request_head(project, pr_number) + except git.PullRequestError as e: + raise RunError(str(e)) from None + + benchmark_text = ','.join(benchmarks) + for runner in runners: + if not runner.enabled: + raise RunError('%s is not accepting runs.' % runner) + if not runner.executables.filter(pk=executable.pk).exists(): + raise RunError('%s cannot build %s.' % (runner, executable)) + duplicate = runner.runs.filter( + project=project, pr_number=pr_number, head_sha=head_sha, + executable=executable, benchmarks=benchmark_text, + status__in=PRRun.ACTIVE).first() + if duplicate: + raise RunError( + 'Run %d is already benchmarking this commit on %s.' % ( + duplicate.pk, runner)) + + limit = settings.PR_RUNNER_MAX_ACTIVE_PER_USER + active = PRRun.objects.filter( + requested_by=user, status__in=PRRun.ACTIVE).count() + if active + len(runners) > limit: + raise RunError( + 'You can have at most %d runs queued or running; you have %d.' % ( + limit, active)) + + with transaction.atomic(): + runs = [PRRun.objects.create( + runner=runner, project=project, executable=executable, + pr_number=pr_number, head_sha=head_sha, + base_branch=project.default_branch, + benchmarks=benchmark_text, reuse_base=reuse_base, + requested_by=user, requested_by_login=requester_name(user), + ) for runner in runners] + + for run in runs: + logger.info('%s queued PR run %d: #%d %s on %s', run.requested_by_login, + run.pk, pr_number, head_sha, run.runner) + notify_runner(run) + return runs + + +def notify_runner(run): + """Tell the runner there is work, where it does not poll for itself.""" + runner = run.runner + if runner.backend != PRRunner.GITHUB_ACTIONS: + return + try: + github.dispatch_workflow(runner, run) + except (GitHubError, git.PullRequestError) as e: + logger.warning('Could not dispatch PR run %d: %s', run.pk, e) + _finish(run, PRRun.FAILED, [PRRun.QUEUED], + 'Could not start the workflow: %s' % e) + run.refresh_from_db() + + +def _finish(run, status, expected, message=''): + """Move a run to a final state, unless something else got there first.""" + return PRRun.objects.filter(pk=run.pk, status__in=expected).update( + status=status, finished_at=timezone.now(), message=message[:500]) + + +def expire_stale_runs(): + """Fail running jobs whose runner went quiet. + + Called lazily from the page and the API, so no scheduler is needed. + """ + cutoff = timezone.now() - timedelta(seconds=settings.PR_RUNNER_STALE_SECONDS) + return PRRun.objects.filter( + status=PRRun.RUNNING, heartbeat_at__lt=cutoff + ).update(status=PRRun.FAILED, finished_at=timezone.now(), + message='The runner stopped reporting.') + + +def cancel(run, user): + done = _finish(run, PRRun.CANCELLED, PRRun.ACTIVE, + 'Cancelled by %s.' % requester_name(user)) + if done: + logger.info('%s cancelled PR run %d', requester_name(user), run.pk) + return bool(done) + + +def base_is_benchmarked(run): + """Whether the base commit already has results worth reusing.""" + if not run.base_sha: + return False + base = Result.objects.filter( + revision__commitid=run.base_sha, + revision__branch__project=run.project, + executable=run.executable, environment=run.runner.environment) + names = set(run.benchmark_list) + if names: + have = set(base.filter(benchmark__name__in=names).values_list( + 'benchmark__name', flat=True)) + return have == names + + count = base.count() + if not count: + return False + # Compare against the newest other revision on the branch, the way + # create_report_if_enough_data() judges whether a run is complete. + latest = Result.objects.filter( + revision__branch__project=run.project, + revision__branch__name=run.base_branch, + executable=run.executable, environment=run.runner.environment, + ).exclude(revision__commitid=run.base_sha).order_by( + '-revision__date').values_list('revision_id', flat=True).first() + if latest is None: + return True + typical = Result.objects.filter( + revision_id=latest, executable=run.executable, + environment=run.runner.environment).count() + return count >= BASE_MIN_COVERAGE * typical + + +def claim(runner, run_id=None): + """Hand the runner its oldest queued run, or the one it asks for. + + A machine benchmarks one thing at a time, so nothing is handed out while + it still has a run in progress. + """ + expire_stale_runs() + now = timezone.now() + PRRunner.objects.filter(pk=runner.pk).update(last_seen=now) + if runner.runs.filter(status=PRRun.RUNNING).exists(): + return None + queued = runner.runs.filter(status=PRRun.QUEUED).order_by('id') + if run_id is not None: + queued = queued.filter(pk=run_id) + for pk in list(queued.values_list('pk', flat=True)[:5]): + # The conditional update is the lock: only one caller sees 1 + claimed = PRRun.objects.filter(pk=pk, status=PRRun.QUEUED).update( + status=PRRun.RUNNING, started_at=now, heartbeat_at=now, + message='Picked up by the runner.') + if claimed: + return PRRun.objects.select_related( + 'runner__environment', 'project', 'executable').get(pk=pk) + return None + + +def job(run, run_url): + """What a runner needs to know to do the work. + + No base commit yet: the runner finds the merge base in its clone and + reports it, and the answer tells it whether that commit needs building. + """ + return { + 'id': run.pk, + 'url': run_url, + 'project': run.project.name, + 'repository': run.project.repo_path, + 'environment': run.runner.environment.name, + 'executable': run.executable.name, + 'pr_number': run.pr_number, + 'head_sha': run.head_sha, + 'head_branch': run.head_branch, + 'base_branch': run.base_branch, + 'reuse_base': run.reuse_base, + 'benchmarks': run.benchmarks, + } + + +def record_base(run, base_sha): + """Note the merge base the runner found; returns whether to skip it. + + The first report wins. A runner that changes its mind mid-run would + leave results filed under one commit and a report reading another. + """ + if run.base_sha and run.base_sha != base_sha: + return None + if not run.base_sha: + PRRun.objects.filter(pk=run.pk, base_sha='').update(base_sha=base_sha) + run.refresh_from_db(fields=['base_sha']) + return run.reuse_base and base_is_benchmarked(run) + + +def heartbeat(run, message='', log_url=''): + fields = {'heartbeat_at': timezone.now()} + if message: + fields['message'] = message[:500] + if log_url: + fields['log_url'] = log_url + now = fields['heartbeat_at'] + PRRunner.objects.filter(pk=run.runner_id).update(last_seen=now) + return PRRun.objects.filter(pk=run.pk, status=PRRun.RUNNING).update(**fields) + + +def fail(run, message): + return bool(_finish(run, PRRun.FAILED, [PRRun.RUNNING], + message or 'The runner reported a failure.')) + + +def complete(run, run_url): + """Close a run the runner says is done, and tell the pull request. + + The runner's word is checked against what it uploaded: with no results + for the pull request's commit there is nothing to report, so the run + fails instead. + """ + if not run.base_sha: + return fail(run, 'The runner never reported the base commit.') + data = report.comparison(run) + if data is None: + return fail(run, 'The runner finished without uploading results.') + if not _finish(run, PRRun.COMPLETED, [PRRun.RUNNING], + 'Geometric mean: %s.' % report.format_speedup( + data['geomean'])): + return False + + if settings.PR_RUNNER_COMMENT and settings.PR_RUNNER_GITHUB_TOKEN: + try: + url = github.post_comment( + run.project, run.pr_number, + report.markdown(run, data, run_url)) + except (GitHubError, git.PullRequestError) as e: + # The results are in and the page shows them; a lost comment is + # not worth failing the run for. + logger.warning('Could not comment on #%d for PR run %d: %s', + run.pr_number, run.pk, e) + else: + PRRun.objects.filter(pk=run.pk).update(comment_url=url) + return True diff --git a/codespeed/pr_runner/report.py b/codespeed/pr_runner/report.py new file mode 100644 index 00000000..807c733d --- /dev/null +++ b/codespeed/pr_runner/report.py @@ -0,0 +1,150 @@ +"""Compare a pull request's results with its base, for the page and the +comment posted to the pull request.""" +import math +import re + +from django.conf import settings + +from codespeed.models import Result + +_UNSAFE = re.compile(r'[^\w.+\-]') + + +def comparison(run): + """Base against pull request, benchmark by benchmark. + + Returns None while the pull request has no results. ``speedup`` is + oriented so that above 1 always means the pull request is better, + whichever way the benchmark's units run. + """ + def results(commitid): + found = Result.objects.filter( + revision__commitid=commitid, + revision__branch__project=run.project, + executable=run.executable, + environment=run.runner.environment, + ).select_related('benchmark') + names = run.benchmark_list + if names: + found = found.filter(benchmark__name__in=names) + return {r.benchmark_id: r for r in found} + + head = results(run.head_sha) + if not head: + return None + base = results(run.base_sha) if run.base_sha else {} + + threshold = getattr(settings, 'CHANGE_THRESHOLD', None) or 3.0 + rows, missing = [], [] + for benchmark_id, new in head.items(): + benchmark = new.benchmark + old = base.get(benchmark_id) + if old is None or old.value <= 0 or new.value <= 0: + missing.append(benchmark.name) + continue + if benchmark.lessisbetter: + speedup = old.value / new.value + else: + speedup = new.value / old.value + change = (new.value - old.value) / old.value * 100 + rows.append({ + 'name': benchmark.name, + 'units': benchmark.units, + 'base': old.value, + 'head': new.value, + 'change': change, + 'speedup': speedup, + 'significant': abs(change) >= threshold, + 'better': speedup > 1, + }) + rows.sort(key=lambda row: row['name']) + + geomean = None + if rows: + geomean = math.exp( + sum(math.log(row['speedup']) for row in rows) / len(rows)) + significant = [row for row in rows if row['significant']] + return { + 'rows': rows, + 'missing': sorted(missing), + 'geomean': geomean, + 'threshold': threshold, + 'faster': sum(1 for row in significant if row['better']), + 'slower': sum(1 for row in significant if not row['better']), + 'unchanged': len(rows) - len(significant), + } + + +def format_value(value, units): + if units in ('seconds', 's'): + for factor, unit in ((1, 's'), (1e-3, 'ms'), (1e-6, 'us')): + if value >= factor: + return '%.3g %s' % (value / factor, unit) + return '%.3g ns' % (value / 1e-9) + return '%.4g %s' % (value, units) + + +def format_speedup(speedup): + if speedup is None: + return 'n/a' + if round(speedup, 2) == 1: + return 'no change' + if speedup > 1: + return '%.2fx faster' % speedup + return '%.2fx slower' % (1 / speedup) + + +def _code(text): + """`text` as inline code. Names and labels arrive from uploads, so + anything that could break out of the code span or the table is dropped.""" + return '`%s`' % _UNSAFE.sub('?', text) + + +def _table(rows): + lines = ['| Benchmark | Base | PR | Change |', '|:--|--:|--:|:--|'] + for row in rows: + lines.append('| %s | %s | %s | %s |' % ( + _code(row['name']), + format_value(row['base'], row['units']), + format_value(row['head'], row['units']), + format_speedup(row['speedup']))) + return '\n'.join(lines) + + +def markdown(run, data, run_url): + """The comment posted to the pull request.""" + out = [ + '### Benchmark results: %s on %s' % ( + _code(run.executable.name), _code(run.runner.environment.name)), + '', + 'Comparing %s with its merge base %s on %s.' % ( + run.head_sha, run.base_sha, _code(run.base_branch)), + '', + ] + if not data['rows']: + out.append('The base commit has no results to compare against.') + else: + out.append( + '**Geometric mean: %s** across %d benchmarks: %d faster, ' + '%d slower, %d within %g%%.' % ( + format_speedup(data['geomean']), len(data['rows']), + data['faster'], data['slower'], data['unchanged'], + data['threshold'])) + # Regressions first: they are what a reviewer needs to see + significant = sorted( + (row for row in data['rows'] if row['significant']), + key=lambda row: row['speedup']) + rest = [row for row in data['rows'] if not row['significant']] + if significant: + out += ['', _table(significant)] + if rest: + out += ['', '
', + '%d within %g%%' % ( + len(rest), data['threshold']), + '', _table(rest), '', '
'] + if data['missing']: + out += ['', 'No base result for: %s.' % ', '.join( + _code(name) for name in data['missing'])] + out += ['', '[Full results](%s), requested by %s.' % ( + run_url, _UNSAFE.sub('', run.requested_by_login) or 'an administrator')] + return '\n'.join(out) diff --git a/codespeed/pr_runner/urls.py b/codespeed/pr_runner/urls.py new file mode 100644 index 00000000..cd9763c8 --- /dev/null +++ b/codespeed/pr_runner/urls.py @@ -0,0 +1,15 @@ +from django.urls import re_path + +from . import api, views + +urlpatterns = [ + re_path(r'^pr-runner/$', views.index, name='pr-runner'), + re_path(r'^pr-runner/new/$', views.create, name='pr-run-create'), + re_path(r'^pr-runner/(?P\d+)/$', views.detail, name='pr-run'), + re_path(r'^pr-runner/(?P\d+)/cancel/$', views.cancel, + name='pr-run-cancel'), + # For benchmark machines + re_path(r'^pr-runner/api/claim/$', api.claim, name='pr-runner-claim'), + re_path(r'^pr-runner/api/runs/(?P\d+)/$', api.update, + name='pr-runner-update'), +] diff --git a/codespeed/pr_runner/views.py b/codespeed/pr_runner/views.py new file mode 100644 index 00000000..af523712 --- /dev/null +++ b/codespeed/pr_runner/views.py @@ -0,0 +1,144 @@ +"""The "PR Runner" tab: queue a run, and watch the ones in flight.""" +from django.contrib import messages +from django.http import HttpResponseForbidden +from django.shortcuts import get_object_or_404, redirect, render +from django.views.decorators.http import require_GET, require_POST + +from codespeed.github_auth import login_enabled +from codespeed.models import Benchmark, Executable, PRRun, PRRunner, Project + +from . import feature_required, git, queue, report, start_refusal + +RECENT_RUNS = 50 + + +def _int(value): + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _form_context(): + runners = list(PRRunner.objects.filter(enabled=True).select_related( + 'environment').prefetch_related('executables').order_by( + 'environment__name')) + offered = {exe.pk for runner in runners for exe in runner.executables.all()} + # Only projects with something to run are worth choosing between; with + # no runners yet, list every GitHub project so the page is not blank. + projects = Project.objects.filter(repo_type=Project.GITHUB).order_by('name') + if offered: + projects = projects.filter(executables__in=offered).distinct() + return { + 'runners': [{ + 'runner': runner, + 'executables': ','.join( + str(exe.pk) for exe in runner.executables.all()), + } for runner in runners], + 'projects': projects, + 'executables': Executable.objects.filter(pk__in=offered).order_by('name'), + 'benchmarks': Benchmark.objects.order_by('name').values_list( + 'name', flat=True).distinct(), + } + + +@feature_required +@require_GET +def index(request): + queue.expire_stale_runs() + refusal = start_refusal(request.user) + context = { + 'runs': PRRun.objects.select_related( + 'runner__environment', 'executable', 'project')[:RECENT_RUNS], + 'refusal': refusal, + 'can_start': refusal is None, + 'login_enabled': login_enabled(), + } + if refusal is None: + context.update(_form_context()) + return render(request, 'codespeed/pr_runner/index.html', context) + + +@feature_required +@require_GET +def detail(request, run_id): + queue.expire_stale_runs() + run = get_object_or_404(PRRun.objects.select_related( + 'runner__environment', 'executable', 'project'), pk=run_id) + data = report.comparison(run) + if data: + for row in data['rows']: + row['base_text'] = report.format_value(row['base'], row['units']) + row['head_text'] = report.format_value(row['head'], row['units']) + row['speedup_text'] = report.format_speedup(row['speedup']) + data['geomean_text'] = report.format_speedup(data['geomean']) + # Regressions first, then improvements, then everything unremarkable + data['rows'].sort(key=lambda row: ( + not row['significant'], row['speedup'] if row['significant'] else 0, + row['name'])) + try: + repo = git.repository_url(run.project) + except git.PullRequestError: + repo = None + return render(request, 'codespeed/pr_runner/detail.html', { + 'run': run, + 'comparison': data, + 'repo': repo, + 'can_cancel': run.is_active and start_refusal(request.user) is None, + }) + + +@feature_required +@require_POST +def create(request): + if start_refusal(request.user) is not None: + return HttpResponseForbidden('You may not start runs.') + post = request.POST + project = get_object_or_404(Project, pk=_int(post.get('project'))) + pr_number = git.parse_number(project, post.get('pull_request')) + executable = Executable.objects.filter( + pk=_int(post.get('executable'))).first() + runner_ids = [pk for pk in map(_int, post.getlist('runner')) if pk] + runners = list(PRRunner.objects.filter( + pk__in=runner_ids).select_related('environment')) + benchmarks = post.getlist('benchmark') if post.get('suite') == 'subset' else [] + + try: + if pr_number is None: + raise queue.RunError( + 'Enter a pull request number, or paste its URL.') + if executable is None: + raise queue.RunError('Choose an executable.') + if post.get('suite') == 'subset' and not benchmarks: + raise queue.RunError('Choose at least one benchmark.') + runs = queue.request_runs( + request.user, project, pr_number, runners, executable, + benchmarks=benchmarks, reuse_base=bool(post.get('reuse_base'))) + except queue.RunError as e: + messages.error(request, str(e)) + return redirect('pr-runner') + + failed = [run for run in runs if run.status == PRRun.FAILED] + for run in failed: + messages.error(request, 'Run %d on %s: %s' % ( + run.pk, run.runner, run.message)) + queued = len(runs) - len(failed) + if queued: + messages.success(request, 'Queued %d run%s for #%d.' % ( + queued, '' if queued == 1 else 's', runs[0].pr_number)) + if len(runs) == 1: + return redirect(runs[0]) + return redirect('pr-runner') + + +@feature_required +@require_POST +def cancel(request, run_id): + if start_refusal(request.user) is not None: + return HttpResponseForbidden('You may not cancel runs.') + run = get_object_or_404(PRRun, pk=run_id) + if queue.cancel(run, request.user): + messages.success(request, 'Run %d cancelled.' % run.pk) + else: + messages.error(request, 'Run %d had already finished.' % run.pk) + return redirect(run) diff --git a/codespeed/settings.py b/codespeed/settings.py index 3d6959a9..f124d2e6 100644 --- a/codespeed/settings.py +++ b/codespeed/settings.py @@ -111,3 +111,38 @@ GITHUB_OAUTH_TOKEN = None # Github oAuth token to use when using Github repo type. If not # specified, it will utilize unauthenticated requests which have # low rate limits. + +## GitHub sign-in ## +# Register an OAuth app (or a GitHub App) whose callback URL is +# https:///accounts/github/callback/ and put its credentials in your +# local settings. The "Sign in with GitHub" button only appears once both +# are set. +GITHUB_LOGIN_CLIENT_ID = None +GITHUB_LOGIN_CLIENT_SECRET = None + +GITHUB_LOGIN_ALLOWED_TEAMS = [] # Teams whose members may sign in, as + # 'org/team-slug'. + # Example: ['python/python-core'] + +GITHUB_LOGIN_ALLOWED_USERS = [] # GitHub logins allowed regardless of team + +GITHUB_LOGIN_RECHECK_SECONDS = 24 * 60 * 60 # How long a passed membership + # check is trusted before the + # user must sign in again + +## PR runner ## +PR_RUNNER_ENABLED = False # Show the "PR Runner" tab, where signed-in users + # queue benchmark runs for pull requests + +PR_RUNNER_GITHUB_TOKEN = None # Token used to post the report comment on the + # pull request and to trigger workflows. Without + # it no comment is posted. Finding pull requests + # needs no token: `git ls-remote` does that. + +PR_RUNNER_COMMENT = True # Post the comparison to the pull request when a run + # completes + +PR_RUNNER_MAX_ACTIVE_PER_USER = 4 # Queued or running runs one person may have + +PR_RUNNER_STALE_SECONDS = 30 * 60 # A running job whose runner has been silent + # this long is marked as failed diff --git a/codespeed/static/css/main.css b/codespeed/static/css/main.css index 589aaa46..8688d82b 100644 --- a/codespeed/static/css/main.css +++ b/codespeed/static/css/main.css @@ -110,6 +110,44 @@ ul#links li a:hover, ul#links li a:focus { text-decoration: none; } +/* Sign-in button and signed-in user, at the right edge of the header */ +div#account { + display: flex; + align-items: center; + gap: 0.5em; + margin-left: 0.4em; + padding-left: 0.9em; + border-left: 1px solid var(--border); + font-size: 0.95em; + white-space: nowrap; +} +div#account form { margin: 0; } +div#account img.avatar { height: 24px; width: 24px; border-radius: 50%; } +div#account .account-name { color: var(--text); font-weight: 500; } +div#title a.github-signin { + gap: 0.45em; + padding: 0.35em 0.75em; + color: var(--text); + font-weight: 500; + background-color: var(--surface-alt); + border: 1px solid var(--border-strong); + border-radius: var(--radius-sm); + text-decoration: none; +} +div#title a.github-signin:hover, div#title a.github-signin:focus { + border-color: var(--accent); + text-decoration: none; +} +button.linkbutton { + padding: 0; + font: inherit; + color: var(--text-muted); + background: none; + border: 0; + cursor: pointer; +} +button.linkbutton:hover, button.linkbutton:focus { color: var(--accent); text-decoration: underline; } + div#wrapper { margin: 0; } /* Section tabs */ diff --git a/codespeed/static/css/pr_runner.css b/codespeed/static/css/pr_runner.css new file mode 100644 index 00000000..d4fdde08 --- /dev/null +++ b/codespeed/static/css/pr_runner.css @@ -0,0 +1,145 @@ +/* PR Runner tab. Colours come from the custom properties in main.css, so + this follows whatever theme the site is in. */ + +div#prrunner { display: grid; grid-template-columns: minmax(0, 1fr); gap: 0.8em; align-items: start; } +div#prrunner.with-form { grid-template-columns: minmax(20em, 26em) minmax(0, 1fr); } +div#prrunner > #pr-flash, div#prrunner > p.back { grid-column: 1 / -1; } +div#prrunner > #pr-flash:empty { display: none; } + +.prcard { + padding: 1em 1.2em 1.2em; + background-color: var(--surface); + border: 1px solid var(--border); + border-radius: var(--radius); + box-shadow: var(--shadow); +} +.prcard h2 { margin: 0 0 0.7em; font-size: 1.1em; font-weight: 600; } +.prcard p { margin: 0 0 0.8em; line-height: 1.45; } +.prcard p:last-child { margin-bottom: 0; } +.prcard code { font-family: var(--font-mono); font-size: 0.9em; } +.prcard .empty { color: var(--text-muted); } +div#prrunner p.back { margin: 0; font-size: 0.93em; } + +/* Flash messages */ +p.flash { + margin: 0 0 0.5em; + padding: 0.7em 1em; + background-color: var(--surface); + border: 1px solid var(--border); + border-left: 3px solid var(--accent); + border-radius: var(--radius-sm); +} +p.flash:last-child { margin-bottom: 0; } +p.flash-error { border-left-color: var(--status-red); } +p.flash-success { border-left-color: var(--status-green); } + +/* The form */ +#pr-run-form fieldset { margin: 0 0 1.1em; padding: 0; border: 0; min-width: 0; } +#pr-run-form legend { + padding: 0; + margin-bottom: 0.4em; + font-size: 0.8em; + font-weight: 600; + letter-spacing: 0.04em; + text-transform: uppercase; + color: var(--text-muted); +} +#pr-run-form select, #pr-run-form input[type="text"] { width: 100%; padding: 0.4em 0.5em; } +#pr-run-form label.field { display: block; margin-bottom: 0.6em; font-size: 0.93em; } +#pr-run-form label.option { display: flex; align-items: baseline; gap: 0.45em; margin-top: 0.7em; font-size: 0.93em; cursor: pointer; } +#pr-run-form label.option input { flex: none; margin: 0; } +ul.choices { list-style: none; margin: 0; padding: 0; } +ul.choices li { padding: 0.15em 0; } +ul.choices label { cursor: pointer; } +ul.choices input:disabled + *, ul.choices label.unavailable { color: var(--text-faint); cursor: default; } +ul.choices .choice-note { display: block; margin-left: 1.7em; font-size: 0.87em; color: var(--text-muted); } +ul.choices.scroll { + max-height: 13em; + margin-top: 0.4em; + padding: 0.3em 0.6em; + overflow-y: auto; + border: 1px solid var(--border); + border-radius: var(--radius-sm); + font-size: 0.93em; +} +div#pr-benchmarks { margin-top: 0.5em; } + +#pr-run-form p.hint { margin: 0.4em 0 0; font-size: 0.87em; color: var(--text-muted); } + +p.caution { + padding: 0.6em 0.8em; + font-size: 0.9em; + background-color: var(--surface-alt); + border-left: 3px solid var(--status-yellow); + border-radius: var(--radius-sm); +} + +/* Buttons */ +.primary, a.primary:link, a.primary:visited, button.secondary { + display: inline-block; + padding: 0.5em 1.1em; + font: inherit; + font-weight: 600; + border-radius: var(--radius-sm); + cursor: pointer; + text-decoration: none; +} +.primary, a.primary:link, a.primary:visited { color: var(--surface); background-color: var(--accent); border: 1px solid var(--accent); } +.primary:hover, a.primary:hover { background-color: var(--accent-hover); border-color: var(--accent-hover); text-decoration: none; } +.primary:disabled { opacity: 0.45; cursor: not-allowed; } +button.secondary { color: var(--text); background-color: var(--surface-alt); border: 1px solid var(--border-strong); font-weight: 500; } +button.secondary:hover { border-color: var(--status-red); } + +/* Runs. Not table.tablesorter: that one is set in monospace and + right-aligned for columns of numbers, and these are mostly words. */ +table.prtable { + width: 100%; + font-size: 0.93em; + text-align: left; + border-collapse: separate; + border-spacing: 0; + border: 1px solid var(--border); + border-radius: var(--radius-sm); + overflow: hidden; +} +table.prtable th { + padding: 0.5em 0.8em; + font-weight: 600; + white-space: nowrap; + color: var(--text-muted); + background-color: var(--surface-alt); + border-bottom: 1px solid var(--border); +} +table.prtable td { padding: 0.45em 0.8em; white-space: nowrap; border-top: 1px solid var(--border); } +table.prtable tbody tr:first-child td { border-top: 0; } +/* The benchmarks column takes whatever width is left and truncates to fit, + so the table never grows wider than its card */ +table.prtable td.pr-benchmarks-cell { width: 100%; max-width: 0; overflow: hidden; text-overflow: ellipsis; } +div.table-scroll { overflow-x: auto; } +table.prtable.runs { min-width: 58em; } +table.prtable td.number, table.prtable th.number { text-align: right; font-variant-numeric: tabular-nums; } +table.prtable.comparison { max-width: 46em; } +.run-status { + display: inline-block; + padding: 0.1em 0.6em; + font-size: 0.87em; + font-weight: 600; + white-space: nowrap; + color: var(--text-muted); + background-color: var(--surface-alt); + border: 1px solid var(--border); + border-radius: 1em; +} +.run-status.run-running { color: var(--accent); background-color: var(--accent-soft); border-color: var(--accent); } +.run-status.run-completed { color: var(--status-green-text); background-color: var(--status-green); border-color: var(--status-green); } +.run-status.run-failed { color: var(--status-red-text); background-color: var(--status-red); border-color: var(--status-red); } +.run-head { display: flex; flex-wrap: wrap; align-items: center; gap: 0.8em; margin-bottom: 0.7em; } +.run-head h2 { margin: 0; overflow-wrap: anywhere; } +.run-head form { margin: 0 0 0 auto; } +p.run-message { color: var(--text-muted); } +.prcard table.info { max-width: 60em; margin-bottom: 0; } +.prcard table.info th.infofirst { width: 9em; white-space: nowrap; } + +@media only screen and (max-width: 1005px) { + div#prrunner.with-form { grid-template-columns: minmax(0, 1fr); } +} diff --git a/codespeed/static/js/pr_runner.js b/codespeed/static/js/pr_runner.js new file mode 100644 index 00000000..78f27594 --- /dev/null +++ b/codespeed/static/js/pr_runner.js @@ -0,0 +1,50 @@ +/* The "Benchmark a pull request" form on the PR Runner tab: offer only this + * project's executables and the environments that can build the chosen one, + * and enable "Start run" once there is enough to go on. */ +$(function() { + var form = $('#pr-run-form'); + if (!form.length) { return; } + + function project() { + return form.find('[name=project]').val(); + } + + function refresh() { + var executable = form.find('[name=executable]'); + executable.find('option').each(function() { + var mine = String($(this).data('project')) === String(project()); + $(this).prop('hidden', !mine).prop('disabled', !mine); + }); + if (executable.find('option:selected').prop('disabled')) { + executable.val(executable.find('option:not(:disabled)').first().val()); + } + var chosen = executable.val(); + form.find('[name=runner]').each(function() { + var offers = String($(this).data('executables')).split(','); + var able = chosen !== null && $.inArray(String(chosen), offers) !== -1; + $(this).prop('disabled', !able); + if (!able) { $(this).prop('checked', false); } + $(this).closest('label').toggleClass('unavailable', !able); + }); + var subset = form.find('[name=suite]:checked').val() === 'subset'; + $('#pr-benchmarks').prop('hidden', !subset); + var ready = $.trim(form.find('[name=pull_request]').val() || '') !== '' && + form.find('[name=runner]:checked').length > 0 && + (!subset || form.find('[name=benchmark]:checked').length > 0); + $('#pr-submit').prop('disabled', !ready); + } + + form.on('change input', 'select, input', refresh); + $('#pr-benchmark-filter').on('input', function() { + var needle = $(this).val().toLowerCase(); + $('#pr-benchmarks li').each(function() { + var name = $(this).find('input').val().toLowerCase(); + $(this).prop('hidden', name.indexOf(needle) === -1); + }); + }); + form.on('submit', function() { + $('#pr-submit').prop('disabled', true).text('Queuing\u2026'); + }); + + refresh(); +}); diff --git a/codespeed/templates/codespeed/account_widget.html b/codespeed/templates/codespeed/account_widget.html new file mode 100644 index 00000000..063f7a45 --- /dev/null +++ b/codespeed/templates/codespeed/account_widget.html @@ -0,0 +1,18 @@ +{% if user %} +
+ {% if account.avatar_url %}{% endif %} + +
+ {% csrf_token %} + + +
+
+{% elif login_enabled %} + +{% endif %} diff --git a/codespeed/templates/codespeed/base.html b/codespeed/templates/codespeed/base.html index 1d9b9bb4..2025f603 100644 --- a/codespeed/templates/codespeed/base.html +++ b/codespeed/templates/codespeed/base.html @@ -1,4 +1,4 @@ -{% load static %} +{% load static account pr_runner %} @@ -29,6 +29,7 @@

{% block page_title %}Speed Center{% endblock page_title %}

+ {% account_widget %} {% endblock top_nav %} {% endblock page_header %} @@ -40,6 +41,7 @@

{% block page_title %}Speed Center{% endblock page_title %}

  • Changes{% endblock %}
  • Timeline{% endblock %}
  • Comparison{% endblock %}
    • + {% pr_runner_enabled as show_pr_runner %}{% if show_pr_runner %}
  • PR Runner{% endblock %}
    • {% endif %} {% endblock %} diff --git a/codespeed/templates/codespeed/pr_runner/_messages.html b/codespeed/templates/codespeed/pr_runner/_messages.html new file mode 100644 index 00000000..09f3364a --- /dev/null +++ b/codespeed/templates/codespeed/pr_runner/_messages.html @@ -0,0 +1,3 @@ +{% for message in messages %} +

    {{ message }}

    +{% endfor %} diff --git a/codespeed/templates/codespeed/pr_runner/_status.html b/codespeed/templates/codespeed/pr_runner/_status.html new file mode 100644 index 00000000..6fc345e4 --- /dev/null +++ b/codespeed/templates/codespeed/pr_runner/_status.html @@ -0,0 +1 @@ +{{ run.get_status_display }} diff --git a/codespeed/templates/codespeed/pr_runner/detail.html b/codespeed/templates/codespeed/pr_runner/detail.html new file mode 100644 index 00000000..6591f4b0 --- /dev/null +++ b/codespeed/templates/codespeed/pr_runner/detail.html @@ -0,0 +1,87 @@ +{% extends "codespeed/base_site.html" %} +{% load static %} + +{% block title %}{{ block.super }}: PR run {{ run.id }}{% endblock %} + +{% block extra_head %} + + {# Keep an eye on a run in flight without anyone having to reload #} + {% if run.is_active %}{% endif %} +{% endblock %} + +{% block nav-pr-runner %}class="current">PR Runner{% endblock %} + +{% block body %} +
    +
    {% include "codespeed/pr_runner/_messages.html" %}
    + +
    +
    +

    Run {{ run.id }}: pull request #{{ run.pr_number }}

    + {% include "codespeed/pr_runner/_status.html" %} + {% if can_cancel %} +
    + {% csrf_token %} + +
    + {% endif %} +
    + {% if run.message %}

    {{ run.message }}

    {% endif %} + + + + + + + + + + + {% if run.started_at %}{% endif %} + {% if run.finished_at %}{% endif %} + {% if run.log_url %}{% endif %} + {% if run.comment_url %}{% endif %} + +
    Pull request{% if repo %}#{{ run.pr_number }}{% else %}#{{ run.pr_number }}{% endif %}
    Commit{% if repo %}{{ run.head_sha }}{% else %}{{ run.head_sha }}{% endif %}
    Compared with{% if not run.base_sha %}its merge base on {{ run.base_branch }}, which the runner reports once it has fetched the commits{% else %}{% if repo %}{{ run.base_sha }}{% else %}{{ run.base_sha }}{% endif %}, its merge base on {{ run.base_branch }}{% endif %}
    Environment{{ run.runner.environment.name }}
    Executable{{ run.executable.name }}
    Benchmarks{% if run.benchmarks %}{{ run.benchmark_list|join:", " }}{% else %}The whole suite{% endif %}
    Requested by{{ run.requested_by_login }}, {{ run.created_at|date:"Y-m-d H:i e" }}
    Started{{ run.started_at|date:"Y-m-d H:i e" }}
    Finished{{ run.finished_at|date:"Y-m-d H:i e" }}
    Log{{ run.log_url }}
    ReportComment on the pull request
    +
    + + {% if comparison %} +
    +

    Results

    + {% if comparison.rows %} +

    + Geometric mean: {{ comparison.geomean_text }} across + {{ comparison.rows|length }} benchmark{{ comparison.rows|length|pluralize }}. + {{ comparison.faster }} faster, {{ comparison.slower }} slower, + {{ comparison.unchanged }} within {{ comparison.threshold|floatformat }}%. +

    + + + + {% for row in comparison.rows %} + + + + + + + {% endfor %} + +
    BenchmarkBasePull requestChange
    {{ row.name }}{{ row.base_text }}{{ row.head_text }}{{ row.speedup_text }}
    + {% else %} +

    The pull request's results are in, but the base commit has none to compare them with.

    + {% endif %} + {% if comparison.missing %} +

    No base result for: {{ comparison.missing|join:", " }}.

    + {% endif %} +
    + {% elif run.is_active %} +
    +

    Results

    +

    Results appear here as the runner uploads them.

    +
    + {% endif %} + +

    All runs

    +
    +{% endblock body %} diff --git a/codespeed/templates/codespeed/pr_runner/index.html b/codespeed/templates/codespeed/pr_runner/index.html new file mode 100644 index 00000000..24b3d4bb --- /dev/null +++ b/codespeed/templates/codespeed/pr_runner/index.html @@ -0,0 +1,141 @@ +{% extends "codespeed/base_site.html" %} +{% load static pr_runner %} + +{% block title %}{{ block.super }}: PR Runner{% endblock %} +{% block description %}Benchmark a pull request on demand and compare it with its base{% endblock %} + +{% block extra_head %} + +{% endblock %} + +{% block nav-pr-runner %}class="current">PR Runner{% endblock %} + +{% block body %} +
    +
    {% include "codespeed/pr_runner/_messages.html" %}
    + +
    +

    Benchmark a pull request

    + {% if can_start %} +
    + {% csrf_token %} + +
    + Pull request + {% if projects|length > 1 %} + + {% else %} + {% for project in projects %}{% endfor %} + {% endif %} + {% if projects %} + +

    The pull request must be open and target the main branch.

    + {% else %} +

    No project here is hosted on GitHub, so there are no pull requests to benchmark.

    + {% endif %} +
    + +
    + Environments + {% if runners %} +
      + {% for item in runners %} +
    • + + {% if item.runner.notes %}{{ item.runner.notes }}{% endif %} +
      • + {% endfor %} +
    + {% else %} +

    No environments accept pull request runs yet. Once a benchmark machine is registered as a runner it will be listed here.

    + {% endif %} +
    + + {% if runners %} +
    + Executable + +
    + +
    + Benchmarks +
      +
      • +
      • +
    + + +
    + +

    The pull request's code is built and run on the machines you choose. Read the changes first, as you would before merging.

    + + {% endif %} +
    + {% elif refusal == 'stale' %} +

    Your team membership was last checked a while ago. Sign in again to start runs.

    + {% if login_enabled %}

    Sign in with GitHub

    {% endif %} + {% elif refusal == 'forbidden' %} +

    This account may not start runs.

    + {% else %} +

    Members of the project's teams can have a pull request built and benchmarked on the machines behind this site, and compared with the commit it branched from.

    + {% if login_enabled %} +

    Sign in with GitHub

    + {% else %} +

    Sign-in is not set up on this site yet.

    + {% endif %} + {% endif %} +
    + +
    +

    Recent runs

    + {% if runs %} +
    + + + + + + {% for run in runs %} + + + + + + + + + + + + {% endfor %} + +
    RunPull requestCommitEnvironmentExecutableBenchmarksRequested byStatusQueued
    {{ run.id }}#{{ run.pr_number }}{{ run.head_sha|slice:":7" }}{{ run.runner.environment.name }}{{ run.executable.name }}{% if run.benchmarks %}{{ run.benchmark_list|join:", " }}{% else %}all{% endif %}{{ run.requested_by_login }}{% include "codespeed/pr_runner/_status.html" %}{{ run.created_at|ago }}
    +
    + {% else %} +

    No pull request has been benchmarked yet.

    + {% endif %} +
    +
    +{% endblock body %} + +{% block extra_script %} +{{ block.super }} +{% if can_start %}{% endif %} +{% endblock %} diff --git a/codespeed/templates/codespeed/signin_denied.html b/codespeed/templates/codespeed/signin_denied.html new file mode 100644 index 00000000..75f76334 --- /dev/null +++ b/codespeed/templates/codespeed/signin_denied.html @@ -0,0 +1,13 @@ +{% extends "codespeed/base_site.html" %} + +{% block title %}{{ block.super }}: Sign in{% endblock %} + +{% block navigation %}{% endblock %} + +{% block body %} +
    +

    Could not sign you in

    +

    {{ message }}

    +

    Back to the home page

    +
    +{% endblock %} diff --git a/codespeed/templatetags/account.py b/codespeed/templatetags/account.py new file mode 100644 index 00000000..99a3bd9b --- /dev/null +++ b/codespeed/templatetags/account.py @@ -0,0 +1,27 @@ +from django import template + +from codespeed.github_auth import login_enabled +from codespeed.models import GitHubAccount + +register = template.Library() + + +@register.inclusion_tag('codespeed/account_widget.html', takes_context=True) +def account_widget(context): + """The sign-in button, or who is signed in, for the page header. + + Renders nothing for anonymous visitors unless GitHub sign-in is configured. + """ + request = context.get('request') + user = getattr(request, 'user', None) + account = None + if user is not None and user.is_authenticated: + account = GitHubAccount.objects.filter(user=user).first() + else: + user = None + return { + 'login_enabled': login_enabled(), + 'user': user, + 'account': account, + 'next': request.get_full_path() if request else '/', + } diff --git a/codespeed/templatetags/pr_runner.py b/codespeed/templatetags/pr_runner.py new file mode 100644 index 00000000..c721b6f6 --- /dev/null +++ b/codespeed/templatetags/pr_runner.py @@ -0,0 +1,16 @@ +from django import template +from django.utils.timesince import timesince + +from codespeed import pr_runner + +register = template.Library() + + +@register.simple_tag +def pr_runner_enabled(): + return pr_runner.enabled() + + +@register.filter +def ago(value): + return '%s ago' % timesince(value, depth=1) diff --git a/codespeed/urls.py b/codespeed/urls.py index 70bd0178..b382633a 100644 --- a/codespeed/urls.py +++ b/codespeed/urls.py @@ -1,6 +1,6 @@ -from django.urls import re_path +from django.urls import include, re_path -from codespeed import views +from codespeed import github_auth, views from codespeed.feeds import LatestEntries, LatestSignificantEntries urlpatterns = [ @@ -30,3 +30,17 @@ re_path(r'^result/add/json/$', views.add_json_results, name='add-json-results'), re_path(r'^result/add/$', views.add_result, name='add-result'), ] + +urlpatterns += [ + # Sign in with GitHub + re_path(r'^accounts/github/login/$', github_auth.github_login, + name='github-login'), + re_path(r'^accounts/github/callback/$', github_auth.github_callback, + name='github-callback'), + re_path(r'^accounts/logout/$', github_auth.sign_out, name='sign-out'), +] + +urlpatterns += [ + # Benchmarking pull requests on demand; 404s unless PR_RUNNER_ENABLED + re_path(r'^', include('codespeed.pr_runner.urls')), +] diff --git a/speed_python/settings.py b/speed_python/settings.py index 208b41d5..b7b43e65 100644 --- a/speed_python/settings.py +++ b/speed_python/settings.py @@ -129,4 +129,14 @@ def process_exception(self, request, exception): 'environment': 'ampere-one'}, ] +# Who may sign in with GitHub. The OAuth app's GITHUB_LOGIN_CLIENT_ID and +# GITHUB_LOGIN_CLIENT_SECRET belong in local_settings.py; until they are set +# there is no sign-in button. +GITHUB_LOGIN_ALLOWED_TEAMS = ['python/python-core', 'python/python-triage'] + +# The "PR Runner" tab. It lists no environments until a benchmark machine is +# registered under "PR runners" in the admin. PR_RUNNER_GITHUB_TOKEN, used to +# post the report comment, belongs in local_settings.py as well. +PR_RUNNER_ENABLED = True + from .local_settings import *