ci: run every host test on each push and pull request (aurora-wip)

.github/workflows/host-tests.yml

@@ -0,0 +1,92 @@
+name: Host tests
+
+# Every host-side test of the Aurora drivers: tools/asahi (with their mutation
+# and old-code controls) and the apple-dcp, apple-afkv2, asahi-g15 and asahi-m4
+# selftests. They extract production code from this tree and build it on the
+# host, so a driver change can break them while the kernel build stays green.
+
+on:
+  push:
+    branches: [integration/aurora, aurora-wip]
+  pull_request:
+    branches: [integration/aurora, aurora-wip]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: host-tests-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  RUST_VERSION: '1.96.0'
+  CLANG_VERSION: '18'
+  GCC_VERSION: '13'
+
+jobs:
+  host-tests:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    steps:
+      # Negative controls read older revisions with `git show`, so keep the
+      # whole history. Trees and blobs of old commits load on demand.
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          filter: tree:0
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.14'
+
+      - name: Cache the Rust toolchain
+        uses: actions/cache@v4
+        with:
+          path: ~/.rustup/toolchains/${{ env.RUST_VERSION }}-x86_64-unknown-linux-gnu
+          key: rustup-${{ runner.os }}-${{ env.RUST_VERSION }}
+
+      - name: Pin the host toolchains
+        run: |
+          rustup toolchain install "$RUST_VERSION" --profile minimal --no-self-update
+          echo "RUSTUP_TOOLCHAIN=$RUST_VERSION" >> "$GITHUB_ENV"
+          # Tests that would pick their own rustup toolchain use this one.
+          echo "HOSTRUSTC=rustc" >> "$GITHUB_ENV"
+          need=
+          for p in clang-$CLANG_VERSION gcc-$GCC_VERSION flex bison bc; do
+            command -v "$p" >/dev/null || need="$need $p"
+          done
+          command -v "ld.lld-$CLANG_VERSION" >/dev/null || need="$need lld-$CLANG_VERSION"
+          dpkg -s "libclang-rt-$CLANG_VERSION-dev" >/dev/null 2>&1 || need="$need libclang-rt-$CLANG_VERSION-dev"
+          if [ -n "$need" ]; then
+            sudo apt-get update -q
+            sudo apt-get install -y -q $need
+          fi
+          # The tests call clang, ld.lld, cc and gcc by name.
+          bin="$RUNNER_TEMP/toolchain"
+          mkdir -p "$bin"
+          ln -sf "$(command -v clang-$CLANG_VERSION)" "$bin/clang"
+          ln -sf "$(command -v ld.lld-$CLANG_VERSION)" "$bin/ld.lld"
+          ln -sf "$(command -v gcc-$GCC_VERSION)" "$bin/gcc"
+          ln -sf "$(command -v gcc-$GCC_VERSION)" "$bin/cc"
+          echo "$bin" >> "$GITHUB_PATH"
+
+      - name: Show the toolchains
+        run: |
+          python3 --version
+          rustc --version
+          clang --version | head -1
+          ld.lld --version
+          cc --version | head -1
+          git --version
+
+      - name: Run the host tests
+        run: python3 -I tools/asahi/run-host-tests.py --jobs "$(nproc)" --work "$RUNNER_TEMP/host-tests"
+
+      - name: Keep the logs of a failed run
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: host-test-logs
+          path: ${{ runner.temp }}/host-tests/**/log
+          retention-days: 14

tools/asahi/avd-av1-tail-tests.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # SPDX-License-Identifier: GPL-2.0-only OR MIT
-"""AV1 auxiliary alignment, probability inheritance and reservation controls."""
+"""AV1 auxiliary alignment, probability and segmentation map inheritance, and reservation controls."""
 import argparse
 import hashlib
 import json
@@ -44,28 +44,30 @@ def structure(source, name):
 #include <stddef.h>
 #include "controls.h"
 typedef uint8_t u8;
+typedef int8_t s8;
 typedef uint16_t u16;
 typedef uint32_t u32;
 typedef uint64_t dma_addr_t;
 #define ALIGN(x,a) (((x)+(a)-1)&~((__typeof__(x))(a)-1))
 #define ALIGN_DOWN(x,a) ((x)&~((__typeof__(x))(a)-1))
 #define WARN_ON(x) (x)
+#define DIV_ROUND_UP(n,d) (((n)+(d)-1)/(d))
 @ALIGNMENT@
 @CDF_HEADER@
 @CDF_DEFAULTS@
-struct vb2_buffer {struct {unsigned int length;} planes[1]; unsigned char *memory;};
+struct vb2_buffer {struct {unsigned int length;} planes[1]; unsigned char *memory; dma_addr_t dma;};
 struct vb2_v4l2_buffer {struct vb2_buffer vb2_buf;};
 struct avd_decoded_buffer {
  struct {struct vb2_v4l2_buffer vb;} base;
- struct {unsigned int color_size;} av1;
+ struct {unsigned int color_size; unsigned short width,height; size_t seg_size;} av1;
 };
 struct avd_buf {void *cpu; dma_addr_t addr;};
 struct avd_run {struct {struct vb2_v4l2_buffer *dst;} bufs; dma_addr_t y_out;};
 @RUN_STRUCT@
 @CTX_STRUCT@
 struct v4l2_ctrl {struct {void *p;} p_cur;};
 struct v4l2_pix_format_mplane {unsigned int width,height; struct {unsigned int sizeimage;} plane_fmt[1];};
-struct avd_ctx {void *priv; int ctrl_hdl;};
+struct avd_ctx {void *priv; int ctrl_hdl; struct {struct {struct {unsigned int width,height;} pix_mp;} fmt;} decoded_fmt;};
 static struct avd_decoded_buffer *reference;
 static struct v4l2_ctrl_av1_frame frame;
 static struct v4l2_ctrl_av1_sequence sequence;
@@ -74,6 +76,7 @@ def structure(source, name):
 static struct v4l2_ctrl controls[4];
 static unsigned int lookup_calls, preamble_calls;
 static void *vb2_plane_vaddr(struct vb2_buffer *b,unsigned int p){assert(p==0);return b->memory;}
+static dma_addr_t vb2_dma_contig_plane_dma_addr(struct vb2_buffer *b,unsigned int p){assert(p==0);return b->dma;}
 static struct avd_decoded_buffer *vb2_to_avd_decoded_buf(struct vb2_buffer *b){return (void *)b;}
 static struct avd_decoded_buffer *avd_get_ref_buf(struct avd_ctx *ctx,struct vb2_v4l2_buffer *dst,uint64_t timestamp){(void)ctx;(void)dst;assert(timestamp==1000);lookup_calls++;return reference;}
 static void avd_run_preamble(struct avd_ctx *ctx,struct avd_run *run){(void)ctx;(void)run;preamble_calls++;}
@@ -89,11 +92,14 @@ def structure(source, name):
 
 TESTS = r'''
 static size_t aligned_cdf(size_t length,size_t color){return (length&~255UL)-((color+255)&~255UL)-9216;}
+/* 3 bits per 4x4 block over 128x128 superblocks, 256-byte aligned. */
+static size_t seg_bytes(size_t w,size_t h){return ALIGN(ALIGN(w,128)*ALIGN(h,128)/128*3,256);}
+static void set_frame_ctx(struct avd_ctx *ctx){ctx->decoded_fmt.fmt.pix_mp.width=1920;ctx->decoded_fmt.fmt.pix_mp.height=1080;controls[0].p_cur.p=&sequence;controls[1].p_cur.p=&frame;controls[2].p_cur.p=&tile;controls[3].p_cur.p=&grain;}
 static void read_exact(const char *path,void *data,size_t n){FILE *f=fopen(path,"rb");assert(f);assert(fread(data,1,n,f)==n);assert(fgetc(f)==EOF);assert(fclose(f)==0);}
 static void inheritance(unsigned int residue,unsigned int mode,const char *neighbors,bool check_addresses){
  const size_t ref_len=6964992+residue,dst_len=7000064+(residue?((residue+79)%256):0);
  const size_t color=138240,ref_offset=aligned_cdf(ref_len,color),dst_offset=aligned_cdf(dst_len,color);
- struct avd_decoded_buffer src={0},dst={0};struct avd_av1_ctx av1={0};struct avd_ctx ctx={.priv=&av1};struct avd_av1_run run={0};
+ struct avd_decoded_buffer src={0},dst={0};struct avd_av1_ctx av1={0};struct avd_ctx ctx={.priv=&av1};struct avd_av1_run run={0};set_frame_ctx(&ctx);
  struct avd_av1_cdfs expected={0};unsigned char probabilities[sizeof(expected)];
  src.base.vb.vb2_buf.planes[0].length=ref_len;src.base.vb.vb2_buf.memory=malloc(ref_len);assert(src.base.vb.vb2_buf.memory);
  dst.base.vb.vb2_buf.planes[0].length=dst_len;dst.base.vb.vb2_buf.memory=malloc(dst_len);assert(dst.base.vb.vb2_buf.memory);
@@ -108,7 +114,7 @@ def structure(source, name):
  controls[0].p_cur.p=&sequence;controls[1].p_cur.p=&frame;controls[2].p_cur.p=&tile;controls[3].p_cur.p=&grain;
  run.base.bufs.dst=&dst.base.vb;run.base.y_out=0x120000000UL;av1.bufs.probs.cpu=probabilities;lookup_calls=0;preamble_calls=0;
  assert(avd_av1_run_preamble(&ctx,&run)==0);assert(preamble_calls==1);
- if(check_addresses){assert(run.addresses.probs_out==run.base.y_out+dst_offset);assert(run.addresses.color==run.base.y_out+dst_offset+9216);}
+ if(check_addresses){assert(run.addresses.probs_out==run.base.y_out+dst_offset);assert(run.addresses.color==run.base.y_out+dst_offset+9216);assert(run.addresses.seg==run.base.y_out+dst_offset-seg_bytes(1920,1080));}
  avd_av1_set_prob(&ctx,&run);
  assert(lookup_calls==(mode?0:1));assert(memcmp(probabilities,&expected,sizeof(expected))==0);
  assert(memcmp(dst.base.vb.vb2_buf.memory+dst_offset,&expected,sizeof(expected))==0);
@@ -120,18 +126,68 @@ def structure(source, name):
  struct v4l2_pix_format_mplane pix={.width=width,.height=height};struct avd_ctx ctx={0};pix.plane_fmt[0].sizeimage=compressed;
  avd_av1_adjust_decoded_fmt(&ctx,&pix);
  size_t min=pix.plane_fmt[0].sizeimage,color=avd_color_size(width,height),length=min+residue;
- assert(min%256==0);assert(min==ALIGN(compressed,256)+9216+ALIGN(color,256));
- size_t cdf=AVD_AV1_CDFS_OFFSET(length,color),co=AVD_AV1_COLOR_OFFSET(length,color);
+ size_t seg=seg_bytes(width,height);
+ assert(min%256==0);assert(min==ALIGN(compressed,256)+9216+ALIGN(color,256)+seg);
+ size_t cdf=AVD_AV1_CDFS_OFFSET(length,color),co=AVD_AV1_COLOR_OFFSET(length,color),sg=AVD_AV1_SEG_OFFSET(length,color,seg);
+ assert(sg%256==0);assert(sg>=ALIGN(compressed,256));assert(sg+seg==cdf);
  assert(cdf%256==0&&co%256==0);assert(cdf>=compressed);assert(cdf+sizeof(struct avd_av1_cdfs)<=co);assert(co+color<=length);
  assert(cdf==aligned_cdf(length,color));assert(co==cdf+9216);
 }
+/* The preamble refuses frames larger than the format and out-of-range reference indices. */
+static int preamble(unsigned int w,unsigned int h,unsigned char primary,signed char idx){
+ struct avd_decoded_buffer dst={0};struct avd_av1_ctx av1={0};struct avd_ctx ctx={.priv=&av1};struct avd_av1_run run={0};set_frame_ctx(&ctx);
+ dst.base.vb.vb2_buf.planes[0].length=7000064;memset(&frame,0,sizeof(frame));
+ frame.frame_width_minus_1=w-1;frame.frame_height_minus_1=h-1;frame.primary_ref_frame=primary;if(primary<7)frame.ref_frame_idx[primary]=idx;
+ run.base.bufs.dst=&dst.base.vb;run.base.y_out=0x120000000UL;return avd_av1_run_preamble(&ctx,&run);
+}
+static void guards(void){
+ assert(preamble(1920,1080,0,0)==0);assert(preamble(1280,720,0,0)==0);
+ assert(preamble(1921,1080,0,0)==-EINVAL);assert(preamble(1920,1081,0,0)==-EINVAL);
+ assert(preamble(1920,1080,7,0)==0);assert(preamble(1920,1080,8,0)==-EINVAL);assert(preamble(1920,1080,255,0)==-EINVAL);
+ assert(preamble(1920,1080,6,7)==0);assert(preamble(1920,1080,6,8)==-EINVAL);assert(preamble(1920,1080,0,-1)==-EINVAL);assert(preamble(1920,1080,3,-128)==-EINVAL);
+}
+/* Segmentation map kept with each frame: expect 0 zeroes it, 1 copies the reference's, 2 leaves the AVD's map. */
+static void segmap(unsigned char flags,unsigned char primary,unsigned int ref_w,unsigned int ref_h,bool missing,int expect){
+ const size_t len=7000064+79,ref_len=6964992+64+(ref_w*ref_h>1920*1080?65536:0),color=138240,seg=seg_bytes(1920,1080),ref_seg=seg_bytes(ref_w,ref_h);
+ const size_t off=aligned_cdf(len,color)-seg,roff=aligned_cdf(ref_len,color)-ref_seg;const dma_addr_t none=0x140000000UL;
+ struct avd_decoded_buffer src={0},dst={0};struct avd_av1_ctx av1={0};struct avd_ctx ctx={.priv=&av1};struct avd_av1_run run={0};set_frame_ctx(&ctx);
+ dst.base.vb.vb2_buf.planes[0].length=len;dst.base.vb.vb2_buf.memory=malloc(len);dst.base.vb.vb2_buf.dma=0x120000000UL;assert(dst.base.vb.vb2_buf.memory);
+ src.base.vb.vb2_buf.planes[0].length=ref_len;src.base.vb.vb2_buf.memory=malloc(ref_len);src.base.vb.vb2_buf.dma=0x130000000UL;assert(src.base.vb.vb2_buf.memory);
+ dst.av1.color_size=color;dst.av1.seg_size=seg;src.av1.color_size=color;src.av1.seg_size=ref_seg;src.av1.width=ref_w-1;src.av1.height=ref_h-1;
+ memset(dst.base.vb.vb2_buf.memory,0xa5,len);memset(src.base.vb.vb2_buf.memory,0x3c,ref_len);
+ for(size_t i=0;i<ref_seg;i++)src.base.vb.vb2_buf.memory[roff+i]=(unsigned char)(i*7+1);
+ reference=missing?&dst:&src;av1.bufs.no_prev_seg.addr=none;
+ memset(&frame,0,sizeof(frame));frame.frame_width_minus_1=1919;frame.frame_height_minus_1=1079;frame.frame_type=V4L2_AV1_INTER_FRAME;
+ frame.primary_ref_frame=primary;frame.ref_frame_idx[0]=0;frame.reference_frame_ts[0]=1000;frame.segmentation.flags=flags;
+ run.base.bufs.dst=&dst.base.vb;run.base.y_out=dst.base.vb.vb2_buf.dma;
+ assert(avd_av1_run_preamble(&ctx,&run)==0);assert(run.addresses.seg==run.base.y_out+off);
+ lookup_calls=0;avd_av1_set_seg(&ctx,&run);assert(lookup_calls==(primary!=7));
+ bool usable=primary!=7&&!missing&&(ref_w+7)/8==240&&(ref_h+7)/8==135;
+ assert(run.addresses.prev_seg==(usable?src.base.vb.vb2_buf.dma+roff:none));
+ for(size_t i=0;i<seg;i++){unsigned char b=dst.base.vb.vb2_buf.memory[off+i];assert(expect==0?b==0:expect==1?b==(unsigned char)(i*7+1):b==0xa5);}
+ for(size_t i=0;i<off;i++)assert(dst.base.vb.vb2_buf.memory[i]==0xa5);
+ for(size_t i=off+seg;i<len;i++)assert(dst.base.vb.vb2_buf.memory[i]==0xa5);
+ for(size_t i=0;i<ref_len;i++)assert(src.base.vb.vb2_buf.memory[i]==(i>=roff&&i<roff+ref_seg?(unsigned char)((i-roff)*7+1):0x3c));
+ free(src.base.vb.vb2_buf.memory);free(dst.base.vb.vb2_buf.memory);
+}
+static unsigned int segmaps(void){
+ const unsigned char on=V4L2_AV1_SEGMENTATION_FLAG_ENABLED,update=V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP;
+ segmap(0,0,1920,1080,false,0);segmap(update,0,1920,1080,false,0);
+ segmap(on,0,1920,1080,false,1);segmap(on,0,1916,1080,false,1);
+ segmap(on,0,1928,1080,false,0);segmap(on,0,1920,1088,false,0);segmap(on,0,1280,720,false,0);
+ segmap(on|update,0,1920,1080,false,2);segmap(on|update,0,1280,720,false,2);
+ segmap(on,7,1920,1080,false,0);segmap(on,0,1920,1080,true,0);
+ return 11;
+}
 int main(int argc,char **argv){
  assert(sizeof(struct avd_av1_cdfs)==9166);
+ assert(seg_bytes(1920,1080)==51968&&seg_bytes(1281,721)==25344);
  if(argc>1&&(strcmp(argv[1],"snapshot")==0||strcmp(argv[1],"snapshot-copy")==0)){assert(argc==3);inheritance(64,0,argv[2],strcmp(argv[1],"snapshot")==0);puts("snapshot inheritance PASS");return 0;}
  if(argc>1&&strcmp(argv[1],"copy")==0){inheritance(64,0,NULL,false);puts("probability inheritance PASS");return 0;}
  if(argc>1&&strcmp(argv[1],"seed")==0){inheritance(64,1,NULL,false);puts("default destination seed PASS");return 0;}
  if(argc>1&&strcmp(argv[1],"aligned")==0){for(unsigned int mode=0;mode<=4;mode++)inheritance(0,mode,NULL,true);reservation(1920,1080,6425600,0);puts("aligned compatibility PASS");return 0;}
  unsigned int cases=0;
+ guards();cases++;cases+=segmaps();
  for(unsigned int residue=0;residue<256;residue++){
   inheritance(residue,0,NULL,true);cases++;
   for(unsigned int mode=1;mode<=4;mode++){inheritance(residue,mode,NULL,true);cases++;}
@@ -148,21 +204,36 @@ def main():
     parser.add_argument('--out', type=Path, required=True)
     parser.add_argument('--neighbors', type=Path)
     parser.add_argument('--case', choices=['all', 'snapshot', 'snapshot-copy', 'copy', 'seed', 'aligned'], default='all')
-    parser.add_argument('--mutation', choices=['round-up', 'cpu-unaligned'])
+    parser.add_argument('--mutation', choices=['round-up', 'cpu-unaligned', 'seg-cpu-unaligned', 'seg-any-size', 'ref-index-upper', 'frame-width-unchecked'])
     args = parser.parse_args()
     args.out.mkdir(parents=True, exist_ok=True)
     paths = {name: args.source / DRIVER / name for name in ['avd-av1.c', 'avd-av1-entropymode.h', 'avd-av1-entropymode.c', 'avd.h']}
     text = {name: p.read_text() for name, p in paths.items()}
     src = text['avd-av1.c']
     macros = src[src.index('#define AVD_CDFS_SIZE'):src.index('struct avd_av1_run {')]
-    functions = {n: function(src, n) for n in ['avd_av1_set_prob', 'avd_color_size', 'avd_av1_run_preamble', 'avd_av1_adjust_decoded_fmt']}
-    if args.mutation == 'round-up':
-        if 'ALIGN_DOWN((dst), AVD_ALIGN)' not in macros:
-            parser.error('round-up mutation requires aligned tail macros')
-        macros = macros.replace('ALIGN_DOWN((dst), AVD_ALIGN)', 'ALIGN((dst), AVD_ALIGN)')
-    elif args.mutation == 'cpu-unaligned':
-        functions['avd_av1_set_prob'] = functions['avd_av1_set_prob'].replace('AVD_AV1_CDFS_OFFSET(', 'UNALIGNED_CDFS_OFFSET(')
-        macros += '\n#define UNALIGNED_CDFS_OFFSET(dst,cl) ((dst)-ALIGN(cl,AVD_ALIGN)-ALIGN(AVD_CDFS_SIZE,AVD_ALIGN))\n'
+    functions = {n: function(src, n) for n in ['avd_av1_set_prob', 'avd_av1_set_seg', 'avd_av1_seg_size', 'avd_color_size', 'avd_av1_run_preamble', 'avd_av1_adjust_decoded_fmt']}
+    unaligned = ('\n#define UNALIGNED_CDFS_OFFSET(dst,cl) ((dst)-ALIGN(cl,AVD_ALIGN)-ALIGN(AVD_CDFS_SIZE,AVD_ALIGN))\n'
+                 '#define UNALIGNED_SEG_OFFSET(dst,cl,sl) (UNALIGNED_CDFS_OFFSET(dst,cl)-(sl))\n')
+    # Each mutation names the extracted text it rewrites; a stale one is refused, never run as a no-op.
+    mutations = {
+        'round-up': (None, 'ALIGN_DOWN((dst), AVD_ALIGN)', 'ALIGN((dst), AVD_ALIGN)'),
+        'cpu-unaligned': ('avd_av1_set_prob', 'AVD_AV1_CDFS_OFFSET(', 'UNALIGNED_CDFS_OFFSET('),
+        'seg-cpu-unaligned': ('avd_av1_set_seg', 'AVD_AV1_SEG_OFFSET(dst->', 'UNALIGNED_SEG_OFFSET(dst->'),
+        'seg-any-size': ('avd_av1_set_seg', 'DIV_ROUND_UP(ref->av1.width + 1, 8)', 'DIV_ROUND_UP(frame->frame_width_minus_1 + 1, 8)'),
+        'ref-index-upper': ('avd_av1_run_preamble', 'idx < 0 || idx >= V4L2_AV1_TOTAL_REFS_PER_FRAME', 'idx < 0'),
+        'frame-width-unchecked': ('avd_av1_run_preamble', 'run->frame->frame_width_minus_1 + 1 > ctx->decoded_fmt.fmt.pix_mp.width ||', '0 ||'),
+    }
+    if args.mutation:
+        target, old, new = mutations[args.mutation]
+        body = macros if target is None else functions[target]
+        if old not in body:
+            parser.error(args.mutation + ' mutation no longer matches the driver source')
+        body = body.replace(old, new)
+        if target is None:
+            macros = body
+        else:
+            functions[target] = body
+            macros += unaligned
     header = text['avd-av1-entropymode.h'].replace('#include <linux/types.h>', '')
     defaults = re.sub(r'^#include .*$', '', text['avd-av1-entropymode.c'], flags=re.M)
     alignment = re.search(r'^#define AVD_ALIGN\s+\d+', text['avd.h'], re.M).group()