From 6b02ee4247e69a30fd659c293a5fe86ae22998da Mon Sep 17 00:00:00 2001 From: Val Alexander Date: Wed, 8 Jul 2026 15:49:47 -0500 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20hosted=20deployment=20stack=20?= =?UTF-8?q?=E2=80=94=20production=20compose,=20/healthz,=20image=20publish?= =?UTF-8?q?,=20ops=20runbook?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The operational half of the HOSTED.md beta gates (the six code gates are shipped). Single-VM architecture matching what the code wants: single-writer SQLite and a container worker backend that drives the host Docker daemon. - deploy/hosted/: production compose (adapter + Caddy TLS ingress + Litestream SQLite replication to Cloudflare R2), Caddyfile, litestream.yml, .env.example, and an idempotent provision.sh for Ubuntu 24.04 (Docker, firewall 22/80/443, layout, docker-gid + uid-10001 permissions). - /healthz: unauthenticated liveness/readiness route probing the store — compose healthcheck and uptime monitors point at it; leaks no tenant data. - Dockerfile: add curl for the container healthcheck. - .github/workflows/publish.yml: build/push ghcr.io/opencoven/coven-github (latest + SHA on main, semver on tags). - docs/hosted-deploy.md: first-deploy runbook, monitoring signals, backup and restore drill, upgrade and incident basics. Cross-linked from HOSTED.md. Validated: docker compose config, bash -n provision.sh, full CI gates. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Val Alexander --- .github/workflows/publish.yml | 45 +++++++ Dockerfile | 3 +- HOSTED.md | 5 + crates/server/src/main.rs | 4 +- crates/webhook/src/routes.rs | 36 ++++++ .../coven_github_adapter.cpython-314.pyc | Bin 0 -> 73869 bytes .../test_coven_github_adapter.cpython-314.pyc | Bin 0 -> 4963 bytes deploy/hosted/.env.example | 20 +++ deploy/hosted/Caddyfile | 13 ++ deploy/hosted/compose.yaml | 77 ++++++++++++ deploy/hosted/litestream.yml | 14 +++ deploy/hosted/provision.sh | 55 ++++++++ docs/hosted-deploy.md | 117 ++++++++++++++++++ 13 files changed, 387 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/publish.yml create mode 100644 deploy/coven-github/__pycache__/coven_github_adapter.cpython-314.pyc create mode 100644 deploy/coven-github/__pycache__/test_coven_github_adapter.cpython-314.pyc create mode 100644 deploy/hosted/.env.example create mode 100644 deploy/hosted/Caddyfile create mode 100644 deploy/hosted/compose.yaml create mode 100644 deploy/hosted/litestream.yml create mode 100755 deploy/hosted/provision.sh create mode 100644 docs/hosted-deploy.md diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..3ba22b5 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,45 @@ +# Build and publish the coven-github container image to GHCR. +# Pushes to main publish `latest` + the commit SHA; version tags publish the +# semver tag. deploy/hosted/compose.yaml pins deployments to these images. +name: publish + +on: + push: + branches: [main] + tags: ["v*"] + +permissions: + contents: read + packages: write + +jobs: + image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: docker/setup-buildx-action@v3 + + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/opencoven/coven-github + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=sha,format=long + type=semver,pattern={{version}} + + - uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Dockerfile b/Dockerfile index 6a30433..79b544e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,8 +23,9 @@ RUN cargo build --release --locked -p coven-github FROM debian:bookworm-slim AS runtime # ca-certificates: TLS to api.github.com. git: clone target repos. +# curl: container healthchecks against /healthz (deploy/hosted/compose.yaml). RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates git \ + && apt-get install -y --no-install-recommends ca-certificates git curl \ && rm -rf /var/lib/apt/lists/* # Run as an unprivileged user. Secrets are mounted read-only at runtime, never diff --git a/HOSTED.md b/HOSTED.md index bdbae45..282b21a 100644 --- a/HOSTED.md +++ b/HOSTED.md @@ -88,6 +88,11 @@ Hosted beta should wait for: 5. Usage metering by installation, repo, familiar, and task. 6. Cave oversight dashboard for task history and human intervention. +All six code gates are shipped. The operational side — provisioning, TLS +ingress, continuous store backup, monitoring, restore drills, upgrades — is +the [hosted deployment runbook](docs/hosted-deploy.md) with its stack in +[`deploy/hosted/`](deploy/hosted/). + ## Landing Page Copy Headline: diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index d164aa3..dc31cba 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -12,7 +12,8 @@ use tracing_subscriber::EnvFilter; use coven_github_config::Config; use coven_github_webhook::routes::{ - audit, handle_webhook, list_memory, list_tasks, revoke_memory, routing, usage, AppState, + audit, handle_webhook, healthz, list_memory, list_tasks, revoke_memory, routing, usage, + AppState, }; use coven_github_worker as worker; @@ -172,6 +173,7 @@ async fn main() -> Result<()> { }; let app = Router::new() + .route("/healthz", get(healthz)) .route("/webhook", post(handle_webhook)) .route("/api/github/tasks", get(list_tasks)) .route("/api/github/memory", get(list_memory)) diff --git a/crates/webhook/src/routes.rs b/crates/webhook/src/routes.rs index 6fba2e9..28788d7 100644 --- a/crates/webhook/src/routes.rs +++ b/crates/webhook/src/routes.rs @@ -43,6 +43,23 @@ fn familiar_names(config: &Config) -> std::collections::HashMap .collect() } +/// GET /healthz — liveness/readiness for ingress and uptime checks. +/// Unauthenticated by design, so it must never expose tenant data: the body +/// is only `ok` plus whether the durable store answers a trivial read. +pub async fn healthz(State(state): State) -> impl IntoResponse { + match state.store.delivery_routing("__healthz__").await { + Ok(_) => (StatusCode::OK, Json(json!({"ok": true}))).into_response(), + Err(e) => { + error!("healthz store probe failed: {e:#}"); + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({"ok": false, "store": "unavailable"})), + ) + .into_response() + } + } +} + /// GET /api/github/tasks — task state for CovenCave polling, behind the /// tenant boundary (issue #3). `token` mode fails closed; a tenant token sees /// only its own installation (optionally narrowed to repositories); the @@ -2996,6 +3013,25 @@ mod metering_route_tests { } } +#[cfg(test)] +mod healthz_tests { + use super::tests::app_state; + use super::*; + use axum::extract::State; + + #[tokio::test] + async fn healthz_answers_ok_and_leaks_nothing() { + let state = app_state(); + let response = healthz(State(state)).await.into_response(); + assert_eq!(response.status(), StatusCode::OK); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("body"); + let json: serde_json::Value = serde_json::from_slice(&bytes).expect("json"); + assert_eq!(json, serde_json::json!({"ok": true})); + } +} + #[cfg(test)] mod billing_route_tests { //! Marketplace plan intake: entitlement recording, plan-derived limits, diff --git a/deploy/coven-github/__pycache__/coven_github_adapter.cpython-314.pyc b/deploy/coven-github/__pycache__/coven_github_adapter.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..675ce3c60db3ae86868a2ed8e0c05632c478fc65 GIT binary patch literal 73869 zcmd?Sdstg(dM7BME0U0exQeSVV1v!Y7qIcYu)zj{F+O173$_X)VN}MT90^~jR5zVu zHn=(&G;m99+Dnd+yhot`xPOwYtU)0uX4Dj8>z*&O*mNl}=snn~?UvOD{%d#dK? zO!YiF`+L8mb0on5SM~P(vCkfx?;M@;-7oL$d*AolWzVr_a1H&lzE|cx&}jaee#nOG*j*pdv%4W;V0UB0$nK_yiQUZ+GrMO; zvT@Jqwsct{mM&|=T1olpx@}!KksS7{@3wb2A`bR!=yrDHMsmAc5m%+=YEG5rYF>+0 zs$GA*T;0^YCHdIDGxbh>iWIW+`u-;QP3qpiC-p2pamK5KoTOH)w2>XS2XL>$eGK<{+;8B% z6Zdi4cX1_L>F%sZ1AYhbyAk(SaBsr>ChpB#S&=5Pn=41S=jxs+jWQ;Cua;aby;^p) ze4pN_IaR39MD|_X*B_L}E_LTB@UQYfa!pifdWtogY)yrR^Kez%Hm;hh;j~;Wx4lW* ztdF#EJFaQDx@JvtR%Aa{k5UhCI}x^VyAU4a8W0}h8WA4mnh+l0Cb(wS6Gyq-_EdLkPRL!w9>%BM8rMM-ldL z#}M|SjN`0~v)oJg-N)VH+S%`O+zI^d=T0I#&vhW=xK4xv+$n^E+-ZatxGscVt{dSH zcLw1_Zj$R^=`SJYUiNgEJBu$@xITo#+&P3-xqgJ#xbp~o9Eb2_ZUEs3H;C{$cLCuj z=SAq}h7bn0iwMWKO9*dpml2M0R}codVT7-6R}tRit|6S@dThZ2`s%Y1X8%Ye5DDH2B+L}fOic#rwF!OyNMs!M?5@GV{!X5s;t|!v1%72Z z5RN2F)BMC#C@`5QJbBLB+2=dmJ=k^Ogl}N5eX!Hl(ap`cMyGBECL3=ABjeN88^aM4 zk+Ap!6T#a7J{Slm43UxW&2YkUXNtcW4vmcBm3br*3ET=r!inOHO#9FEcAvcD>u(?I znkml6HZ(O69KF}{N_c8AQJSnmdw;*LpX>HgMS41u$;z8D)A7OEs8Hk0z`dqW;8r3h zEp2y4LX)sd6*zgWqtkbyyKlxKB_Eyg2NKS-Z)XvqcgA!h6lrX2YE2X*^BL+q(RJ=z zk8hy!B-c4Onp~`GshAJ~5NSS1Aehw*vzHX~Cl`+Lg5Ck$uD`q*PC-2Vl-s&XUbrE> zSDy*`ycqUU&2YAyz^j)LSI97&BflNY8g|Ifms(}eC*P|z9?dW#6ex~S<2On`bx=E< zRj#Sl1T}|s<(in0+Y9cJO)X3BM?a#!Vx|gO6Y=TZ>M-h{*^bNO zN$m0mLK9Q>n$w3{`r(tNqHn}M5{d-)rqDfBEgx#Z*CK-|{Dpl8W;JUT+k>6=cRpyk z-?Wgwz>Ah`F-u+4QYTt=&Yt|xYL7b#=j_k3bcUieXWsnDubbx#pBgnb$K0K_bPK1& z;$5P(LC`n+-KPePt?*Ng)=-2LuiKyN@bp=j`sJTn+emE^SV)V)`)MJTT$uG8Dy zIpjOlebLu@?p*&seHQOQe4c8T(1piG&;x;Rc;rT46mOJ^dVtNUKf%K+MwyXTnviCg z-m=T9I-7<=`Y?rbp*wz_mO{de`4hS`9rW`+M4E&VKk*}B4BhehgM1>p^X_OM6bWLc>BpyT z0gUlP_=U9yJbVo$v5kz5PEAineDs~)jaNMsqZ7fb=6ROEQ2JM9*T)VF>!aR>z3;fb zRVopTLJ)$S0e)ee+dmm+S1p^gVvu$&!R%? zl0itdCIuGccU)FFRmkbme*vrL6J=@Ok>KY=-{85P&b|Q#zZp=Nu_mE0fz!`&DZJ~8 zOx+Ajh8ZZYH?h&>i%=OJM1+RNMAAqTz!JC{8kzJ@hXXttJRm5*=E^1~?r3henAd2Hypu42|%CNuV-w z;BGJ+@lD-KWMQJ}QImTKn?D#1O^n?0F(9WuN$(7|&Q``u z^qWF?B2;VM)%MjJcq%lZo1D7CQLdmPo>Y2v3p}Y$X!*|!(~+^pws*CdEt44d;N*=ytRy`n zu=I?&(N9c`_!GIOf)jzhsmLinFuz2T=@NQ2p(|1OgmD~eVH-}~y`IenM*Kd? zg};FJVfs6TfWVy|C@d#;zGgwUcq5k6w3^ejynQXN=+VKi9bE8=c{M-NY3$ATfd67n z)6YM)mTf?1yO7nd6}IPS777mxt=Tp?|6An^sRkpzAL5Q$`=i`N-I&GQFJynActXrR!ecr&lNEqTnJ%7Npgr zY-1{_iJXpFQO*9gUV6P#66uMBml}=FS<6ACG^LhE35=P_Nwt=#bd?}|72?-A$hV+2JWZ;sa9>P8free&COU z*3wt+Kw^eXfFy2GGBdGBV*mvp25*Mx2Ty<$^kIZg5=LWw0v7p>ff%piFZ>@6%xdC#^KbTir6;B@i0TU#{7Zv^zF6VJ0u2G3fK4=k@+=G)#H zSj_+Cr4{pzkF7Zm4%|QR;Mo0R3w1GT^)qYrn$6Dg8UMgm`BR<7R`W9}5~Gx#e`?X- z+h);MZI#a{`e)%9H1)lTs*W1Xd&hTmeB7E&l;ba|Y$=gWH0+8VPSiCT|59ulpcg1%G2?diQj7?btNQiZtx?a@q10hO0EBHBq|}Ol+cHRHzpJ2{+^Qqdv5P(O&ABd+V{5Z zuQ&2_R1c*xQVC`x@jMO*K8EJyHu$#sD_}`+)%{4!cr|Y+O66EW#9#r$$Jpm|J-X zh8TfLldm}|q~#1=&9x&9%2d{9I3tKyQ-fS6?wmPQw`Ls(ScNE0TLP%-`U^Nz%jIy{ z@+4PV2A~a+%TK;zCn7DQFHlz27cVQlB-Ps4~P?DT+eQHNAW+ zQ~QiNLk{Q(hCMvwok4Jx3EjxVjf8<4z%)(hBKIJ^Ix{o~8P-T7p#x7EEW<3q&v2MH z(r^F>8yJxfOZa^VUCI-~1I|#}kkB5gwh(OyEdW+P;RkqvBKH`dkV9(VY zuy{ep*)@AA?#Q1XT{!WL@kPU8cxm9<_m*paR4CYAnmzro+5TIR2Q#nFENox8yy9q! zIgUpi$Df!)N0(^sj$3m6i%y$u|JdSuBmYs!*Gd*nFI!e}565y(L~~C(IRM66wDkTL zLss@#?az!FQ_fe$=7xTK>Up-t;Cc?k?2cJ0qt?nr6A;YOzEx|}4+?)|d2&>2J1_3x zR;`?%=cF}Hjh8QF&DXFsFENXp&YQ^x=p|acSs&7KW}pUL06Y#Lft-t~s5YHqRTJB$ zsPd43vjF{Aw%H4!OnYTEEj<38{2Pyl%FY2mq|B{Pl0gvH-`AB$P~ayU?%@! zE5=xCnP$G5f2nda?LbLE=7V19Jz#)`cYX|s%m0qt5&{`}D5^;k~*YEJ#q#ki~F zQRBnLuQ$hBbx~K{QjzG|Gk5aC?e%Z8JUaBXLkpvC9ADyzE7^$@_%G(v|D3iA3O?0j zA-A|a_rdu6@vpuTvsXs#0BXzGg1u6-ADYuKNZb}HJsvGR{>1onKrB5k+Bw0@{rtIs zl71E@QRaKjf|EMUd$q?;G#h?WW5M%Jnyn`_**`g?$MYX)bQJ2W9S+?e*^hKsbnjaX zc+Ox*6w>=G)`=;hnoQ7!wCS`Er&WV;DK^Cm$k396y`d}+;R>Wf_Q{J=L~VrMWf+vC zyA&!1;puNGh2BpC}#`&Tw*ylXhjKVRhiYk7$o6>Ny19shky}(=*$><|vPw z?Q!$TnE6c9eCFwK(R?Lt&W)QbpJW@H#@Q}R?wkU~3KxmCs@cvDtxh(HE{JxoVD`qX z&e`6LlLQ7lJHlYyoQk(8lLm69T(<#^Q^ zz606R8xq#E4Pbs8@0gS!VejZX)qbIO&?kLn!cD%5KBpwkR*8>c+r<1o#kW7fU$_nd zF{xl0%A?kDP!i9qm1}1ATf5%*(u#RMHh12*@=jB_{5hRRrB8U_YGH%SRz?x+OkSU}bUip?wbaT! zvs&qDDN-_cwG@n7S1ZRL0}8cNs^us(3mJ!MyvBV5UW^G_`fB74qZ8P^6>-R+sc;Z< z{XHIp3N{m!7f@-a*RdJKk5YUB2-wGZM6ryLngahl`amn13E`#y8u&k;UqGYC)`Uj_ z2)HEKij`jbl4<`CUx_{s{}=)0Zcg!6ug;!YD|9a$c>D01hZnyjmhO+09uZ29Jl+;7 zX^)n)KdBK*PRB~l3MFTs?iCCBX3xcQiyxIfEL|8Db9csacMG|@m#)vAS+nQP?|jts zuxT-0bZw8h8U$Cv(l*iFID0ytU-Ib8!!rwC7W129`K>~J>+;@M-l1sTp~uI?yq;Lz zc_HuoQ+~ELZYz%2%A&Tig=?a%A!gev*!C`G#jFRS)&q}0p}z!NYb{wE6tGpMua|&N z`jVpM#Fo@f*18l}pbedLE-iDn@m=kVLE16Y5;W-T_I7gJodcwQ0_w(^)Hbr1m~;Fzf1< z&EB!P5ws#b5v6YV3#P>#tHt%J)}4ZW=f=7zNc?wLV^STdrdZ7zQEQ!;*3-#eLN*#rKK|I0-OsQ9 z2%Pi@&7webqsV2|AQFfk^;AkPs;1I&fbw8kwH(Pd)k;;M8pKO_)fh&3oH!F8zFHos z7}$zYjiE4sTy`4%P)It?lA;G>%*a_e8^lC*r~qP1r{2M!Ap?T^d+4%+Apo7RNWwTW zN*mk~z$EaS$WZ_-t>#4b*!0ANv@b!Un*gXB4L~63$2J717m>0Vm#-4igSyYDdzvb0~lX zMMHke$MF5H@fZFk0&Iq;!rypa$)m#$4=;R4%xj9}wFr4F%iCsqKQb50zZy3?=1byc z=lqUGyB_XZ7#CeT;%4{4kyz=jXz8w{4zY9(aXBS#SG-xVq!-n!d6(W}m z4Y*Vqo18jz$M!3i7DGiW!s%ovs!+w~!7@$HJq_sv=&L1VSOZZQ5;>7`eJbK3O%_rE z3D4Bm9Y0carM>a;NaM`z#)H15U5!`kUwZYmy5s3@oADA@#4t87aw9x=s@^P3+fh_i zQj_^Mo)TF+c5edV(UB1DVF)8e#}k(B(|zZ-&Xer}ouptF3EYkF|C}OQCP`HTyAcp` zn&tWsM}H$YNis!fW%83rCA$=Pl6?zH6{|Gh=RZa+pW-k4B?L6j3m%osmn^tu&!|xl z&+=R6%%nKi_4S;XvpVXm7M-jch8X_*zU}dcW@4 z3S>7a;5?M2wj|i(1}+0Zt&peKQ75F#E}6BFCx{;;M;U=vwR$Lv=&U6rg-^+_w?eQX z{m_YnC{faQ@9AL^Fe^C$B@Wt-ouC^x0=#cxf35M%tGwpZ`CQT^2?r zJcb7%Y@PGlAMJd&Qz&l~T}{~P!5+`@HZf;EM9sNHkILrD7WRIvY|gM&+q`V}K_pgt zJdBdaZhuI6;SDaPvS3e2-(axA-AnF(p9Tyt>f;P4ePZSfxtMpuS&W$T0@VYm<_;U?fwkgEUhB=Dp|#W13El zF{Mqh$x1M)woU=oSP9BX@hSvL$dvZhBm?xV`r4~YTOAkJ>Ht_~whHLyX>2r4W4k#> zfch`d9jBm~9;O|YKmf9wa?qxeA}^A5IQjF9&PCZUxI2#Y)aH~7Z? zEdr>r8W=6Y;4c5~=>7jr2`!`0+X|8zewhD1=|#5kf)x`c+{6?wl=V~~5jDx`_#t2N z0iEzaLqsbrCjk!t#jS_`;PUq`KXHhy{jpZBfd7LOia8gft==W`Z;j5^ef3t%UKO=h zePFM~yV+Ah&PD72{iP%Sque||a7X@wd-v}xl!%U+*{%;wj=0$sGrOapL+Vz{)p5OD zQtp@!JPJMxE?ULHhFIZVkQd8YEBbwZT~Ilfy|%Mu`O^FP*v`&pJ3HsIALTsES-7#{ z-2UhLp5FVjV_3>o``q+b4@okyC2Ff2^W44Vc2 z5m7)$GMH#eLO9YgXVEjY@*FY%PJZS88`753$ThQ8#O?VHUb+8@;NB(L8^DU2?7#Nq zzjWlqt*$pl7k0eexX`%BiG{?W2>Ln+>*f#s+_e6b_Y1K+dT0&V-#OvLLiTg#+EU z{^G3i_;I#Pn4we*(x_9*BS*=@zDW$b93!oDRZ{CXhbjbqK}m9coO6>BopOw{5`lTB zKQ|cxwG?a}fIQ9RvIdQuYm<~Nm9ntvBx^b@zgf!_aD`k^Q&zJ*WCC+q+;&l;>D2na zPH{Gwdv29HleNLzGjEcrWXn|9X_Qj7>PVK#=|M>)XPt~MZ$OO*LTsx%gS;lLOpcST z3GdZ$bx)L^NqFP>YH_30eZTip+xR?Fm*edwCPg6W`zR4 z+%{Ej)kFVZX^}!lNiD*f*_B)~=eJlhFPQzCtQlpVZR~%nv~=#`1>^U*)=Wcc-lwcb zux1+N_wsC4S#M2poGr$G4?1i_HGhm zuM%V5CNcIYF(W1m?-6Te<9HTk=bF9FP^L2l6zJhQ2CBygwa?L_kua&!W5E$9N z5jG!6%@xFIquTgk1OW(+xYC7LcUX?K#jHCb#{s04a{@1hoD?0Bpp3IcCAo+OiiDIm7ilII^{TIwQcoq% zW`oMi=wC;5q{&E#>Hv;D&(`$sZGbZ8<t#NXk7mV-#M)*8N`0OSlth9j}+$6@J z+%MdPO=4V7VtC;+K!#1~JmQcXcVjQ7BtEJu?FIdrL1|x1^-lfe)V5H%2{jCuQ!>ys|>#`0YUcn}UZ9QSH$2T=O zagSf2QdSYnY_`vI05+U#HwubMBtYj$Zk{m7zn17%c}5K19k@_{(w;9VZKDRed{_2t&(>=?LH@5k^&SH zE6D}fq=J(1Y{@D^>I7vzVn{Y^b4#K$y&|X(S+&9_m-RrR3YExwwV}zm8=@r2H@wPd zdLis7(1!53oDH8J-S>V7P@zDD{xEdzr8&e@OyEf1)JwcSd!gYED~}>C(LE z$y+0nu=Dq`Eh7GZqw> z#<0beU{f07&-?>OYcdE6?%ebdW}Cqf_f-K6^4k#9+oatRr>u>3s{L$tZ+AP#HjyRm zz>m>bzKK$hkPpUo3}!K1eA0id!OZ=c z1sKBD#vBI)$HA5QgR@;9nRDk4W3QQZuJhlx@%5l`bIcb*Zam-q#kQ~W<}W|G_VC(b zk65t#XBur`kM=nWfxb({RCf}WWd_`>P7OiLE=Hi8&Z#TW!B-9@iOAf_K z{6dNU@z{@F`Qa-<_a*VzW$BG@%};gdqKe*rH1lv~u~97A9V_Y)ih7p4kFN?XJ)|*= zJ@S2Lg~Gn)l}?*+w&Qt?28n#Drkc33=#llI6^=ffH8JN-!MSrOYt9gNx!ERtP;}MB zTup+jY3anAY0Xi*Q1o``o283h(cQ2#7IU;dbF{AIl`OOfRWFHo?Sid+%~7#%d-2OL z$AM>#1IbTE#Jr<|?I?6L9~``YaQ<4%TJy|W6NkxT&;6c{^2-;#9M3O{7gsFWf4`a#o80k4Z6z0=VnboDPjtqmU!{@@9LI1zt{b4_wu;d za6Hz~AvAP6*(26<;?t7pd)9ZY%csP;!?C({p|1Tg|Jgdtkk9{mAKFI39NvV&7LN+a1r}_Rh&A!}qe^ z&0el!`MxC7z4Z83#F}<|T0H%oo^SOm8^xLfv6`bo&C$nuQJ|ZwJ95hx{EJ6cD|QPl zeZszTs}<*-JIp!uPj!}T%kx5wGk?zdkttUw*t=@ltKRYPqKZeaK74iYm{_zoR@5dG zwJl$tvu!yX9b#UmVC!VVA?!F5a~yu=IGj0Dsa6W5O>#3^;`x>D z4xyHMwyIOTp-8e=^qtagl`dTos}7>K7EZt2^JdSIQ7miT`d4PUibd0RtlzROoe@0; zp66N6d4+17SBio9ROdwJZIe51{^F{se3Pl1gajAG-0gG5xHVUB?_RO)PIkp%G4F_A zJA&8qmshNn-*r4T3NQ7?>drr_JD-elRLnajz#=ypWu@-u<3Oyg_gP(UGQvqQuS2jw zCop&Z(u%cWWd|bsIQYZhQ?q#NyfAPnw&U`%9hZ}l4~cn)1>0dc@;hH%zWw;M*p9Q$ zcAQ-oLz#m@=`Lw>7J9avgH?;I-#PfLgG)nVb!)8puuy&Yu`8bMS*-m|{kQ6uPKn$0 z#kL(3wjEsNVK{)!wW)P(ZZ>tU6`fnGbndlPQ}t$3kXHMbVvhaK9Q%{?m%CK1KMWV* zPLHtdVjRoqSln5R#lCRk?VE4jT&fc*_R+72SVdE`qG{=_SkV^GuUIULRqcsZ?OEQA z=@hFvE>sKL8`n3!bYbZ2D{o$bjYat$ zX>=vX)h*T?i`8`qbsbN(i8Y-VF;q5V#N^5rSgEo(sB9Ul?8jEue2d_25Uq`ZzHy@p z32mg&-zT)mHw&#ku!7p{cph?cS#@B~tUur*An8TAyVJ%{a* z@w613$buge7ACz4Y}n?P&vwieedTPtyjsY1NA)GpUzqLLSSLNAvpOxabm}S#^2V9E zx(b+N%3<9lr@;n3h_pMXFyE5ec1u%0L8aGUa$0W|G0I8xlUlO2rcL$a7CY{eX1|XA zDKfy8I6N4HnQ{qq?*0K@F$EH7XIuv(V&;xuy*`OeGH>!R4`6&3D)YzGi=9Z0Ud+jT zaQXh_uMR_L1c>0Z`PUY=#R_*t3wM1`*m&Ops~T)MhrilJegH&R4d4F2S{u(Re!cgn z22D}ZPoZbzlzKe9e;^K$dYnWc*iub-fdSbCVWH9=DHS3ZtYpa|0e=;XrR12>aaU7S zNNL>5>80M}pvJ?!SPxgDrXG|sr15G~su*exOJqYnHIrL;jMF!0pzuV!!!C8_B~XJ> zX?Yscub-(>b)J`QggaxBbbybZVj9D+EyISHhRqz<+=8=>go#-Oi*ajY6h4UpcO;9o zrxfc?Dfrh25_a%;etK{-nrxQz$SB&Q3DdQC#X2 zlM}*G^XH+@P9{yb*EzrkO43MDaPT4sB5Otc>t%*Q{p=D&4vG5ec+v9i`^ zS?h8zR`$}fvX`K15qA{7)$(@R@3bwR`p(&Jon77`)*O1=@&1l~)%0EyOuk~JSE8j? z#L}y)j;pg>YgR|hTJg+Uv2gdhc}ti6QH5x26ZCBxwE#%3hr0cfc`V~}Qx!}Af<)|+ z2SJ`_Nv$+>Qx0YK@T_)FODdU3O^U3LK4r@a@Tlyrh79SJtb@8VDUyj6M7kq~p-y?? zg^Ytnxjr}03TQw{R=L^O?YA^Thb>jwpay&V$<~C-gNiL{MmihPwdEp8nn?Xn)rY3JN9v%Gp z-=Ni!X5=^Mi7*j^0ttJj7f=aS^;4X8C+4|NRha#M!p=j~2goEjO1+2t z8A4=$$>0xMpT3c>D^Q04EhxH=+>xYk=ctDqN^cNugpJWZqS7*@3Pg^C;a77rfq_&B z$_U~oxi0+6$o>QTg>NE&s^*S8%QZjJ#daKdw&TcL?ED7Q53%%2Y| zT!bru(?V;P;OG|2-5-M;ydms>OMo``;CKMDw22hVUnz+KyVyA04rFY}37Oa(0yK-Y?S9bGd=S+VW0=?vM3agc+V^ z9ms_tgRkk=BLf+*5J2CGCX_*>{{W+9paLXaUOkL`We{wT-73I7g&dq1)YP9iWpf1% zPt;>53ZBhS&P*xx3^_qLC25Ha^-@K_><+&2cSG6U6i6RtfY#(SN??&Q<0+c~NK+a@ za#1WOWEfRzthh=+S_?~S{@iI56*Huj96u-O7Y#3jr;^hGF);4E(EnF_OJGH@{ zrZ{Bb4CyL7@{ICYQmsS!j5z2E1LcSJ#`Ls!&yW|k{nXm0ri_+sAD4%;Y+O@yOz+?I zC2@HS4Hr;5f#4PNy`iQGIU|qJhMY`occ6g4>cJJ9)h5T#us)}B6~-^pd#XL5tPrR+ zzM!wrGwXU8ZST@(nt|Sd&J`pM)V*IC^+6pR&ga87`G#w7-8c>tTgT4Jy&fiB5BiTX2O8KZVqQZ6{(3wA6Dnk!f(sN-Z!yy_ zo{V~#EjyFN^M8V$z~1 zw=xfblK)xs0NlJMi3+I`{sm>vb{AYo7ACdFw5|H(}?l>IP z*UX+<7=uoFysUQ4JYNLe^LXv4IkS*k6*X^*Z*PrlzbI_K__+ATWj`zvP7R3%FVe?t zQK+Za?26U&3N^h;cfa?m@BZrJb7FHZvZ#!jt6=ZH(70mWD_3)h$T_u7Ja#pHok^6W|Nnk|2R zaH0MizQw?btpNh{?3_7ldN@(=(6@7z-7B6$4|hJvPHxrY&bKUNf8#i;1K)7Jb#QTT zss7tOv9xtLBIX{6<-Qcned$S&nA=T;h&7Mu9@as+D!Llj&-pvwXk2WIx^_W6PsW61 zgYoY^HEEn~&@`yWU)pj7*O8d(WYl%?Nyk(7AD>&fa4mKr`0PSZyl_)=O?+Ux1tJGA zKVv4&e`3k&GHRY0k9He%e`?So%%ET_n4^HOnonpWAM7kb>DtV?*$eqgS}f2#>#wc9 zLV*kz`9Km2!5x%jd8=1*^#+`+SF?%+CIEl|m7`a)MU=i0p076#lKIu>9rzl9yL@7M z{NM%H9ZV1ZEmWQV5J5tZEqt7u0D;9cKS_=m`4E<8AO9-|gHJyC@Bbb7Si;lS;aiA& zo?vrz8cuimyK#yG*}=2eEHcDxJiVBtV2XlovHU5aY}_i_R9guy(FQpzRrtGzyARMH zd;{?T608MeH4TAo+~&alHHZ7bm+yZ$dBY|H(;=+nBb~i{~HUD=9NJ*_YUBbX+=5+N6UjAn9S9<5+fVGJ6DjS8$ zN{^1Q98)4qZ<>vUOwjE>9J+3l62XuX5px$WZWgl9QHzDV{W15^sQc(+^v+4q-XWMf&^z1OqNZZ0gJ{{wml+Ri8GaWJfXXO9 zx?qTv9xwS;mLJR0z^hGnf0^zG$`29q=a^V&4lz@^6W2899ECh(12%3PlDos~%Y=FD_(~ z80e|_rmnvNfwGU45$0bYY z);~Hca09CygQ7d>)d|C6&e=G5m9crB6_7qQn+t${@dcgY(o>GWu#eCnDjt3aOB>M@zvL^y!zVJKA3F&4H6}w*9AzB z8nf&G^wt9K$r)gvf&3n0Oh~b7fTA17-S*vto^3_x@rH~oC9e(z%GgPu#m5TdNdie- zC?6nUEtnsB%b#>9tqx$;iaaoFEULS2h0UY6a6XchK@E!K2R|?$`b4KGR`HFa$ngIq zO3YkdTMQwSzox_F)YMN&&NBuHtCwx|xzZ*{R?6jpJS>|+eXZJ-lN!`dvF0Go6ge^+ zIi%rki~zgR(kDR)_6w9Wm|ADa+J0y+vk_R2hjeN=4k@R`rS(Y4o+?XChMxi}oc;xQ zrY7w?UZ@d+e6rUIX zJS5r=3+BUX?rn<)K5*~4pMzE8a4&3srzYmu@yxMf%~d9ppNy5CiI$&v+A5Zx7hRlS z zK?1*MhH@$3_dj9w{xa4sPV39P1$XG`Yq(Q;>R&3LR;dTJUa21J`UO|9B3eybGl~Ec z`70}OlU1BbufB?bN8z*LB(374(g0vV8uU_M$u)9oQnex*e%P=o>sf9$Yzh^6K4cg= zuL>^=YHK^?DIrx#nxIgtAjGxDfT6ls&(Y(Q6F!t)o1RDZ!nblLDVB zHpDHIdt-fEKjsH4U5wYXZ4iO`*TYeZYQLn$G&$48wJ+=!MGVf_UXZ5(M_-tyBJSqw zpF2;|wPDvLz3!058oq z@7pD;yW^dfn4|8QqmE(So>(~-E$4)Ri(>gD(REp{T_&`9=eu^HZ7|k$CE9jH7#_8H{X?dcWw{ z#@qruvzCJK?f5~B#Q6Sb=4s7J|3K{c;Ird{!o^|1_wvfcmr+>Uc|PX67d$q)}7av;}c7i$VM)5f|1ii3^|!#v)Xwu?YV^BQDbR z>{QQ&Ns$?xIys?&I)H?foyR0z!i`gih)oF=b>c$VS(lL*5Hof-rE8P26f%QTb-b@e zumIr!H8Q73p0gi9hfUIH$i!&TST8}-QwdGNRL_E zL6kjHy+wM)UXi?e5D;>vN(DPo)giGY{Ukz+b_nco;D#?e4LLb~Z)WFKnP`lt}g;0yM^T?2j z5qpaA_<#Ul4UlmPJOe_~!4L|yd&7Y}E(Mh+S@7RS#IF%9A>UIHE(yMqAG2)SET0S5QUJ}bMi>@nz?F!+N$U8n^ z|3Ga2<>>y)!j+fB{Ua;p>mL@>Byov{;gX(MJ{QercqsqnOk7ecf$GSkyASU!W-|nF zKGtwC+Hg_0bWLpVt(adXzlFsQix=v|+-hd`umV{z;hL9X<)@?Nr=J?d@^hlAU$FHv zsO@~vb-zm}s1Y#?ReC7{E0!dosyt-sis(kB{X61hza`Sw;^mM|7eqk5!>0^2wX(}sXI=M4F+Rh5{2L0)pn z@*O(W>M0Z#6)ARXy31e>H$pgZ(@ z6y_rhAFH=R!6jQTjO#^)ibfFd2UfLoZBi>mar6u1uP|Ag_NC&mDzIlAGppW8CH@zv zg`!QgX)W^P(c$)}&Zqn@Fsqfk0<0&Pk47-e$(;f_RBcUg4lT73U_LCk@7=Ad`BltK<^6ZE2~<5h1%a>8olvm3#*b#0uClO^qB^ zz9UW53)0leak=ejwR=&Ub||UpQmJrw*Isf4>Hi^HUk0w8t=sh@yHj;MR7V!hzi7Sg zW%;T1t}-TyfNyfplCKV-(mHNFs+sVDN!}6S?8TAz@0g1n{TLGi7~J!6!EuGqq_sZo#l< z_pTSuQ9V5CYXB}}rYw_DVeu3mWhOsaUPk>C!CA7JX|0!bJ(UQ_>TigEb7YyPl)jnm z$pWO`8zRbby@VBK4}-U1c9H9uD^};U@5Ef83)vY?Gy9VTq<@$7M@S{Yb7wp-DRX5w z!%rE@WgAHjzWTp5Oneo?u z=YJ0vo&S9V2^UmvWm%#;itD7b8)GD_6LfYhlf8y#+?j=?eM;Dg?@gX`2Vc802a-du zzR8)<$XHM|*ny%fn>EQT&>09Fs6*wpr`r{JC~mRq{2)zjJiCaYg^q zU*+C}j}NQ!18W5n33SBDd!pq%PdmkOPIL_jwgCvNT@zX|BeMVI%vWXvSB+?@UCLWA zH8254d(7Pxb$2~2{EL!5EfEIC#O@oSdt9^!1#>WN%Aq9lb)u>Co#QK}y)4CnnEP1N zeeCh=AJ6=7=4qXHd_Z&$iuMbF`9eG=kL;Tl%wkRzPJz! zY>U6W^Ua-$KCz@#IB<4F-v=Gd`LRc@JbY!*Ef((jE>z+6JhSd$1<4!hFN~31&u69` zKd?OB|KmeHJoIE-JbZ5D{9tVNg=f1jNUas^Cj|2eIYFZc&18+S-uS7;Xf{Hn+Tj-5 zM`G@iQTNFwr~cKM_s%?hP3*WTy03|LpJ4VO-Mo9nRI+gU?U^@cmKw#%wr7=k4PAOELH9sQdI&hv>d2+Aj&_OCQSvDd0%!lKn!P zcSS#>jIL07KrCq!4qaZ+Um;y^X?RQEfqg~axTY^mdbimHjr$dS^P0X?R$V_VmK_m} zy}Y6yQSw^6DwgaMTKiY@=b=t6IQFcV_Tb1ap`vNU+QeqOH&%W%T7Ffyc3mtV6ddt$qLpY85t(G@XXYD)+%4c@-|=4GMoh**yGRibziN=pB1 zg+fvNclX6?&ChJjLd&UGOK-HLH`dbktflW`lEV#)uB(FWDmr!U_HTS?p;>4=x>9nK z_1)Q+dob$8W)CzL1EPIQFpohcuJqVfz9em&=m8$&OEH9N03b-}4J)jbM(wB@_46RK z*R(QIz_vv?VY-ELy`hJ!Pgg!93s?}XCa(czXE@C}<6Ab~WT(G@Cp`<^S<>HeR=lxh zyumR_DZAa2mK4mij3D1rP6t42muZH!b9OSc;vDeKt(>#zhj;5gA^*fPLP#`KM3_Y&|oc{qlzJdyDLC^|ZeH{kd5 za~_;-J3i5yIk_-S^I z74*6dVboVTKm{lEur3j)Use$$ikQl%Z$mo%8AYM)Oc<{7LHI=jB3E)vo(VyUin3Bw zP8jO`IFbiW3lW?DDT&Ky- z{~3<+u8x<~E*V!#nuI-Ft0i4Gy%)TdeUMb;t`BE0KA1%ZpHeAy%t`I1lj!QWU z&n#-LN>l1Qv=lhiqYAu=G&94Tz>N5+PWv~BpN;scM-prjKmBkMb4vLMR-fBtt4|r+ zK;EJBMVidt2x}4?hJcgT$Wn$~oHb3|QjyGYwzRj33xTHexQa{+pbICqDF>5V)4+~p zm_~tJUPZMhRY!mu8~(^nRgr|t@im(DJii}GIqR%2vpI*HTgaBShiQ|g0kFn&0<7kG z%{aOso7p(#!-aE!*TxlkbCR}=iqsIcjUZidytqBp`awm?9m=8l!s&q{WIt~2ZItdQ zrw8>#q%0ww1Uh2@HxU{M@{Qok@{_Bsk*zxT4`qjPCpkm;rv*?=d2Vc!eI(8}@GPAK zF2O?NOlN`J~~a3T$K{DD*a$~hV+`BCMZJ`HXUcmA}~e)t_K*kKS7NqP{OQ!cqoWzcoLgQ zWIpT+{BZFIu8P!Z0D{2^QXXmcjzn%VV@90!w+`}I>1;9ZOJwmP86bT~k>eCxqB1_B zr+-Jmzo$T{D*xB?^B*Bfn5G~ZF5)6cNSCCzosTX@t#H&ILVv)=I)wiZNN59)2>0P( zR`cAY$#*Zbzj5e3w|FF0y+2yLe|cK0K1!!el?j!tvC1RS$|H}xVr9o_ZU>ITag{GR zVxIj`&;I4xqUYGE>)32h+)=VH94p@)E#JLdD3-UaI@)HtK8E$4*|i2IgMOUOYCiX= zL1!?+0MJ_av8@CLrp9gMEX;##4?!M1w=IvcK9# z##PSzPpfjR#@Umf*1(i-etgAq;IU5h99hvHg^?9%omYh8SWT{&$sIMhVV?QSR0^jH z*(GZQ6$`K8$iRX~uCSma{|1*|ErBh^@Kl3*3 zaHaDk@*0!xo7Pv^di=umHG5Ouo%B>+C?T~H&!SrEeAQZmRM@)K$kQh&Hl>*>DSof6 zS=!#fb_3AhroB~=>H)4$Z3{7FPXahnG*i<)9hrj9W?2L|F*OzPfE|Ig5{P*SF`Gd- z%)|%5aD)hQK+5Nlhz2=@WC{$vMB)x05hP0wN#VH3a|&4#w<5m*`vWc32F7SfT+Bq^ zHV$wE#}gR6DX}^@e~x_*kBkK-?ln;QNlz%illuX1T2uJVd#+DS!)(eg*J5gn6_fPH zj;#quuHm-{j+dlTeq}mH6_bD;6p9&h6mFXNjqWkeh+I1lN=Ei%3yLnGCM9by5g55m z_MxC^MtDz3R?PT5_-UJfgZ5-WX#>&GK z*D|Zw4id~1 z;0u?bwM=nl&Yy$FdDh^uAP+6HJ-g*D3^O@28Ndbnhh?Uf#e|tMlXMe>#)RH9Fm#qO zMgzLl*U(Z(TN&BM%zg-=^Z%Iw5a$}c9o0$bUzrL{)@QSwfJC-3O%wK8Bk(#d&)P5! zLxgg-q^8WjLkZ0o4K|}$gIU|cWZb3@ZW7s6s31pVmtuGMpCONPKvrQQ0%tWJnVVwf z)~Fdy^h9$%*xc+<9fjwz;$}{loLOar*Z&>rQeV`K5-VjXPl>xvibWlB@Ee`; zz;+*|?iDdxZPZp9cfuo;8ln&5rZ zsez|mawUCAbXLcl^@6j0k&iicM;*IG$KE;phgMkA=RWAY-@8yAb2moajZ6OJ)_;6c zbRSxQL)?<8w|Bj{OW3|&EIAM>IVzOEgL2H>5p{Qn?o)I253O$4)8fK{G^;AV%s!lW!ryaeN}SSleBJVFQ=iq zm&{%bjCGoM(_-ld{0lSSn1pt0n^1;h9&+%WTg)IU86;G2Z|X;?;DtW6dRUw|pVfGAI5RglLiS9d^|(ymdm8GHgU68h1p z&^_p(l3RKb;Ne_NK=I%x+Z4tr9#9{q`Nh~e2{aV5xa8s!Tqd#KxUq=*e?x|o1RQ=3 zDQHW_@!-(whu-i%dgW`cEar=a+ePaR2UYq=lACmp8+YN1BicIjhlb8Dcte!=fT? zf)pNdc$qsvX(Y&CS7vFPDI+)XUYe#npDDdppp-G?uQPSDJkBL1{K+mg$5{P}YLpx+ zB@_#p(^{=?Nf2`}=VsU;FiURahQ>J?^);NqnPnM2o}8@z72#Y;p-LVJx#ox~xpb?N z>mbL-aG$r0Alq=Y9LG!Z$HnFK*UE2F_hBlUU2+*RA5*OvTHx~OGx(VLWTdp4T(ZoO z+?Zgc?vkTOXN+pMl^rwP2nwI6V$2WXYy!nudU1G5Vv&a(7T zA8u5>C{F)U^AwKM%F|1O;F%PVFiX<=XsDX2K+MYYn5mu^Zk9_JY8@KKoJp=a+`lX7^ggtU;-t=$V*tdQ1x1k?_FG==gfv&(t7HkTmpLP2N*+5i&1(5!Ez3_J-vTJ zcD&h<@y3BSbs2A*X+5Rz-)L|Bx}Hk^wv)bjotfi~G`m^X-};<} z56Nw0k}1r-^qIs~TLagq)^qD~!tcx(m!#HG{fYi)O7D-cQ8vqRR56Hhnx#CdNe72} z;NWm?DA)UfRB1Q^@P)aO-1q;a6(Lt94D#lBU8$B0=7jQ^G@<@|3;UcJ}sEkF#D6hGwlx-(~Q<_ugnZ_rkidDEiW;{{viyy0u+ ztV%)&x$hy%qTR1!>4ZcPJ0?yZ93Ooe3A)426~ixIbB^({ncI z+5IGKlzPbLYB=nPOx=XAawGyNS;BsV8Y`r`f^%CGhln#47N70wG7pM4DUO{XDcNc=SBf)v5+^Kg(6Y#hD^_51r`lBPAJuNPE?|AdCY z9-Jf_VOe66NNNfsmB5+eFP*=c6k@01Ig|4SLK`17Bg_u03{3>k>-@XOng3S^u=PkZ zAMGe={+94`eI5TNw0JL2zy!La>n5E}2o4Lo{_TT`PP*X;kn<6#Q5E2@DJ+rim#W@R(eqY#sX8I>c#|n4XeH<8M>MTw3`T zCBK*hzW(;XE=gB_e2VfPBS;itmxGpgd_e}HYpGFdZ|DM`X_CigC5 zXTRdSNqL%%(Xt+c)-^UYn0^Da+5#cIzD#mbP0pB^b2&-r;eVx{Oy}!6)C(rri6=oB z1-7O28*f1anSK~=r>5GN;4WGCx|Pj$P$ot{G~t31)J<5LlDb2}Ml%JeMsY@Mn9V;) zg`Y{_pb)~OXe3r665I{pSVl7cT`=7rB0C0Shl z{1DTPU9*x`;=K^ALlkGS{^s4U+?_Xxrs9RJ6`ZQ*$itz2^66J=MK~ex?Y1}D7SD*K z&9TxJp|oXrk66_DSi4$u5GQ28n{xh|+w)B~p39>8LehWrujs3A4#Z;7s;ln1`<5ch zXI7h!i@RQ0b-e@~4ZTBhiMrrl8e|Sp_pUi9uCs1w|EhBj$pNor;cyD)w#62q?%-SXhTTJ z{^Ojdq;! z&vkxUrpc*TxVhB3TG6(0hzp+qRy^AuMnLVQ$2)TpW=q4j>p%8%U9vXMJyVbv&9Rl78_Ozn&&L>!flI3 zR}1&dS>yJSh1OMj6$Z2R|L*O}quaX9{O~}K0EmZ`;J$(TCV~{Hg^Nf^lqgdhB|+3; zSr)aBmYB#A{U~Wpz;db1G48oaFSJ_V?X~_W+8IvYpNpqAniZyZ64k-|v3Q@2j6JdTnIdB-*J9346Rt zs_UiSRS`>dyt;0dk5#Xx-#!8lFC!52GXJz``tgXRn6SjFqn4UGa4Ys~%vKVu?v2`d z(@I*Kh9N^*nBE$AIO%x(IR#F}YNnnwqi0=k9ul&|2&T3n-mqG3=!`UU&imztEgv*& ziR9LZJ4dEBqb7204St1BOS!8*H5l`d8iCk(M(#Wx={z5I<;-l4xhlf#;R~<#;2m9# z>CFJL!=OkGF8D{uOEni<`sG)lHS!DLBD1h8^i;HR&AdBS*(DctPj9)EUmJdAt|L~r zPRj3@9=x4XoN(pdDr%i8nD2~kJ}4DEHoXllUEnLFSSlL8b#Y7F1urV_VOYj4a@ck8 zeW#_wF})ocSMUfkK;vm$%-I_42t=I$j4BLJ_J@&rME0JDcu&L&D&hsz;k~hfCXC5~ zPVL8FtYGzhvndxHKdfcDr=BqPpo!>REa4#Q;HJZs;Gah z5Z5IUzD%8I@uJ$7x80lM3`Ohip63ktmEq=CUMo$N!ca}j;hWC*jJXi6Yn)vdtLvEG z60IA!YP->aIkNwv{Z?u9tB>OH5Z20Dmg?~Ed}h?rt4e~&nTc?b!Wtdph?u(nB?RfH z9g4@6UG(gMPfCL~CBjjU)c%aa*#QlLoYdA9n+$-ZJZK;#v&^752{9hbXg+m3XisWc z`kK!Zfgi>@*@*;HAETcbg=L#OIR?M)Pn|n+b`0m3H;6dMgbeT4SUQ@xP)h^`1%5EI zCW!?L!5Q?t3&R7meR9)=NYjR&Hr+TO_3x9K_KO7v#A8^A$EF5g1vOI=vz3J_BDN~g zRJ9bDir5LPwT4-yfkl|mY8&Xa8MUU$9!qD}0oWZmsUNh9F~QXZGwUkx4>dfFC&CE? z4IG1H#^?pLqL3tOsFoD8Q!T6T0GjYN$Dcv7{goma^Tk~i-Z)z*x!Ys5_POIRTc>F1 zd`OkZzt_|stDUH;WtVdGsK=XO5^$=|1D}85f#x-doI1mgESD}p^c(>rBfGiUQ7lI*lt9Dp{ zFqtV1|AC&e$kdOM=ul?z$2vOUfn_o=Ur-m2v;00Y`dTwI$S14u}k`9B`Rcx0BPaNx+pRO-%7(O#$~Z_0X3l zex7=Gbn9{O3)Q3kbJWAOsOAlyBL%;x2Xpjm9`FWo{cvdpan*4Fr)jpuEsb0jjiB`HsYP20 z)$|yxOY2QC;|E7;fnM$QKeqT?YObl#`bFwLVNV;&X5Dq9P|aCyKhjO1Nd=as2xxrh zhxH0cZnbqzf$~Vr;S#-8Nv$(#X#ox5Q1ckY1_8Urzh{zp=viPFO66CopWiNp#SAm!k%8f0{mHXuCnmdET8>VP`@!6X6IFsOYymgt%p??tTPC31Nf&|@n<2p1 z$6ypG*l;bF=tdU2sh$`=9pot<#V6Aw;qS522JI{^%#l)@?_jyY^9GOI$DWb}W)=9U zl!`HQ$q5m_a7mLG21?k)Pt&GNT&|!?y{l#!tW4#mK6^03phvSBCD*P>Q3T44Fb*^; zQRdVrm_0bKVG}&CWkWx!0^Vw5o!mVASgx< zjKA~E;J!}6NSEDDrW~dwhr|RP@t_3FBPxK7iM~?Ew7048*y+PFaTMbT`qiT&pvYKN zl(WfkigJnq;_@o)aB1=+c*TBpJUE6FkWw&fTIH9Lev#qW8Y!l4UDyfKid1HPsLJNqc$d>>ThR z*Gvt3%;(3gju|r!TN`H!=6dD!!ASeyjY6q?4_wWX`Il<%mGcSog-MZ_Cws3ihj+&E z+M}z<`@rfwQPXhT%ugSR0V&}qLm4O%i`U7;8zaRVuWu3eKK`p0q~a0DaYVEn`Pf%I1;?0rb&DWn6A3Gikj7r5x8~cxKc^~Ar{~%}n(eIbt*b}wwMl;w8J}BJq)5#m1 zKY3n!>~PdRLI@S?Jsq(;Vl2ijj+rvi<%8W>XjPbx6*tehqs47==jTtw)^5A;RCMjG z%SXk1`^g5aS@zUMJhkDovxBccFT)vNUAI)%Gk^a2$ynb`xo>ZzZ?Dw1U-BGaj>Bdy ze8V4JGg}{neX^$=_Q`v=tow${Zu2MbAZ7vI)WT11kAa}y9W|GsVky{8D=Po1UWq>l zZkZFF|BO)2wiw`<@8q}26xC~~utfx)xu7_7?AjAop9t@J^|V~pCYH6$8RgQokV~?nc~%UV1n)d@AN^kHc|c2RLU{opIhLR<5EaS>X#YU#nukDET_jM7h~_Z8q|; zW>bBvrJpST`K+8#eu;}IIFOylI1In<(2B&YQa)rGrbiX*2UM53pQfE9GGT9}olev| zAY#s3N;~+qa}ea>W;O1^V;&x5J`6Bv$(We5UH6%wpPB}9H(}a#l*!3TX^QtC2yXnm zn`jGL&-!2p1M%sJv7VK!t+njpndh*?YKx$7ABAP+67Wa#1G;9*6kEDziGl}lq)ud0 zNH!BsPAVw7DP3n_3l)xne@^Re)j`$@GY7#k23BpjEo!N~1vYgMas$y>9tub{U${ML1B9*n z4w>5Dg-O1BhWnS+ubGyzLA)pj4UhWRYf{ul%IRt~1#2iEDmq~~26F|XN)!UCogT1#c_lq{!6plHfY_SQ z`=?m{6}Q31W6t|zj9d4Z4h(G>c;Djc>S~IB=72f`sYQV{%f1B zZWgOnOC@XW=48|Ao2{!FOqXyoW5XU$}FCSm4UNVwqU|SnlHI3 zWLKT&f_*rw;Vd56QV6%ocU(oGP1gpm4u%g(C9SYv3sqmMzgi#ekcw77louMlcHruP z@Rz01wg-O|CdKtzq+&nbIHTaKXTI_RY~VtvuY0YYcFJw@A>mZrkH zGGWf8zaFPg3SHcp6q_ECLO(|Oq|lF%J}LBLq$Y(0h8TJiTpRwFBwAmj3|nA+VsaVw zW}P$o4d;xT!B~0f3Eg}&Oqe0#HY{a?9DmL@kp=V_jQ~tBh5=}$k$9=QE$}TtDz=nv z+yH0l8)%%fV;(!&5%+d%6r6aNkl0NA?fBDr<5HYs-% zZCQg?wqM>JE|R?P%>DU)m4r@SJ9YJx*wiPLt&dwu0NPaZ>X+h{{CD@lG48v2=eTP% zS8KxAH;bCy-5V+Fj$2BP`E#==S7Ms^g6&vp6swarwFSsB%X4LFYEQK4p zD;J4Tse{>MEGvT*in7Cv+wwZ5U;= z!^d#G;gkh?7P^Bz*eOB8VyEQvkP&cBLCNTYl^;tFb8iBxvw&t9175DKr%Sb@q^H}I z!ZX2JOAO-}z@^BM{tbUd>h6qE!#D^3q(rMeUbo6L%qLlVBqQ9Vu>R(5e+NUOaaiKi^H4R}0 z68~<>5Uzcr>Dx`SW6{c0Y_(-t1>K$yV517KZlS z$}Nl*t(rT2Gq(%QK!C3`2GijzqjiwiQQg#n40(O{JfjardUR zaLI!|Op@g1t%eONGJmwf)^CGVIex1jZT$WkW&9(hz+|nN9Aj#>^qrF4E!|cEMr=kV z$azU-fD<_Y&In~NlSV=SxO{1gG?!)drL`6hqXGZwL$D$$`ShiSrfKC8;u#|D$N2QPspUXnni(`0xN<< z6;w%pC4EL1fu{Yrv5}Q4N0=V0g5)rA0>tZN!kvDLVQUO2+oYHBp)ESmuqeux$0sJh z9|pv-k}I;;r1wm=5p4S`jtoo41RQjB96Jr|Ur_1>hAojS5Y%84STZ~n=x7SE``j^D zm@^cLk#oW+p&Ln{Z=7~FncK+tC^QPv?km!S#Z^^s(yR)}u$QtbNKc|>MUDp*Pz|Pe z&A3D$_fZ@LJWWxQ9-llZG+|i*?wW_1?LSfQ230B5FAP+Qow;Y}0pl!vm#+RT1#cmk zq~W3=YJjP&tf`H(rcxl$ntA|P<8RUkjKI6k)L%{&wCm+W;@o9|fdmkKTx%+yNOLfKj_ zTESY9E!EJS3?G&(t3EYw*41|@i1W;E(#$^Ko5_5o$XAAqfU1hvh?KFdx|?apw0%mb zy7B6r>F=^tg*QpIl_<+1u6sPYSI!=YWDi^){pCh+*S=qDld=y?ZMCGtpP9(YEunr5CA~F(t>y7^&uS8oEY&?$#Tio+XzwChnB>j zMjB%a)7eVXHd1ZWs#_2U1>H1gL;IqfHGr=HZgdALtwony(#*4$_9@1XZJD<@b^A;I z4cf-7)&VeR>BL~NJNN0QPHpG<&ovM%Sa zAho6pqNOhW0;x6MNos1pw!V-WjW!>cu`_yEBhFH@D^RNUzCf8C6kDJ?pOkZa=_)}k z?J%j9;lb~6!w&Z{@7BB^EjQv*BwLzVTXHpCE)8Z$rCyynB`ENf!Ob~Vxsvt6|t;Y1vgLHg!=XjZfM3y zTqyml-yto_+ zwl(`OhM;tmG{sb{WqqWYbNuK8#uOpz0p;#&5=-76%x!FBEvF*7C&!Ajv2i04J545U zY#JGc(l8qke&4PX?a$EYPm_i1Xu63levuCrAJFF~2u$k?@c8^A3W#3vOz=9&tU%}8 z$5leL{8cSDM-+%W#_wOiFi+&CkhP%V!`!ndQHUR@9t_waV3=kK%_(K}ES@+Ci`py< zqn3mg0?tP?9g^n*91M~l<$UlI9tjlOMT<}VB`$D2NOPJY+x?>5AAS^i(GD+J!G#JX zN0kgWDfp}FnR;aL?FP{SnHuhTi>_?FyfyTKsm+TkqFFTW{fDnsYoG(51;&U&(@Rb)rYp?ZO?Rm9t_QX3+zWL-^XQIVDNff-9 zW1+fh%~zXWZ4u#D1q$9K>r2~T+#Yp9A4PJLmyP6|naY{dqPr$)s=aM?PF;u>_^$C+ zd3awbSS=THiv`{Do@r8AXRjrd;J#kl!>{=0Vt(ZOV z&Y?FC%{NNT1J^J7&9gsxRvbPoJu)J>j(n~tt7KPq#MM1N_`~h*Zx^>bF0DH(xkloy zwX&->;_98h@WW@{e^%UnMCv;#xsJtMt+H!v#I<(b@`%Q0zHkW&;XC7Rjz`vxiGdOE$nl#{2wFQP`NyQfC!+j`-`!CvD&}s9LIX-|v-;UQ z$=wP6R4;h9xr#`Y!Ir@{?F;Glq55V4!}F#qPn+GE6}21h&j`ZU0^7xwfY-rHUaW)n`0vXR3fz zRm)c?0_+QCl29623yf}IR5`cUA&D-HPGC(~O6a4u-)JLCt>04FdM%a4&e!C1YWdV} z&`kuA$P2}^ju9yl%ppo3z}8TLIb??a!DpL&Td-6oadkDY(&xH>k_wxAL{uwr`?Qj$ z9?{+4_GaI9LIR#Vdxp4WzM~AX5&ESmv-1>6aQ$RMx9{9(6zu z7by9WI2oX-VEViBKJ^$Z_L)tIS<_GdaMlb-&U6;EfbKCjZSrdD1GT>D zH%(r(Ek}X84K|1V8?3sEOtzjJFJQy5&bV0L{_`B`^vZkKn}+zUUov}IUG&@Z;p__1 zz|m_{r(TWI%BYPU_5JEM+BD_o>0xc^(tATdzfIj}(~SS8Hr2}eIoecXpr9`LZK|UTOe6Qpsu=_~`XO;WK%K@NaB&-@15QsPEaH`i!i}3jMYoxhOUUnq;)OUdytT7-*St~CyiYoM4DP;kq z=C|w{E7cfF+YMvay3AXv^xm4*DjK6(&0YOQIjoCEEJo>{f@sYK8%uA5R;wu&A(DF7 zxU11CMalsb@Xrhr{4%4fRyVDF8S2zBEN!De{W9flct{!(&`*TJW0^NK>%9qF3a!l6jlj=5?$eFe8-)+84-i$%>+*N)!{&(PbR zMG$P&YXG07ftoyCw6*$3T0TBD=@bxB12d!S`tHR z+y7t9DqKVjm$awwHQe31xS@7Mx^^yYsGUkf5qI%{%Cf3Fw3Yr1wUUFFv&z|#85I6^ z8jb%z0l5ni{wD>b^CsLy5cK%K!}Ti+Xq@Mo@k{X%!ZhjcQIy(LhD+WKPaj$z{g5JK zY!97*&&%ZDRdK1#%JJ{$ae1?En9%fbQl);xv^WMQC<>e(i#1EJGFdr-E+4*$^Q+7& zl#K90f=<=p2`M$eCjjH@2kVra^&2U@nAa%}Hx9HZ@84|fWYOEew{K2=^qJW~QJRSh zUZQc`$T-5OnWJ)slLvOiWOE#T=0^QcQhOZ}?*T1qfmp8s`8$D~A@kTpM_{&=uqmJb zrd^_Y-=W~Q6i~Y}Kj^|7T`^AVqm(z*G?AThPor2Es@zuPTj9CBfYJ*TbkPufA3M{O z0iM4S&V!=mAh9xS(6EFHHrWC!vC_~J*S>W1OW>iEwaR5ZVp-4J7Rk~Zw^YcM`iKS0 zwRei%ED~1@NDZ6ihCO1#o*NC4Wf=BdWIPCdC;Sq_L$v5?g$Fn}cdeMab}leAl;p*U z?mF4+kGTDBPr}ndOTXma7&UFWZH8;OCHr!AsGGU4X%=0ifV@R(ls4Jj8F6?1plW{n z`+mv2C2E3UL1C!siettBzK)_wBo;R@q`ccJ+Gp~wlwB?ho1^Y}@V>LO9@Kp1D4sY9 znd6%d(bXZ=b*RP(1(K^$cD0DEmhd>?l#vbOZ!^I2y=OMK%Rli_eLMg3`H&GF!jl<` zxxGGea`!|{z45$? zaFv`_E9TV#7)}{yGY77WTpoc5L~iR%%Stem^6Ne}XHR!a=KN4))Lf#yb5qnZz-;Ce zZc&oY!@i)>qp+R-GcM8rqf~Va(BDI#%3!0Iro9O)X6<~IuJO$2sarhE7h+1ARF6BN zmVxYrG|meaA#h%d3ZC^gIB#L%K(JMzgP5&W4LXP{9_cf0J7zG98BDDxP>BZIYq&q>D!YXxZ)!;L&41a7TtIQ0rQ-w@gOT` z&7|NWt(jlKbF#{=z3Hl-Hr+L5S#yE87a9(?!L8HuCW1ONU9()Zgv+FYR=Hq}Sg>ZU zU-GV%y*&|ckL2x}9=PrF#`8!0{=7w_Pk8(K*%WWI{nLldg5FSTSw2ui_Ah`|qX>U-Zu_;A&Mc$hFt*7`tVOXs0 zo(`&#K14v$o3*!U7lAY-VBP1&R35s|H8#zn0itm3$+>x&&PBkO{wqxbNMrY zw6sb~K;88rITDo=!r_rW2D6?M2GeNvStjxonu5!MMeLl6*H< zf_4mLvK5(RXcGym>X3I*L9ot(!`t!mV+o5IO(4;GFsHF4ZLlk8hhREVMPn8`IGAlw zz{Wcb7Y2KqFr7Gc=2$~!BAcNbkDNU7Br+DpPT+JRJS}`3X%kv;M#yWra(2M*H!xyC zp(QTSJ1F=D&5L*OkOb?uvWq`36@Toml-&&xcf-_H0@f=EpNSQ&o)aWz*VG^^D4BKe zW8q54QghqNU(A`_KXVK^oXYiqmrL)ONg)#sksNVHL8vR{sF^j#94+rv%{}qGrWnM( z>-WSQdp@x2x#KMh!})j9?1`9nHKe>|FWk>Y%*C>~^aC^CNGoJ-ZNytE)~&ng?U~+$ zJ>y?&y`N#o?Zw=)l>MGbroYnPzA=yc@fsh(MTZA%Jq@C6>Fa3ooSDrpKRY$}k+TY@qy=@emUs9!`MEt( zeWzUCC)W4P2c&}a)0UU*v}9G^vE@wdTq>-f;Q@lua6b>(FA`SZE|z&LPY_s`c5)Li zC{4;R1vFg7{2{RFjw^rD^muOG?&sETU$9+AQ$yRAZBTCw3UGw>SN^xP)|)d zGc~|5Hz?4LMM*OsFy7DH1|@i~_Ko{Y9N5w6?}2kD^_BSEUlTBLhV(lV#%)g}yOOz7 z!`wSLITFnBW!tTFl|#Yo$JEYx95S(2Y3d=yxsilT!POze^pT+RLbI~dHM32v3z{(| zI(n(*{;72&uCEQ= zvm0`Xa9-fO>bNqcHL5nQo@GUZ{XYz>Xp-!&P~73AU?I|aTpJZC3n-8k$UyG_^^(%2 z_BwS{#}j7mkz#o5*Ki@h;DVS9>*||mO=fluq#WR0gL?!5#%3$<7QCH7(xAVIC8KEGZ=#G_&>53db#g4O%-2fX5-9j<>ZCzDg8$@TnkMM?Y!)D4JoH=?aW>gOEc3*=7hebk#tS>nQjI z-TM^Jk;1?9g6nspAD+*qx;9Uy-BLyE(K#2Yi0`s4sdkn&sz+Dw2Du&s_ zfK#^8J!TTmbV!*=A(_809XF=Lz&J1LD8c9^1(S-c3~ZXy>}K04oB9dMx+mcR^VC?c z@Gf$}tKq60;|d1@j{C@9|H$C@$Y5iC){hMMbAM!TvY)QohPuBq6#Si`^0uMr9?TG3 zGnSWoWgD<_Da-vCUg{N19#n<%Xn z4Mho0|CBS~?405g#XChq!GbZ{ZJe^-%`N{XCiQAah;!dDml->R&F(y^~nV1L{h}{ zEI5o@&qB758{ihQGr0ZS0&n8F7aTlyh+8Nt<+2ynb6$l0O&oWE`((&i$hkl1ScB98 zXXfe`yd_-8La!a~@I7hd`WB948oBC)Y>w+%uvl<6Zv%(8gGN?18!k&+T;4)0dwnKX sw%{s4qOKNXGRozm{H@zLgolk*q%ClveQHgdmx|;4q%4bbu}SxT0Vn%AAOHXW literal 0 HcmV?d00001 diff --git a/deploy/coven-github/__pycache__/test_coven_github_adapter.cpython-314.pyc b/deploy/coven-github/__pycache__/test_coven_github_adapter.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3a5d4e8cd99da290c2b8cf241695b41f0cd3f63d GIT binary patch literal 4963 zcmcIoOKcm*8J;DVUv2fsB-pn4~~}6lil(BcmyD>OZ?&k+JC* zDcY4dJFowr+1c;^=9@j-=5-;s=KeVKdcOyuFId66Z1v3UuOPFAt|5`TfHY3OTQrM) zw`x|nTQ2aEHqADGL~A$F?44*3iF`K_ZC9-`ZKk`%{g~CrYr<8hXn*cigwh=dg%I?H z`@`2DI##a&|6D}>MIB$Vx~V8f`59STPBk0xP+%kM{tJ>dG|!oh^A^Z>LhH*s31v;N(M_3b>0L^DJN$! zbt>6Ho@g0mf$~L7$U$!Cq;nS4stlzfi*v-xxp%=ISLiP|%HIgJT*;}tM5 znxj_A7vQZ?-sob~s%!$`aG3rejSHFlYC@l9%y@u=hE+{!HFNez9yW;ij3fwt9Pm?L zgJcbT=Jx$Y`1z@-J5q5+$_HNDa-Uio|IFQS_tYJzJW_EVDUVKmaA3Yor$yEQD~82U9F5vaU3w0DK4jUg*GrE!^ym$MGdcT^$$p%bl}tE`!?G4$(YwX* zikyWhmD~k_7Zsy2%InahE?I@g&}K;yLMMxJEHZGKx{c~{az@2b9betMeI*8z8dH^}oU9cIj*UL^Y{{iA%W!{DUP!5o{@?uZECm{-wlrQSF44A|cwsr8 zzb>gbMX*Ngss^^I@&Lk{cAmd5JNe3(G&eCeIX!(*x;$}KoS3CHOh}$gnXu|4`J#p; zHY`p{X`E4RV6tk$YT}KeoS}k&u?xAVm4sjxqz@8mCs-|JG@W1pRR*Yz==f;qQzDr0 zy((oDRaJ6J=A_3ThqCYEB@8n%nYj+TKuL4LR%FU~Z3n=Z@s|)Wdi0>J7bel9RZirD&D*6ik6Cgj1`)u5+JO)p2`*ucG)caOoLc6hLdl;f z6f+q$ab)ypg6SN!wN&5YZ)17HdG!njC`?P*pz1kzfpwUp1%IL z2!m~=z@2Sr_LhGqo$Nu$KHm1sq$Bc$E%_ z4nBgQMrgK)Bhfw0eQz7jV{Dw?F^=~!HZIeqv1fteuk!?etnI|qmy9W9uwJl;T z1xi;YODc5%^%qGF1Px|8KpF_0M4+!;0JaBHw&hKg+S1D6A|{|67|vWJArc8sPMjHc z0k6Nr420A&PF)B4k`Rlf0i&tqdQ-^>7>NN`R7`nhQ?Dl=;ZVsD1Bbxveul@ia7vYHY?}Sbm{*I)TV(>!2DE0Bh-BDZ;$dQ{;MC5a<^Fai}ijV^`6-Fop@582aM6%T8$DI_8uj={#1$IgqVc8E_LmuV|CS z1<;-b3xy440#=D~AdaV|lo3E49NyTOhql!ZdnTK>n--D^8qO9N=h5IS40w zv+6zck@ry5JG|{3-qgAA%4Vb-ir@2;PbJIkb0C-loz=j>O5otTp7OI-s)6~B1M@Y1 zchw)S_(2r^VfMYq?_axbyH7qmR-U>1m*=WYHq>^w;vN2bU>>Z#(Tgxp z**pA=${6BL2ggQje}4MV*nsnWuK@Y?2i#)`@B71ge$;)|>U;-2hd9{*o*kgz0Pm9O z$8E@aB#G(6dW(-8@`(xu`W(*XC5eP#1QKHjJ2KNwiESvFl9FUiBgz8EP8FO4Bq@~v zoTQbMmcp5gB*EtwZ&M|hF&;dvZ*ZiO`C|!FC@K@8|Govw{FSB6HqLz&cG*sIU-ula zCAl5MQaxz&{_Gjx#FYd8ZrZl@?IF%SJrK63kjs!zp_o%N1|;fW9~0o~gbqd{6ogr5 zI1BrFUNF8soFWR;vFWQ5knC7Fj{6jKeu{kX|5w!WmEFb# wANbF3AVAi8s{4j3`-UGNy|^=JLA>X7Y1_K5#(QpE`q`zors06Z@7IU<52%IZw*UYD literal 0 HcmV?d00001 diff --git a/deploy/hosted/.env.example b/deploy/hosted/.env.example new file mode 100644 index 0000000..3617652 --- /dev/null +++ b/deploy/hosted/.env.example @@ -0,0 +1,20 @@ +# Secrets for deploy/hosted/compose.yaml. Copy to .env (mode 0600) and fill +# in. NEVER commit the filled-in file. + +# Public hostname the GitHub App webhook points at. +WEBHOOK_DOMAIN=gh.opencoven.ai + +# Image tag to run (a released ghcr.io/opencoven/coven-github tag). +COVEN_GITHUB_TAG=latest + +# Cloudflare R2 (S3-compatible) for Litestream SQLite replication. +# Create an R2 API token scoped to one bucket with object read+write. +R2_ACCOUNT_ID= +R2_BUCKET=coven-github-store +R2_ACCESS_KEY_ID= +R2_SECRET_ACCESS_KEY= + +# Host docker group id, so the unprivileged adapter user can drive the +# daemon for the container worker backend. provision.sh fills this in +# (getent group docker | cut -d: -f3). +DOCKER_GID= diff --git a/deploy/hosted/Caddyfile b/deploy/hosted/Caddyfile new file mode 100644 index 0000000..b55885e --- /dev/null +++ b/deploy/hosted/Caddyfile @@ -0,0 +1,13 @@ +# TLS ingress for hosted coven-github. Caddy provisions and renews +# Let's Encrypt certificates automatically for $WEBHOOK_DOMAIN. +{$WEBHOOK_DOMAIN} { + encode gzip + + # GitHub webhook deliveries and the tenant-scoped API. + reverse_proxy coven-github:3000 + + log { + output stdout + format console + } +} diff --git a/deploy/hosted/compose.yaml b/deploy/hosted/compose.yaml new file mode 100644 index 0000000..36a428b --- /dev/null +++ b/deploy/hosted/compose.yaml @@ -0,0 +1,77 @@ +# Production Docker Compose for hosted coven-github: adapter + TLS ingress +# (Caddy) + continuous SQLite replication (Litestream → Cloudflare R2). +# +# Runbook: docs/hosted-deploy.md. Secrets come from ./.env (never committed); +# copy .env.example and fill it in. +# +# Layout on the host (created by provision.sh): +# /opt/coven-github/ +# compose.yaml this file +# .env secrets (mode 0600) +# Caddyfile TLS ingress +# litestream.yml replication config +# config/production.toml adapter config +# keys/app.pem GitHub App private key (mode 0600) +# data/ SQLite store (replicated) + +services: + coven-github: + image: ghcr.io/opencoven/coven-github:${COVEN_GITHUB_TAG:-latest} + command: ["serve", "--config", "/config/production.toml"] + restart: unless-stopped + expose: + - "3000" + volumes: + - ./config:/config:ro + - ./keys:/keys:ro + - ./data:/data + - ./workspaces:/tmp/coven-github-tasks + # Container worker backend: the adapter launches per-task hardened + # containers through the host Docker daemon (docs/container-isolation.md). + - /var/run/docker.sock:/var/run/docker.sock + # The image runs unprivileged (uid 10001); grant the host docker group so + # the worker can drive the daemon. provision.sh fills DOCKER_GID in .env. + group_add: + - "${DOCKER_GID:?set in .env (getent group docker | cut -d: -f3)}" + environment: + RUST_LOG: "coven_github=info" + healthcheck: + test: ["CMD", "curl", "-fsS", "http://localhost:3000/healthz"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + + caddy: + image: caddy:2 + restart: unless-stopped + ports: + - "80:80" + - "443:443" + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + - caddy-data:/data + - caddy-config:/config + environment: + WEBHOOK_DOMAIN: ${WEBHOOK_DOMAIN:?set in .env, e.g. gh.opencoven.ai} + depends_on: + - coven-github + + litestream: + image: litestream/litestream:0.3 + restart: unless-stopped + command: ["replicate", "-config", "/etc/litestream.yml"] + volumes: + - ./litestream.yml:/etc/litestream.yml:ro + - ./data:/data + environment: + LITESTREAM_ACCESS_KEY_ID: ${R2_ACCESS_KEY_ID:?set in .env} + LITESTREAM_SECRET_ACCESS_KEY: ${R2_SECRET_ACCESS_KEY:?set in .env} + R2_BUCKET: ${R2_BUCKET:?set in .env} + R2_ACCOUNT_ID: ${R2_ACCOUNT_ID:?set in .env} + depends_on: + - coven-github + +volumes: + caddy-data: + caddy-config: diff --git a/deploy/hosted/litestream.yml b/deploy/hosted/litestream.yml new file mode 100644 index 0000000..5f79ac1 --- /dev/null +++ b/deploy/hosted/litestream.yml @@ -0,0 +1,14 @@ +# Litestream: continuous SQLite replication to Cloudflare R2 (S3-compatible). +# Credentials come from the environment (compose.yaml). Restore drill: +# litestream restore -config /etc/litestream.yml /data/coven-github.db +dbs: + - path: /data/coven-github.db + replicas: + - type: s3 + bucket: ${R2_BUCKET} + path: coven-github + endpoint: https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com + # R2 ignores regions but the S3 client requires one. + region: auto + retention: 720h # 30 days of point-in-time history + sync-interval: 10s diff --git a/deploy/hosted/provision.sh b/deploy/hosted/provision.sh new file mode 100755 index 0000000..e08820a --- /dev/null +++ b/deploy/hosted/provision.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Provision a fresh Ubuntu 24.04 host (e.g. Hetzner CPX41) to run hosted +# coven-github. Idempotent; run as root ON THE SERVER: +# +# curl -fsSL https://raw.githubusercontent.com/OpenCoven/coven-github/main/deploy/hosted/provision.sh | bash +# +# Then follow docs/hosted-deploy.md to place config, keys, and .env. +set -euo pipefail + +DEPLOY_DIR=/opt/coven-github +RAW=https://raw.githubusercontent.com/OpenCoven/coven-github/main/deploy/hosted + +echo "==> Installing Docker Engine + compose plugin" +if ! command -v docker >/dev/null 2>&1; then + curl -fsSL https://get.docker.com | sh +fi + +echo "==> Hardening: unattended upgrades + firewall (22/80/443 only)" +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get install -y -qq unattended-upgrades ufw curl +ufw default deny incoming +ufw default allow outgoing +ufw allow 22/tcp +ufw allow 80/tcp +ufw allow 443/tcp +ufw --force enable + +echo "==> Laying out ${DEPLOY_DIR}" +mkdir -p "${DEPLOY_DIR}"/{config,keys,data,workspaces} +chmod 700 "${DEPLOY_DIR}/keys" +# The image runs as uid 10001 (user `coven`); it owns its state dirs. +chown 10001 "${DEPLOY_DIR}/data" "${DEPLOY_DIR}/workspaces" +cd "${DEPLOY_DIR}" +for f in compose.yaml Caddyfile litestream.yml .env.example; do + curl -fsSL "${RAW}/${f}" -o "${f}" +done +[ -f .env ] || { cp .env.example .env && chmod 600 .env; } + +echo "==> Recording the docker group id for the compose worker" +DOCKER_GID="$(getent group docker | cut -d: -f3)" +grep -q '^DOCKER_GID=' .env \ + && sed -i "s/^DOCKER_GID=.*/DOCKER_GID=${DOCKER_GID}/" .env \ + || echo "DOCKER_GID=${DOCKER_GID}" >> .env + +cat <<'NEXT' + +Provisioned. Next (docs/hosted-deploy.md): + 1. Fill /opt/coven-github/.env (domain, image tag, R2 credentials) + 2. Place /opt/coven-github/config/production.toml and keys/app.pem (0600) + 3. Point DNS A/AAAA for your webhook domain at this host + 4. docker compose run --rm coven-github doctor --config /config/production.toml + 5. docker compose up -d + 6. curl -fsS https:///healthz +NEXT diff --git a/docs/hosted-deploy.md b/docs/hosted-deploy.md new file mode 100644 index 0000000..dfef646 --- /dev/null +++ b/docs/hosted-deploy.md @@ -0,0 +1,117 @@ +# Hosted deployment runbook + +How the hosted OpenCoven GitHub adapter runs in production: one VM, Docker +Compose, Caddy TLS ingress, and continuous SQLite replication to Cloudflare +R2. This is the operational half of the [HOSTED.md](../HOSTED.md) beta gates; +the code half (durable queue, isolation, metering, entitlements) is shipped. + +**Architecture choice:** the store is single-writer SQLite and the container +worker backend drives the host Docker daemon, so the natural unit is one +well-sized VM (Hetzner CPX41-class: 8 vCPU / 16 GB). Tier task caps keep beta +load well inside that. Hosted Dedicated later = this same stack on a +customer-dedicated VM. + +Everything lives in [`deploy/hosted/`](../deploy/hosted/): + +| File | Role | +|---|---| +| `provision.sh` | One-shot Ubuntu 24.04 host setup (Docker, firewall, layout) | +| `compose.yaml` | Adapter + Caddy + Litestream services | +| `Caddyfile` | TLS ingress for the webhook domain | +| `litestream.yml` | SQLite → R2 continuous replication | +| `.env.example` | Secrets template (domain, image tag, R2 credentials) | + +## First deploy + +1. **VM**: create an Ubuntu 24.04 host (Hetzner CPX41 or similar). Point DNS + `A`/`AAAA` for the webhook domain (e.g. `gh.opencoven.ai`) at it. +2. **Provision** (as root on the VM): + ```sh + curl -fsSL https://raw.githubusercontent.com/OpenCoven/coven-github/main/deploy/hosted/provision.sh | bash + ``` + Installs Docker, enables unattended upgrades, opens only 22/80/443, and + lays out `/opt/coven-github`. +3. **Secrets** (never in git): + - Fill `/opt/coven-github/.env` — domain, image tag, R2 token (create an + R2 bucket + API token scoped to it first). + - Place the GitHub App PEM at `keys/app.pem`, mode `0600`. + - Write `config/production.toml` from [`config/example.toml`](../config/example.toml): + `storage.path = "/data/coven-github.db"`, `worker.backend = "container"`, + `api.mode = "token"` with tenant tokens, `[billing] require_plan = true` + at Marketplace go-live (see + [marketplace-listing.md](marketplace-listing.md)), and `[[installations]]` + entries for grandfathered beta tenants. +4. **Preflight + start**: + ```sh + cd /opt/coven-github + docker compose run --rm coven-github doctor --config /config/production.toml + docker compose up -d + curl -fsS https://gh.opencoven.ai/healthz + ``` +5. **Wire GitHub**: set the App's webhook URL to + `https://gh.opencoven.ai/webhook` and send a ping delivery — the adapter + answers `pong` and records the delivery. + +The image is published by [`.github/workflows/publish.yml`](../.github/workflows/publish.yml) +to `ghcr.io/opencoven/coven-github` (`latest` + SHA on main, semver on tags). + +## Monitoring + +- **Liveness/readiness**: `GET /healthz` — 200 when the store answers, 503 + otherwise. Point an external uptime monitor at it (the compose healthcheck + restarts the container; the external check catches host/DNS/TLS failure). +- **Queue and task health**: the tenant API is the dashboard — + `/api/github/tasks` (states), `/api/github/usage` (per-installation load), + `/api/github/audit` (ignored deliveries, attempt outcomes). A growing + `queued` count with idle workers, or repeated `attempt:failed` audit rows, + is the primary alert condition. +- **Logs**: `docker compose logs -f coven-github` (`RUST_LOG` in + compose.yaml). Secrets are redacted by the adapter. +- **Replication**: `docker compose exec litestream litestream dbs -config + /etc/litestream.yml` should list the db; alert if the R2 bucket's latest + WAL segment is older than a few minutes. + +## Backup and restore + +Litestream streams every WAL segment to R2 (10 s sync, 30-day retention). +**Run a restore drill before onboarding paying tenants**, and quarterly: + +```sh +docker compose stop coven-github +docker compose run --rm litestream restore -if-replica-exists \ + -config /etc/litestream.yml -o /data/restored.db /data/coven-github.db +# inspect, then swap in and restart +``` + +Losing this database loses delivery idempotency, task history, usage +metering, and **billing entitlements** — treat replication health as a +page-worthy alert. + +## Upgrades + +```sh +cd /opt/coven-github +docker compose pull coven-github +docker compose up -d coven-github +curl -fsS https://gh.opencoven.ai/healthz +``` + +Schema migrations run forward-only at boot. GitHub redelivers webhooks that +5xx during the restart window, and the delivery-id dedup makes redelivery +safe. Pin `COVEN_GITHUB_TAG` to a SHA or semver tag for controlled rollouts; +roll back by re-pinning the previous tag (schema rollbacks are not supported +— restore from R2 if a migration must be undone). + +## Incident basics + +- **Webhook 5xx / down**: GitHub retries; fix the host, deliveries catch up. + Check `docker compose ps`, `/healthz`, disk space (`df -h /opt`). +- **Stuck task**: find it via `/api/github/tasks`, inspect + `/api/github/audit`; the worker enforces timeouts and cleans workspaces — + a task that outlives its timeout is a bug, capture logs before restarting. +- **Runaway load from one tenant**: intake caps and claim-time concurrency + already gate per installation; drop the tenant's tier or set explicit + `[installations.limits]` and restart. +- **Compromise suspected**: rotate the webhook secret and App PEM in GitHub, + replace `keys/app.pem` and the secret in `production.toml`, restart, and + audit `/api/github/audit` for the exposure window. From a718e51b95b298308e307bc3bd6bd299343c9766 Mon Sep 17 00:00:00 2001 From: Val Alexander Date: Wed, 8 Jul 2026 15:58:31 -0500 Subject: [PATCH 2/2] fix(webhook): minimal healthz failure body per review Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Val Alexander --- crates/webhook/src/routes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/webhook/src/routes.rs b/crates/webhook/src/routes.rs index 28788d7..85af861 100644 --- a/crates/webhook/src/routes.rs +++ b/crates/webhook/src/routes.rs @@ -53,7 +53,7 @@ pub async fn healthz(State(state): State) -> impl IntoResponse { error!("healthz store probe failed: {e:#}"); ( StatusCode::SERVICE_UNAVAILABLE, - Json(json!({"ok": false, "store": "unavailable"})), + Json(json!({"ok": false})), ) .into_response() }