From f5bceeed3ea53423f4efefee6e5bc25991e94b7f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jun 2026 08:51:12 +0000 Subject: [PATCH] build(deps): bump github.com/containerd/containerd/v2 Bumps [github.com/containerd/containerd/v2](https://github.com/containerd/containerd) from 2.2.3 to 2.2.5. - [Release notes](https://github.com/containerd/containerd/releases) - [Changelog](https://github.com/containerd/containerd/blob/main/RELEASES.md) - [Commits](https://github.com/containerd/containerd/compare/v2.2.3...v2.2.5) --- updated-dependencies: - dependency-name: github.com/containerd/containerd/v2 dependency-version: 2.2.5 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- go.mod | 14 +- go.sum | 28 +- .../containerd/v2/client/container_opts.go | 17 + .../containerd/v2/core/mount/temp.go | 2 +- .../containerd/v2/pkg/labels/labels.go | 12 + .../containerd/v2/pkg/labels/validate.go | 9 + .../containerd/v2/pkg/oci/spec_opts.go | 85 +- .../containerd/v2/version/version.go | 2 +- vendor/golang.org/x/crypto/blake2b/go125.go | 11 - .../chacha20poly1305_amd64.go | 6 +- .../chacha20poly1305/chacha20poly1305_amd64.s | 8664 ++++------------- vendor/golang.org/x/crypto/pkcs12/crypto.go | 4 + vendor/golang.org/x/crypto/pkcs12/mac.go | 8 + vendor/golang.org/x/crypto/ssh/certs.go | 18 +- vendor/golang.org/x/crypto/ssh/channel.go | 68 +- vendor/golang.org/x/crypto/ssh/cipher.go | 2 +- vendor/golang.org/x/crypto/ssh/client.go | 85 + vendor/golang.org/x/crypto/ssh/client_auth.go | 64 +- vendor/golang.org/x/crypto/ssh/connection.go | 10 +- vendor/golang.org/x/crypto/ssh/control.go | 155 + vendor/golang.org/x/crypto/ssh/kex.go | 75 +- vendor/golang.org/x/crypto/ssh/keys.go | 99 +- vendor/golang.org/x/crypto/ssh/mux.go | 39 +- vendor/golang.org/x/crypto/ssh/server.go | 165 +- vendor/golang.org/x/crypto/ssh/session.go | 3 + vendor/golang.org/x/net/http2/server.go | 21 - .../golang.org/x/net/http2/server_common.go | 22 + vendor/golang.org/x/net/http2/server_wrap.go | 40 + vendor/golang.org/x/net/http2/transport.go | 34 - .../x/net/http2/transport_common.go | 34 + .../x/net/http2/writesched_common.go | 41 + .../net/http2/writesched_priority_rfc7540.go | 41 - vendor/golang.org/x/sync/errgroup/errgroup.go | 2 +- .../golang.org/x/sync/semaphore/semaphore.go | 10 +- .../x/sync/singleflight/singleflight.go | 8 +- vendor/golang.org/x/sys/cpu/cpu.go | 19 +- .../golang.org/x/sys/cpu/cpu_linux_riscv64.go | 2 + vendor/golang.org/x/sys/cpu/cpu_loong64.go | 16 +- vendor/golang.org/x/sys/cpu/cpu_riscv64.go | 1 + vendor/golang.org/x/sys/unix/mkerrors.sh | 3 + vendor/golang.org/x/sys/unix/readv_unix.go | 103 + .../golang.org/x/sys/unix/syscall_darwin.go | 89 - vendor/golang.org/x/sys/unix/syscall_linux.go | 106 +- .../golang.org/x/sys/unix/syscall_openbsd.go | 4 + vendor/golang.org/x/sys/unix/zerrors_linux.go | 61 +- .../x/sys/unix/zerrors_linux_386.go | 7 +- .../x/sys/unix/zerrors_linux_amd64.go | 7 +- .../x/sys/unix/zerrors_linux_arm.go | 7 +- .../x/sys/unix/zerrors_linux_arm64.go | 7 +- .../x/sys/unix/zerrors_linux_loong64.go | 7 +- .../x/sys/unix/zerrors_linux_mips.go | 7 +- .../x/sys/unix/zerrors_linux_mips64.go | 7 +- .../x/sys/unix/zerrors_linux_mips64le.go | 7 +- .../x/sys/unix/zerrors_linux_mipsle.go | 7 +- .../x/sys/unix/zerrors_linux_ppc.go | 7 +- .../x/sys/unix/zerrors_linux_ppc64.go | 7 +- .../x/sys/unix/zerrors_linux_ppc64le.go | 7 +- .../x/sys/unix/zerrors_linux_riscv64.go | 1114 +-- .../x/sys/unix/zerrors_linux_s390x.go | 7 +- .../x/sys/unix/zerrors_linux_sparc64.go | 7 +- .../golang.org/x/sys/unix/zsyscall_linux.go | 8 +- .../x/sys/unix/zsyscall_openbsd_386.go | 84 + .../x/sys/unix/zsyscall_openbsd_386.s | 20 + .../x/sys/unix/zsyscall_openbsd_amd64.go | 84 + .../x/sys/unix/zsyscall_openbsd_amd64.s | 20 + .../x/sys/unix/zsyscall_openbsd_arm.go | 84 + .../x/sys/unix/zsyscall_openbsd_arm.s | 20 + .../x/sys/unix/zsyscall_openbsd_arm64.go | 84 + .../x/sys/unix/zsyscall_openbsd_arm64.s | 20 + .../x/sys/unix/zsyscall_openbsd_mips64.go | 84 + .../x/sys/unix/zsyscall_openbsd_mips64.s | 20 + .../x/sys/unix/zsyscall_openbsd_ppc64.go | 84 + .../x/sys/unix/zsyscall_openbsd_ppc64.s | 24 + .../x/sys/unix/zsyscall_openbsd_riscv64.go | 84 + .../x/sys/unix/zsyscall_openbsd_riscv64.s | 20 + .../x/sys/unix/zsysnum_linux_386.go | 4 + .../x/sys/unix/zsysnum_linux_amd64.go | 5 + .../x/sys/unix/zsysnum_linux_arm.go | 4 + .../x/sys/unix/zsysnum_linux_arm64.go | 4 + .../x/sys/unix/zsysnum_linux_loong64.go | 5 + .../x/sys/unix/zsysnum_linux_mips.go | 4 + .../x/sys/unix/zsysnum_linux_mips64.go | 4 + .../x/sys/unix/zsysnum_linux_mips64le.go | 4 + .../x/sys/unix/zsysnum_linux_mipsle.go | 4 + .../x/sys/unix/zsysnum_linux_ppc.go | 4 + .../x/sys/unix/zsysnum_linux_ppc64.go | 4 + .../x/sys/unix/zsysnum_linux_ppc64le.go | 4 + .../x/sys/unix/zsysnum_linux_riscv64.go | 4 + .../x/sys/unix/zsysnum_linux_s390x.go | 4 + .../x/sys/unix/zsysnum_linux_sparc64.go | 5 + vendor/golang.org/x/sys/unix/ztypes_linux.go | 123 +- .../golang.org/x/sys/unix/ztypes_linux_386.go | 12 + .../x/sys/unix/ztypes_linux_amd64.go | 12 + .../golang.org/x/sys/unix/ztypes_linux_arm.go | 12 + .../x/sys/unix/ztypes_linux_arm64.go | 12 + .../x/sys/unix/ztypes_linux_loong64.go | 12 + .../x/sys/unix/ztypes_linux_mips.go | 12 + .../x/sys/unix/ztypes_linux_mips64.go | 12 + .../x/sys/unix/ztypes_linux_mips64le.go | 12 + .../x/sys/unix/ztypes_linux_mipsle.go | 12 + .../golang.org/x/sys/unix/ztypes_linux_ppc.go | 12 + .../x/sys/unix/ztypes_linux_ppc64.go | 12 + .../x/sys/unix/ztypes_linux_ppc64le.go | 12 + .../x/sys/unix/ztypes_linux_riscv64.go | 12 + .../x/sys/unix/ztypes_linux_s390x.go | 12 + .../x/sys/unix/ztypes_linux_sparc64.go | 12 + .../x/sys/windows/syscall_windows.go | 3 + .../golang.org/x/sys/windows/types_windows.go | 4 +- .../x/sys/windows/zsyscall_windows.go | 35 + vendor/modules.txt | 16 +- 110 files changed, 5088 insertions(+), 7590 deletions(-) delete mode 100644 vendor/golang.org/x/crypto/blake2b/go125.go create mode 100644 vendor/golang.org/x/crypto/ssh/control.go create mode 100644 vendor/golang.org/x/sys/unix/readv_unix.go diff --git a/go.mod b/go.mod index 4c49c07b20e60..ab5a868ccb544 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/cloudflare/cfssl v1.6.5 github.com/containerd/cgroups/v3 v3.1.3 github.com/containerd/containerd/api v1.10.0 - github.com/containerd/containerd/v2 v2.2.3 + github.com/containerd/containerd/v2 v2.2.5 github.com/containerd/continuity v0.5.0 github.com/containerd/errdefs v1.0.0 github.com/containerd/fifo v1.1.0 @@ -103,10 +103,10 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 go.opentelemetry.io/otel/sdk v1.43.0 go.opentelemetry.io/otel/trace v1.43.0 - golang.org/x/net v0.54.0 - golang.org/x/sync v0.20.0 - golang.org/x/sys v0.44.0 - golang.org/x/text v0.37.0 + golang.org/x/net v0.55.0 + golang.org/x/sync v0.21.0 + golang.org/x/sys v0.46.0 + golang.org/x/text v0.38.0 golang.org/x/time v0.15.0 google.golang.org/genproto/googleapis/api v0.0.0-20260406210006-6f92a3bedf2d google.golang.org/grpc v1.80.0 @@ -287,10 +287,10 @@ require ( go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.51.0 // indirect + golang.org/x/crypto v0.53.0 // indirect golang.org/x/mod v0.36.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/term v0.43.0 // indirect + golang.org/x/term v0.44.0 // indirect golang.org/x/tools v0.45.0 // indirect google.golang.org/api v0.274.0 // indirect google.golang.org/genproto v0.0.0-20260401024825-9d38bb4040a9 // indirect diff --git a/go.sum b/go.sum index 52104b3064d74..9f1c3fbacd566 100644 --- a/go.sum +++ b/go.sum @@ -162,8 +162,8 @@ github.com/containerd/console v1.0.5 h1:R0ymNeydRqH2DmakFNdmjR2k0t7UPuiOV/N/27/q github.com/containerd/console v1.0.5/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/containerd/containerd/api v1.10.0 h1:5n0oHYVBwN4VhoX9fFykCV9dF1/BvAXeg2F8W6UYq1o= github.com/containerd/containerd/api v1.10.0/go.mod h1:NBm1OAk8ZL+LG8R0ceObGxT5hbUYj7CzTmR3xh0DlMM= -github.com/containerd/containerd/v2 v2.2.3 h1:mOBRLaHGvmgy0bRo1Sg6OD8ugMKZIvCoWWMeMMygliA= -github.com/containerd/containerd/v2 v2.2.3/go.mod h1:ns24cwt+p36mRnuKE3hLRxVBpuSP+a/Y25AMki1t/RY= +github.com/containerd/containerd/v2 v2.2.5 h1:KTFzB02LviYmmfRmz8r9UFd+n6YlddVFK+5lbgQXUTU= +github.com/containerd/containerd/v2 v2.2.5/go.mod h1:5t2+xFv2dGd/iDYp9Z8DXB4cmWrWQi1XqxGJPS2gBzU= github.com/containerd/continuity v0.5.0 h1:7a85HZpCSs+1Zps0Ee3DPSuAWY+0SJM1JNM51nlEVDg= github.com/containerd/continuity v0.5.0/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -910,8 +910,8 @@ golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20250911091902-df9299821621 h1:2id6c1/gto0kaHYyrixvknJ8tUK/Qs5IsmBtrc+FtgU= golang.org/x/exp v0.0.0-20250911091902-df9299821621/go.mod h1:TwQYMMnGpvZyc+JpB/UAuTNIsVJifOlSkrZkhcvpVUk= @@ -945,8 +945,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= -golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -962,8 +962,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -992,15 +992,15 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= -golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -1008,8 +1008,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/vendor/github.com/containerd/containerd/v2/client/container_opts.go b/vendor/github.com/containerd/containerd/v2/client/container_opts.go index 04f2a9062114e..762dcb4b961b1 100644 --- a/vendor/github.com/containerd/containerd/v2/client/container_opts.go +++ b/vendor/github.com/containerd/containerd/v2/client/container_opts.go @@ -21,14 +21,17 @@ import ( "encoding/json" "errors" "fmt" + "maps" "github.com/containerd/containerd/v2/core/containers" "github.com/containerd/containerd/v2/core/content" "github.com/containerd/containerd/v2/core/images" "github.com/containerd/containerd/v2/core/snapshots" + "github.com/containerd/containerd/v2/pkg/labels" "github.com/containerd/containerd/v2/pkg/namespaces" "github.com/containerd/containerd/v2/pkg/oci" "github.com/containerd/errdefs" + "github.com/containerd/log" "github.com/containerd/typeurl/v2" "github.com/opencontainers/image-spec/identity" v1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -113,6 +116,10 @@ func WithContainerLabels(labels map[string]string) NewContainerOpts { // The existing labels are cleared as this is expected to be the first // operation in setting up a container's labels. Use WithAdditionalContainerLabels // to add/overwrite the existing image config labels. +// +// Image config labels in the namespaces reserved for containerd +// (containerd.io/) and the CRI plugin (io.cri-containerd) are not copied +// to the container. func WithImageConfigLabels(image Image) NewContainerOpts { return func(ctx context.Context, _ *Client, c *containers.Container) error { ic, err := image.Config(ctx) @@ -138,6 +145,16 @@ func WithImageConfigLabels(image Image) NewContainerOpts { config = ociimage.Config c.Labels = config.Labels + // Labels in the containerd.io/* namespace are interpreted by containerd + // itself, and labels in the io.cri-containerd.* namespace are interpreted + // by the CRI plugin, so they are not copied from untrusted image configs. + maps.DeleteFunc(c.Labels, func(k, _ string) bool { + if labels.IsReserved(k) { + log.G(ctx).Warnf("skipping image label %q: the label namespace is reserved for containerd; possible malicious image attempting to alter containerd behavior", k) + return true + } + return false + }) return nil } } diff --git a/vendor/github.com/containerd/containerd/v2/core/mount/temp.go b/vendor/github.com/containerd/containerd/v2/core/mount/temp.go index 39a7aaaddf6ac..9d436d67428f5 100644 --- a/vendor/github.com/containerd/containerd/v2/core/mount/temp.go +++ b/vendor/github.com/containerd/containerd/v2/core/mount/temp.go @@ -88,7 +88,7 @@ func RemoveVolatileOption(mounts []Mount) []Mount { continue } for j, opt := range m.Options { - if opt == "volatile" { + if opt == "volatile" || opt == "fsync=volatile" { if out == nil { out = copyMounts(mounts) } diff --git a/vendor/github.com/containerd/containerd/v2/pkg/labels/labels.go b/vendor/github.com/containerd/containerd/v2/pkg/labels/labels.go index 0f9bab5c5d466..ba4c245e45be3 100644 --- a/vendor/github.com/containerd/containerd/v2/pkg/labels/labels.go +++ b/vendor/github.com/containerd/containerd/v2/pkg/labels/labels.go @@ -16,6 +16,18 @@ package labels +// ReservedPrefix is the prefix of the label namespace reserved for labels +// defined and consumed by containerd itself. Labels in this namespace must +// not be copied from untrusted sources such as image config labels. Use +// IsReserved to check for such labels. +const ReservedPrefix = "containerd.io/" + +// CRIContainerdPrefix is the prefix of the label namespace reserved for +// labels defined and consumed by containerd's CRI plugin. Labels in this +// namespace must not be copied from untrusted sources such as image config +// labels. Use IsReserved to check for such labels. +const CRIContainerdPrefix = "io.cri-containerd" + // LabelUncompressed is added to compressed layer contents. // The value is digest of the uncompressed content. const LabelUncompressed = "containerd.io/uncompressed" diff --git a/vendor/github.com/containerd/containerd/v2/pkg/labels/validate.go b/vendor/github.com/containerd/containerd/v2/pkg/labels/validate.go index 6f23cdd7c65ed..495427bb46796 100644 --- a/vendor/github.com/containerd/containerd/v2/pkg/labels/validate.go +++ b/vendor/github.com/containerd/containerd/v2/pkg/labels/validate.go @@ -18,6 +18,7 @@ package labels import ( "fmt" + "strings" "github.com/containerd/errdefs" ) @@ -39,3 +40,11 @@ func Validate(k, v string) error { } return nil } + +// IsReserved returns true if the label key is in a namespace reserved for +// containerd (ReservedPrefix) or its CRI plugin (CRIContainerdPrefix). +// Reserved labels are interpreted by containerd and must not be copied from +// untrusted sources such as image config labels. +func IsReserved(k string) bool { + return strings.HasPrefix(k, ReservedPrefix) || strings.HasPrefix(k, CRIContainerdPrefix) +} diff --git a/vendor/github.com/containerd/containerd/v2/pkg/oci/spec_opts.go b/vendor/github.com/containerd/containerd/v2/pkg/oci/spec_opts.go index c298e4bb29c2c..7e695fda138e2 100644 --- a/vendor/github.com/containerd/containerd/v2/pkg/oci/spec_opts.go +++ b/vendor/github.com/containerd/containerd/v2/pkg/oci/spec_opts.go @@ -24,6 +24,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "io/fs" "math" "os" @@ -626,14 +627,25 @@ func WithUser(userstr string) SpecOpts { return nil } + isErrRange := func(err error) bool { + var numErr *strconv.NumError + return errors.As(err, &numErr) && numErr.Err == strconv.ErrRange + } + parts := strings.Split(userstr, ":") switch len(parts) { case 1: v, err := strconv.Atoi(parts[0]) - if err != nil || v < minUserID || v > maxUserID { - // if we cannot parse as an int32 then try to see if it is a username + if err != nil { + if isErrRange(err) { + return fmt.Errorf("invalid USER value %q: uid out of range", userstr) + } + // Non-numeric user value; treat it as a username. return WithUsername(userstr)(ctx, client, c, s) } + if v < minUserID || v > maxUserID { + return fmt.Errorf("invalid USER value %q: uid out of range", userstr) + } return WithUserID(uint32(v))(ctx, client, c, s) case 2: var ( @@ -642,14 +654,24 @@ func WithUser(userstr string) SpecOpts { ) var uid, gid uint32 v, err := strconv.Atoi(parts[0]) - if err != nil || v < minUserID || v > maxUserID { + if err != nil { + if isErrRange(err) { + return fmt.Errorf("invalid USER value %q: uid out of range", userstr) + } username = parts[0] + } else if v < minUserID || v > maxUserID { + return fmt.Errorf("invalid USER value %q: uid out of range", userstr) } else { uid = uint32(v) } v, err = strconv.Atoi(parts[1]) - if err != nil || v < minGroupID || v > maxGroupID { + if err != nil { + if isErrRange(err) { + return fmt.Errorf("invalid USER value %q: gid out of range", userstr) + } groupname = parts[1] + } else if v < minGroupID || v > maxGroupID { + return fmt.Errorf("invalid USER value %q: gid out of range", userstr) } else { gid = uint32(v) } @@ -1800,10 +1822,13 @@ type readLinker interface { // openUserFile attempts to open a file within the root fs. // It handles cases where the file is an absolute symlink (e.g., NixOS /etc/passwd -> /nix/store/...), // which triggers "path escapes from parent" errors in Go 1.24+ due to stricter os.DirFS validation. +// +// The returned file rejects non-regular sources and returns an error if more +// than maxUserFileBytes are read from it. func openUserFile(root fs.FS, name string) (fs.File, error) { f, err := root.Open(name) if err == nil { - return f, nil + return wrapUserFile(f, name) } // Check if the FS implements our local ReadLink interface. @@ -1820,7 +1845,11 @@ func openUserFile(root fs.FS, name string) (fs.File, error) { if rerr == nil { // filepath.Rel might return OS-specific separators (backslashes on Windows). // fs.Open strictly expects forward slashes, so we convert it. - return root.Open(filepath.ToSlash(rel)) + f, oerr := root.Open(filepath.ToSlash(rel)) + if oerr != nil { + return nil, oerr + } + return wrapUserFile(f, name) } } } @@ -1829,3 +1858,47 @@ func openUserFile(root fs.FS, name string) (fs.File, error) { // Return the original error if we couldn't resolve it return nil, err } + +// maxUserFileBytes caps how much data is read from any user-database file +// opened via openUserFile. Real systems keep these files well under 1 MiB; +// 10 MiB is generous headroom while keeping peak memory during +// user.ParsePasswd/ParseGroup bounded to single-digit MiB. +const maxUserFileBytes = 10 << 20 + +// wrapUserFile rejects non-regular sources and returns an fs.File that +// errors out if more than maxUserFileBytes are read from it. +func wrapUserFile(f fs.File, name string) (fs.File, error) { + info, err := f.Stat() + if err != nil { + f.Close() + return nil, fmt.Errorf("stat %s: %w", name, err) + } + if !info.Mode().IsRegular() { + f.Close() + return nil, fmt.Errorf("%s is not a regular file", name) + } + return &limitedFile{ + File: f, + // Allow one byte past the cap so an overflow surfaces as an + // error rather than a silent EOF that the parser would treat as + // a clean end-of-file (and miss any entries past the cap). + r: &io.LimitedReader{R: f, N: maxUserFileBytes + 1}, + name: name, + }, nil +} + +// limitedFile is an fs.File whose Read returns an error once more than +// maxUserFileBytes have been read. +type limitedFile struct { + fs.File + r *io.LimitedReader + name string +} + +func (l *limitedFile) Read(p []byte) (int, error) { + n, err := l.r.Read(p) + if l.r.N == 0 { + return n, fmt.Errorf("%q exceeds %d bytes", l.name, maxUserFileBytes) + } + return n, err +} diff --git a/vendor/github.com/containerd/containerd/v2/version/version.go b/vendor/github.com/containerd/containerd/v2/version/version.go index cf4cb8818a00c..9d8e01e0eb786 100644 --- a/vendor/github.com/containerd/containerd/v2/version/version.go +++ b/vendor/github.com/containerd/containerd/v2/version/version.go @@ -24,7 +24,7 @@ var ( Package = "github.com/containerd/containerd/v2" // Version holds the complete version number. Filled in at linking time. - Version = "2.2.3+unknown" + Version = "2.2.5+unknown" // Revision is filled with the VCS (e.g. git) revision being used to build // the program at linking time. diff --git a/vendor/golang.org/x/crypto/blake2b/go125.go b/vendor/golang.org/x/crypto/blake2b/go125.go deleted file mode 100644 index 67e990b7e1695..0000000000000 --- a/vendor/golang.org/x/crypto/blake2b/go125.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2025 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build go1.25 - -package blake2b - -import "hash" - -var _ hash.XOF = (*xof)(nil) diff --git a/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go b/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go index b850e772e1656..bfe546b60a2db 100644 --- a/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go +++ b/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go @@ -20,7 +20,7 @@ func chacha20Poly1305Open(dst []byte, key []uint32, src, ad []byte) bool func chacha20Poly1305Seal(dst []byte, key []uint32, src, ad []byte) var ( - useAVX2 = cpu.X86.HasAVX2 && cpu.X86.HasBMI2 + useAVX2 = cpu.X86.HasSSSE3 && cpu.X86.HasAVX2 && cpu.X86.HasBMI2 ) // setupState writes a ChaCha20 input matrix to state. See @@ -47,7 +47,7 @@ func setupState(state *[16]uint32, key *[32]byte, nonce []byte) { } func (c *chacha20poly1305) seal(dst, nonce, plaintext, additionalData []byte) []byte { - if !cpu.X86.HasSSSE3 { + if !useAVX2 { return c.sealGeneric(dst, nonce, plaintext, additionalData) } @@ -66,7 +66,7 @@ func (c *chacha20poly1305) seal(dst, nonce, plaintext, additionalData []byte) [] } func (c *chacha20poly1305) open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) { - if !cpu.X86.HasSSSE3 { + if !useAVX2 { return c.openGeneric(dst, nonce, ciphertext, additionalData) } diff --git a/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s b/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s index fd5ee845f9faa..c703c13471acc 100644 --- a/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s +++ b/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s @@ -6,27 +6,6 @@ // func polyHashADInternal<>() TEXT polyHashADInternal<>(SB), NOSPLIT, $0 - // Hack: Must declare #define macros inside of a function due to Avo constraints - // ROL rotates the uint32s in register R left by N bits, using temporary T. - #define ROL(N, R, T) \ - MOVO R, T; \ - PSLLL $(N), T; \ - PSRLL $(32-(N)), R; \ - PXOR T, R - - // ROL8 rotates the uint32s in register R left by 8, using temporary T if needed. - #ifdef GOAMD64_v2 - #define ROL8(R, T) PSHUFB ·rol8<>(SB), R - #else - #define ROL8(R, T) ROL(8, R, T) - #endif - - // ROL16 rotates the uint32s in register R left by 16, using temporary T if needed. - #ifdef GOAMD64_v2 - #define ROL16(R, T) PSHUFB ·rol16<>(SB), R - #else - #define ROL16(R, T) ROL(16, R, T) - #endif XORQ R10, R10 XORQ R11, R11 XORQ R12, R12 @@ -192,676 +171,112 @@ hashADDone: // Requires: AVX, AVX2, BMI2, CMOV, SSE2 TEXT ·chacha20Poly1305Open(SB), $288-97 // For aligned stack access - MOVQ SP, BP - ADDQ $0x20, BP - ANDQ $-32, BP - MOVQ dst_base+0(FP), DI - MOVQ key_base+24(FP), R8 - MOVQ src_base+48(FP), SI - MOVQ src_len+56(FP), BX - MOVQ ad_base+72(FP), CX - - // Check for AVX2 support - CMPB ·useAVX2+0(SB), $0x01 - JE chacha20Poly1305Open_AVX2 + MOVQ SP, BP + ADDQ $0x20, BP + ANDQ $-32, BP + MOVQ dst_base+0(FP), DI + MOVQ key_base+24(FP), R8 + MOVQ src_base+48(FP), SI + MOVQ src_len+56(FP), BX + MOVQ ad_base+72(FP), CX + VZEROUPPER + VMOVDQU ·chacha20Constants<>+0(SB), Y0 + VBROADCASTI128 16(R8), Y14 + VBROADCASTI128 32(R8), Y12 + VBROADCASTI128 48(R8), Y4 + VPADDD ·avx2InitMask<>+0(SB), Y4, Y4 // Special optimization, for very short buffers - CMPQ BX, $0x80 - JBE openSSE128 - - // For long buffers, prepare the poly key first - MOVOU ·chacha20Constants<>+0(SB), X0 - MOVOU 16(R8), X3 - MOVOU 32(R8), X6 - MOVOU 48(R8), X9 - MOVO X9, X13 - - // Store state on stack for future use - MOVO X3, 32(BP) - MOVO X6, 48(BP) - MOVO X9, 128(BP) - MOVQ $0x0000000a, R9 - -openSSEPreparePolyKey: - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - DECQ R9 - JNE openSSEPreparePolyKey + CMPQ BX, $0xc0 + JBE openAVX2192 + CMPQ BX, $0x00000140 + JBE openAVX2320 + + // For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream + VMOVDQA Y14, 32(BP) + VMOVDQA Y12, 64(BP) + VMOVDQA Y4, 192(BP) + MOVQ $0x0000000a, R9 + +openAVX2PreparePolyKey: + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x04, Y4, Y4, Y4 + DECQ R9 + JNE openAVX2PreparePolyKey + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD 32(BP), Y14, Y14 + VPADDD 64(BP), Y12, Y12 + VPADDD 192(BP), Y4, Y4 + VPERM2I128 $0x02, Y0, Y14, Y3 - // A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded - PADDL ·chacha20Constants<>+0(SB), X0 - PADDL 32(BP), X3 + // Clamp and store poly key + VPAND ·polyClampMask<>+0(SB), Y3, Y3 + VMOVDQA Y3, (BP) - // Clamp and store the key - PAND ·polyClampMask<>+0(SB), X0 - MOVO X0, (BP) - MOVO X3, 16(BP) + // Stream for the first 64 bytes + VPERM2I128 $0x13, Y0, Y14, Y0 + VPERM2I128 $0x13, Y12, Y4, Y14 - // Hash AAD + // Hash AD + first 64 bytes MOVQ ad_len+80(FP), R9 CALL polyHashADInternal<>(SB) + XORQ CX, CX -openSSEMainLoop: - CMPQ BX, $0x00000100 - JB openSSEMainLoopDone - - // Load state, increment counter blocks - MOVO ·chacha20Constants<>+0(SB), X0 - MOVO 32(BP), X3 - MOVO 48(BP), X6 - MOVO 128(BP), X9 - PADDL ·sseIncMask<>+0(SB), X9 - MOVO X0, X1 - MOVO X3, X4 - MOVO X6, X7 - MOVO X9, X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X1, X2 - MOVO X4, X5 - MOVO X7, X8 - MOVO X10, X11 - PADDL ·sseIncMask<>+0(SB), X11 - MOVO X2, X12 - MOVO X5, X13 - MOVO X8, X14 - MOVO X11, X15 - PADDL ·sseIncMask<>+0(SB), X15 - - // Store counters - MOVO X9, 80(BP) - MOVO X10, 96(BP) - MOVO X11, 112(BP) - MOVO X15, 128(BP) - - // There are 10 ChaCha20 iterations of 2QR each, so for 6 iterations we hash - // 2 blocks, and for the remaining 4 only 1 block - for a total of 16 - MOVQ $0x00000004, CX - MOVQ SI, R9 - -openSSEInternalLoop: - MOVO X14, 64(BP) - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X3 - PXOR X14, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X3 - PXOR X14, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X4 - PXOR X14, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X4 - PXOR X14, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X5 - PXOR X14, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X5 - PXOR X14, X5 - MOVO 64(BP), X14 - MOVO X7, 64(BP) - PADDD X13, X12 - PXOR X12, X15 - ROL16(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x0c, X7 - PSRLL $0x14, X13 - PXOR X7, X13 - PADDD X13, X12 - PXOR X12, X15 - ROL8(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x07, X7 - PSRLL $0x19, X13 - PXOR X7, X13 - MOVO 64(BP), X7 - ADDQ (R9), R10 - ADCQ 8(R9), R11 - ADCQ $0x01, R12 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x0c - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - LEAQ 16(R9), R9 - MOVO X14, 64(BP) - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X3 - PXOR X14, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X3 - PXOR X14, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X4 - PXOR X14, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X4 - PXOR X14, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X5 - PXOR X14, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X5 - PXOR X14, X5 - MOVO 64(BP), X14 - MOVO X7, 64(BP) - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - PADDD X13, X12 - PXOR X12, X15 - ROL16(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x0c, X7 - PSRLL $0x14, X13 - PXOR X7, X13 - PADDD X13, X12 - PXOR X12, X15 - ROL8(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x07, X7 - PSRLL $0x19, X13 - PXOR X7, X13 - MOVO 64(BP), X7 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x04 - DECQ CX - JGE openSSEInternalLoop - ADDQ (R9), R10 - ADCQ 8(R9), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - LEAQ 16(R9), R9 - CMPQ CX, $-6 - JG openSSEInternalLoop - - // Add in the state - PADDD ·chacha20Constants<>+0(SB), X0 - PADDD ·chacha20Constants<>+0(SB), X1 - PADDD ·chacha20Constants<>+0(SB), X2 - PADDD ·chacha20Constants<>+0(SB), X12 - PADDD 32(BP), X3 - PADDD 32(BP), X4 - PADDD 32(BP), X5 - PADDD 32(BP), X13 - PADDD 48(BP), X6 - PADDD 48(BP), X7 - PADDD 48(BP), X8 - PADDD 48(BP), X14 - PADDD 80(BP), X9 - PADDD 96(BP), X10 - PADDD 112(BP), X11 - PADDD 128(BP), X15 - - // Load - xor - store - MOVO X15, 64(BP) - MOVOU (SI), X15 - PXOR X15, X0 - MOVOU X0, (DI) - MOVOU 16(SI), X15 - PXOR X15, X3 - MOVOU X3, 16(DI) - MOVOU 32(SI), X15 - PXOR X15, X6 - MOVOU X6, 32(DI) - MOVOU 48(SI), X15 - PXOR X15, X9 - MOVOU X9, 48(DI) - MOVOU 64(SI), X9 - PXOR X9, X1 - MOVOU X1, 64(DI) - MOVOU 80(SI), X9 - PXOR X9, X4 - MOVOU X4, 80(DI) - MOVOU 96(SI), X9 - PXOR X9, X7 - MOVOU X7, 96(DI) - MOVOU 112(SI), X9 - PXOR X9, X10 - MOVOU X10, 112(DI) - MOVOU 128(SI), X9 - PXOR X9, X2 - MOVOU X2, 128(DI) - MOVOU 144(SI), X9 - PXOR X9, X5 - MOVOU X5, 144(DI) - MOVOU 160(SI), X9 - PXOR X9, X8 - MOVOU X8, 160(DI) - MOVOU 176(SI), X9 - PXOR X9, X11 - MOVOU X11, 176(DI) - MOVOU 192(SI), X9 - PXOR X9, X12 - MOVOU X12, 192(DI) - MOVOU 208(SI), X9 - PXOR X9, X13 - MOVOU X13, 208(DI) - MOVOU 224(SI), X9 - PXOR X9, X14 - MOVOU X14, 224(DI) - MOVOU 240(SI), X9 - PXOR 64(BP), X9 - MOVOU X9, 240(DI) - LEAQ 256(SI), SI - LEAQ 256(DI), DI - SUBQ $0x00000100, BX - JMP openSSEMainLoop - -openSSEMainLoopDone: - // Handle the various tail sizes efficiently - TESTQ BX, BX - JE openSSEFinalize - CMPQ BX, $0x40 - JBE openSSETail64 - CMPQ BX, $0x80 - JBE openSSETail128 - CMPQ BX, $0xc0 - JBE openSSETail192 - JMP openSSETail256 - -openSSEFinalize: - // Hash in the PT, AAD lengths - ADDQ ad_len+80(FP), R10 - ADCQ src_len+56(FP), R11 +openAVX2InitialHash64: + ADDQ (SI)(CX*1), R10 + ADCQ 8(SI)(CX*1), R11 ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 IMULQ R12, R15 + MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 @@ -878,3187 +293,494 @@ openSSEFinalize: ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 + ADDQ $0x10, CX + CMPQ CX, $0x40 + JNE openAVX2InitialHash64 - // Final reduce - MOVQ R10, R13 - MOVQ R11, R14 - MOVQ R12, R15 - SUBQ $-5, R10 - SBBQ $-1, R11 - SBBQ $0x03, R12 - CMOVQCS R13, R10 - CMOVQCS R14, R11 - CMOVQCS R15, R12 - - // Add in the "s" part of the key - ADDQ 16(BP), R10 - ADCQ 24(BP), R11 - - // Finally, constant time compare to the tag at the end of the message - XORQ AX, AX - MOVQ $0x00000001, DX - XORQ (SI), R10 - XORQ 8(SI), R11 - ORQ R11, R10 - CMOVQEQ DX, AX + // Decrypt the first 64 bytes + VPXOR (SI), Y0, Y0 + VPXOR 32(SI), Y14, Y14 + VMOVDQU Y0, (DI) + VMOVDQU Y14, 32(DI) + LEAQ 64(SI), SI + LEAQ 64(DI), DI + SUBQ $0x40, BX - // Return true iff tags are equal - MOVB AX, ret+96(FP) - RET +openAVX2MainLoop: + CMPQ BX, $0x00000200 + JB openAVX2MainLoopDone -openSSE128: - MOVOU ·chacha20Constants<>+0(SB), X0 - MOVOU 16(R8), X3 - MOVOU 32(R8), X6 - MOVOU 48(R8), X9 - MOVO X0, X1 - MOVO X3, X4 - MOVO X6, X7 - MOVO X9, X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X1, X2 - MOVO X4, X5 - MOVO X7, X8 - MOVO X10, X11 - PADDL ·sseIncMask<>+0(SB), X11 - MOVO X3, X13 - MOVO X6, X14 - MOVO X10, X15 - MOVQ $0x0000000a, R9 - -openSSE128InnerCipherLoop: - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X5 - PXOR X12, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X5 - PXOR X12, X5 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X5 - PXOR X12, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X5 - PXOR X12, X5 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - DECQ R9 - JNE openSSE128InnerCipherLoop - - // A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded - PADDL ·chacha20Constants<>+0(SB), X0 - PADDL ·chacha20Constants<>+0(SB), X1 - PADDL ·chacha20Constants<>+0(SB), X2 - PADDL X13, X3 - PADDL X13, X4 - PADDL X13, X5 - PADDL X14, X7 - PADDL X14, X8 - PADDL X15, X10 - PADDL ·sseIncMask<>+0(SB), X15 - PADDL X15, X11 - - // Clamp and store the key - PAND ·polyClampMask<>+0(SB), X0 - MOVOU X0, (BP) - MOVOU X3, 16(BP) + // Load state, increment counter blocks, store the incremented counters + VMOVDQU ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA Y0, Y6 + VMOVDQA Y0, Y7 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA Y14, Y10 + VMOVDQA Y14, Y11 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA Y12, Y8 + VMOVDQA Y12, Y15 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 + VMOVDQA Y4, 96(BP) + VMOVDQA Y1, 128(BP) + VMOVDQA Y2, 160(BP) + VMOVDQA Y3, 192(BP) + XORQ CX, CX - // Hash - MOVQ ad_len+80(FP), R9 - CALL polyHashADInternal<>(SB) - -openSSE128Open: - CMPQ BX, $0x10 - JB openSSETail16 - SUBQ $0x10, BX - - // Load for hashing - ADDQ (SI), R10 - ADCQ 8(SI), R11 - ADCQ $0x01, R12 - - // Load for decryption - MOVOU (SI), X12 - PXOR X12, X1 - MOVOU X1, (DI) - LEAQ 16(SI), SI - LEAQ 16(DI), DI - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - - // Shift the stream "left" - MOVO X4, X1 - MOVO X7, X4 - MOVO X10, X7 - MOVO X2, X10 - MOVO X5, X2 - MOVO X8, X5 - MOVO X11, X8 - JMP openSSE128Open - -openSSETail16: - TESTQ BX, BX - JE openSSEFinalize - - // We can safely load the CT from the end, because it is padded with the MAC - MOVQ BX, R9 - SHLQ $0x04, R9 - LEAQ ·andMask<>+0(SB), R13 - MOVOU (SI), X12 - ADDQ BX, SI - PAND -16(R13)(R9*1), X12 - MOVO X12, 64(BP) - MOVQ X12, R13 - MOVQ 72(BP), R14 - PXOR X1, X12 - - // We can only store one byte at a time, since plaintext can be shorter than 16 bytes -openSSETail16Store: - MOVQ X12, R8 - MOVB R8, (DI) - PSRLDQ $0x01, X12 - INCQ DI - DECQ BX - JNE openSSETail16Store - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - JMP openSSEFinalize - -openSSETail64: - MOVO ·chacha20Constants<>+0(SB), X0 - MOVO 32(BP), X3 - MOVO 48(BP), X6 - MOVO 128(BP), X9 - PADDL ·sseIncMask<>+0(SB), X9 - MOVO X9, 80(BP) - XORQ R9, R9 - MOVQ BX, CX - CMPQ CX, $0x10 - JB openSSETail64LoopB - -openSSETail64LoopA: - ADDQ (SI)(R9*1), R10 - ADCQ 8(SI)(R9*1), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - SUBQ $0x10, CX - -openSSETail64LoopB: - ADDQ $0x10, R9 - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - CMPQ CX, $0x10 - JAE openSSETail64LoopA - CMPQ R9, $0xa0 - JNE openSSETail64LoopB - PADDL ·chacha20Constants<>+0(SB), X0 - PADDL 32(BP), X3 - PADDL 48(BP), X6 - PADDL 80(BP), X9 - -openSSETail64DecLoop: - CMPQ BX, $0x10 - JB openSSETail64DecLoopDone - SUBQ $0x10, BX - MOVOU (SI), X12 - PXOR X12, X0 - MOVOU X0, (DI) - LEAQ 16(SI), SI - LEAQ 16(DI), DI - MOVO X3, X0 - MOVO X6, X3 - MOVO X9, X6 - JMP openSSETail64DecLoop - -openSSETail64DecLoopDone: - MOVO X0, X1 - JMP openSSETail16 - -openSSETail128: - MOVO ·chacha20Constants<>+0(SB), X1 - MOVO 32(BP), X4 - MOVO 48(BP), X7 - MOVO 128(BP), X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X10, 80(BP) - MOVO X1, X0 - MOVO X4, X3 - MOVO X7, X6 - MOVO X10, X9 - PADDL ·sseIncMask<>+0(SB), X9 - MOVO X9, 96(BP) - XORQ R9, R9 - MOVQ BX, CX - ANDQ $-16, CX - -openSSETail128LoopA: - ADDQ (SI)(R9*1), R10 - ADCQ 8(SI)(R9*1), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - -openSSETail128LoopB: - ADDQ $0x10, R9 - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - CMPQ R9, CX - JB openSSETail128LoopA - CMPQ R9, $0xa0 - JNE openSSETail128LoopB - PADDL ·chacha20Constants<>+0(SB), X0 - PADDL ·chacha20Constants<>+0(SB), X1 - PADDL 32(BP), X3 - PADDL 32(BP), X4 - PADDL 48(BP), X6 - PADDL 48(BP), X7 - PADDL 96(BP), X9 - PADDL 80(BP), X10 - MOVOU (SI), X12 - MOVOU 16(SI), X13 - MOVOU 32(SI), X14 - MOVOU 48(SI), X15 - PXOR X12, X1 - PXOR X13, X4 - PXOR X14, X7 - PXOR X15, X10 - MOVOU X1, (DI) - MOVOU X4, 16(DI) - MOVOU X7, 32(DI) - MOVOU X10, 48(DI) - SUBQ $0x40, BX - LEAQ 64(SI), SI - LEAQ 64(DI), DI - JMP openSSETail64DecLoop - -openSSETail192: - MOVO ·chacha20Constants<>+0(SB), X2 - MOVO 32(BP), X5 - MOVO 48(BP), X8 - MOVO 128(BP), X11 - PADDL ·sseIncMask<>+0(SB), X11 - MOVO X11, 80(BP) - MOVO X2, X1 - MOVO X5, X4 - MOVO X8, X7 - MOVO X11, X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X10, 96(BP) - MOVO X1, X0 - MOVO X4, X3 - MOVO X7, X6 - MOVO X10, X9 - PADDL ·sseIncMask<>+0(SB), X9 - MOVO X9, 112(BP) - MOVQ BX, CX - MOVQ $0x000000a0, R9 - CMPQ CX, $0xa0 - CMOVQGT R9, CX - ANDQ $-16, CX - XORQ R9, R9 - -openSSLTail192LoopA: - ADDQ (SI)(R9*1), R10 - ADCQ 8(SI)(R9*1), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - -openSSLTail192LoopB: - ADDQ $0x10, R9 - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X5 - PXOR X12, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X5 - PXOR X12, X5 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X5 - PXOR X12, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X5 - PXOR X12, X5 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - CMPQ R9, CX - JB openSSLTail192LoopA - CMPQ R9, $0xa0 - JNE openSSLTail192LoopB - CMPQ BX, $0xb0 - JB openSSLTail192Store - ADDQ 160(SI), R10 - ADCQ 168(SI), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - CMPQ BX, $0xc0 - JB openSSLTail192Store - ADDQ 176(SI), R10 - ADCQ 184(SI), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - -openSSLTail192Store: - PADDL ·chacha20Constants<>+0(SB), X0 - PADDL ·chacha20Constants<>+0(SB), X1 - PADDL ·chacha20Constants<>+0(SB), X2 - PADDL 32(BP), X3 - PADDL 32(BP), X4 - PADDL 32(BP), X5 - PADDL 48(BP), X6 - PADDL 48(BP), X7 - PADDL 48(BP), X8 - PADDL 112(BP), X9 - PADDL 96(BP), X10 - PADDL 80(BP), X11 - MOVOU (SI), X12 - MOVOU 16(SI), X13 - MOVOU 32(SI), X14 - MOVOU 48(SI), X15 - PXOR X12, X2 - PXOR X13, X5 - PXOR X14, X8 - PXOR X15, X11 - MOVOU X2, (DI) - MOVOU X5, 16(DI) - MOVOU X8, 32(DI) - MOVOU X11, 48(DI) - MOVOU 64(SI), X12 - MOVOU 80(SI), X13 - MOVOU 96(SI), X14 - MOVOU 112(SI), X15 - PXOR X12, X1 - PXOR X13, X4 - PXOR X14, X7 - PXOR X15, X10 - MOVOU X1, 64(DI) - MOVOU X4, 80(DI) - MOVOU X7, 96(DI) - MOVOU X10, 112(DI) - SUBQ $0x80, BX - LEAQ 128(SI), SI - LEAQ 128(DI), DI - JMP openSSETail64DecLoop - -openSSETail256: - MOVO ·chacha20Constants<>+0(SB), X0 - MOVO 32(BP), X3 - MOVO 48(BP), X6 - MOVO 128(BP), X9 - PADDL ·sseIncMask<>+0(SB), X9 - MOVO X0, X1 - MOVO X3, X4 - MOVO X6, X7 - MOVO X9, X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X1, X2 - MOVO X4, X5 - MOVO X7, X8 - MOVO X10, X11 - PADDL ·sseIncMask<>+0(SB), X11 - MOVO X2, X12 - MOVO X5, X13 - MOVO X8, X14 - MOVO X11, X15 - PADDL ·sseIncMask<>+0(SB), X15 - - // Store counters - MOVO X9, 80(BP) - MOVO X10, 96(BP) - MOVO X11, 112(BP) - MOVO X15, 128(BP) - XORQ R9, R9 - -openSSETail256Loop: - ADDQ (SI)(R9*1), R10 - ADCQ 8(SI)(R9*1), R11 - ADCQ $0x01, R12 - MOVO X14, 64(BP) - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X3 - PXOR X14, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X3 - PXOR X14, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X4 - PXOR X14, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X4 - PXOR X14, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X5 - PXOR X14, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X5 - PXOR X14, X5 - MOVO 64(BP), X14 - MOVO X7, 64(BP) - PADDD X13, X12 - PXOR X12, X15 - ROL16(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x0c, X7 - PSRLL $0x14, X13 - PXOR X7, X13 - PADDD X13, X12 - PXOR X12, X15 - ROL8(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x07, X7 - PSRLL $0x19, X13 - PXOR X7, X13 - MOVO 64(BP), X7 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x0c - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - MOVO X14, 64(BP) - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X3 - PXOR X14, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X3 - PXOR X14, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X4 - PXOR X14, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X4 - PXOR X14, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X5 - PXOR X14, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X5 - PXOR X14, X5 - MOVO 64(BP), X14 - MOVO X7, 64(BP) - PADDD X13, X12 - PXOR X12, X15 - ROL16(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x0c, X7 - PSRLL $0x14, X13 - PXOR X7, X13 - PADDD X13, X12 - PXOR X12, X15 - ROL8(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x07, X7 - PSRLL $0x19, X13 - PXOR X7, X13 - MOVO 64(BP), X7 - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x04 - ADDQ $0x10, R9 - CMPQ R9, $0xa0 - JB openSSETail256Loop - MOVQ BX, CX - ANDQ $-16, CX - -openSSETail256HashLoop: - ADDQ (SI)(R9*1), R10 - ADCQ 8(SI)(R9*1), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - ADDQ $0x10, R9 - CMPQ R9, CX - JB openSSETail256HashLoop - - // Add in the state - PADDD ·chacha20Constants<>+0(SB), X0 - PADDD ·chacha20Constants<>+0(SB), X1 - PADDD ·chacha20Constants<>+0(SB), X2 - PADDD ·chacha20Constants<>+0(SB), X12 - PADDD 32(BP), X3 - PADDD 32(BP), X4 - PADDD 32(BP), X5 - PADDD 32(BP), X13 - PADDD 48(BP), X6 - PADDD 48(BP), X7 - PADDD 48(BP), X8 - PADDD 48(BP), X14 - PADDD 80(BP), X9 - PADDD 96(BP), X10 - PADDD 112(BP), X11 - PADDD 128(BP), X15 - MOVO X15, 64(BP) - - // Load - xor - store - MOVOU (SI), X15 - PXOR X15, X0 - MOVOU 16(SI), X15 - PXOR X15, X3 - MOVOU 32(SI), X15 - PXOR X15, X6 - MOVOU 48(SI), X15 - PXOR X15, X9 - MOVOU X0, (DI) - MOVOU X3, 16(DI) - MOVOU X6, 32(DI) - MOVOU X9, 48(DI) - MOVOU 64(SI), X0 - MOVOU 80(SI), X3 - MOVOU 96(SI), X6 - MOVOU 112(SI), X9 - PXOR X0, X1 - PXOR X3, X4 - PXOR X6, X7 - PXOR X9, X10 - MOVOU X1, 64(DI) - MOVOU X4, 80(DI) - MOVOU X7, 96(DI) - MOVOU X10, 112(DI) - MOVOU 128(SI), X0 - MOVOU 144(SI), X3 - MOVOU 160(SI), X6 - MOVOU 176(SI), X9 - PXOR X0, X2 - PXOR X3, X5 - PXOR X6, X8 - PXOR X9, X11 - MOVOU X2, 128(DI) - MOVOU X5, 144(DI) - MOVOU X8, 160(DI) - MOVOU X11, 176(DI) - LEAQ 192(SI), SI - LEAQ 192(DI), DI - SUBQ $0xc0, BX - MOVO X12, X0 - MOVO X13, X3 - MOVO X14, X6 - MOVO 64(BP), X9 - JMP openSSETail64DecLoop - -chacha20Poly1305Open_AVX2: - VZEROUPPER - VMOVDQU ·chacha20Constants<>+0(SB), Y0 - BYTE $0xc4 - BYTE $0x42 - BYTE $0x7d - BYTE $0x5a - BYTE $0x70 - BYTE $0x10 - BYTE $0xc4 - BYTE $0x42 - BYTE $0x7d - BYTE $0x5a - BYTE $0x60 - BYTE $0x20 - BYTE $0xc4 - BYTE $0xc2 - BYTE $0x7d - BYTE $0x5a - BYTE $0x60 - BYTE $0x30 - VPADDD ·avx2InitMask<>+0(SB), Y4, Y4 - - // Special optimization, for very short buffers - CMPQ BX, $0xc0 - JBE openAVX2192 - CMPQ BX, $0x00000140 - JBE openAVX2320 - - // For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream - VMOVDQA Y14, 32(BP) - VMOVDQA Y12, 64(BP) - VMOVDQA Y4, 192(BP) - MOVQ $0x0000000a, R9 - -openAVX2PreparePolyKey: - VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol16<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x0c, Y14, Y3 - VPSRLD $0x14, Y14, Y14 - VPXOR Y3, Y14, Y14 - VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol8<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x07, Y14, Y3 - VPSRLD $0x19, Y14, Y14 - VPXOR Y3, Y14, Y14 - VPALIGNR $0x04, Y14, Y14, Y14 - VPALIGNR $0x08, Y12, Y12, Y12 - VPALIGNR $0x0c, Y4, Y4, Y4 - VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol16<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x0c, Y14, Y3 - VPSRLD $0x14, Y14, Y14 - VPXOR Y3, Y14, Y14 - VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol8<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x07, Y14, Y3 - VPSRLD $0x19, Y14, Y14 - VPXOR Y3, Y14, Y14 - VPALIGNR $0x0c, Y14, Y14, Y14 - VPALIGNR $0x08, Y12, Y12, Y12 - VPALIGNR $0x04, Y4, Y4, Y4 - DECQ R9 - JNE openAVX2PreparePolyKey - VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 - VPADDD 32(BP), Y14, Y14 - VPADDD 64(BP), Y12, Y12 - VPADDD 192(BP), Y4, Y4 - VPERM2I128 $0x02, Y0, Y14, Y3 - - // Clamp and store poly key - VPAND ·polyClampMask<>+0(SB), Y3, Y3 - VMOVDQA Y3, (BP) - - // Stream for the first 64 bytes - VPERM2I128 $0x13, Y0, Y14, Y0 - VPERM2I128 $0x13, Y12, Y4, Y14 - - // Hash AD + first 64 bytes - MOVQ ad_len+80(FP), R9 - CALL polyHashADInternal<>(SB) - XORQ CX, CX - -openAVX2InitialHash64: - ADDQ (SI)(CX*1), R10 - ADCQ 8(SI)(CX*1), R11 - ADCQ $0x01, R12 - MOVQ (BP), DX - MOVQ DX, R15 - MULXQ R10, R13, R14 - IMULQ R12, R15 - MULXQ R11, AX, DX - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), DX - MULXQ R10, R10, AX - ADDQ R10, R14 - MULXQ R11, R11, R8 - ADCQ R11, R15 - ADCQ $0x00, R8 - IMULQ R12, DX - ADDQ AX, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - ADDQ $0x10, CX - CMPQ CX, $0x40 - JNE openAVX2InitialHash64 - - // Decrypt the first 64 bytes - VPXOR (SI), Y0, Y0 - VPXOR 32(SI), Y14, Y14 - VMOVDQU Y0, (DI) - VMOVDQU Y14, 32(DI) - LEAQ 64(SI), SI - LEAQ 64(DI), DI - SUBQ $0x40, BX - -openAVX2MainLoop: - CMPQ BX, $0x00000200 - JB openAVX2MainLoopDone - - // Load state, increment counter blocks, store the incremented counters - VMOVDQU ·chacha20Constants<>+0(SB), Y0 - VMOVDQA Y0, Y5 - VMOVDQA Y0, Y6 - VMOVDQA Y0, Y7 - VMOVDQA 32(BP), Y14 - VMOVDQA Y14, Y9 - VMOVDQA Y14, Y10 - VMOVDQA Y14, Y11 - VMOVDQA 64(BP), Y12 - VMOVDQA Y12, Y13 - VMOVDQA Y12, Y8 - VMOVDQA Y12, Y15 - VMOVDQA 192(BP), Y4 - VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 - VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 - VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 - VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 - VMOVDQA Y4, 96(BP) - VMOVDQA Y1, 128(BP) - VMOVDQA Y2, 160(BP) - VMOVDQA Y3, 192(BP) - XORQ CX, CX - -openAVX2InternalLoop: - ADDQ (SI)(CX*1), R10 - ADCQ 8(SI)(CX*1), R11 - ADCQ $0x01, R12 - VPADDD Y14, Y0, Y0 - VPADDD Y9, Y5, Y5 - VPADDD Y10, Y6, Y6 - VPADDD Y11, Y7, Y7 - MOVQ (BP), DX - MOVQ DX, R15 - MULXQ R10, R13, R14 - IMULQ R12, R15 - MULXQ R11, AX, DX - ADDQ AX, R14 - ADCQ DX, R15 - VPXOR Y0, Y4, Y4 - VPXOR Y5, Y1, Y1 - VPXOR Y6, Y2, Y2 - VPXOR Y7, Y3, Y3 - VPSHUFB ·rol16<>+0(SB), Y4, Y4 - VPSHUFB ·rol16<>+0(SB), Y1, Y1 - VPSHUFB ·rol16<>+0(SB), Y2, Y2 - VPSHUFB ·rol16<>+0(SB), Y3, Y3 - MOVQ 8(BP), DX - MULXQ R10, R10, AX - ADDQ R10, R14 - MULXQ R11, R11, R8 - ADCQ R11, R15 - ADCQ $0x00, R8 - VPADDD Y4, Y12, Y12 - VPADDD Y1, Y13, Y13 - VPADDD Y2, Y8, Y8 - VPADDD Y3, Y15, Y15 - VPXOR Y12, Y14, Y14 - VPXOR Y13, Y9, Y9 - VPXOR Y8, Y10, Y10 - VPXOR Y15, Y11, Y11 - IMULQ R12, DX - ADDQ AX, R15 - ADCQ DX, R8 - VMOVDQA Y15, 224(BP) - VPSLLD $0x0c, Y14, Y15 - VPSRLD $0x14, Y14, Y14 - VPXOR Y15, Y14, Y14 - VPSLLD $0x0c, Y9, Y15 - VPSRLD $0x14, Y9, Y9 - VPXOR Y15, Y9, Y9 - VPSLLD $0x0c, Y10, Y15 - VPSRLD $0x14, Y10, Y10 - VPXOR Y15, Y10, Y10 - VPSLLD $0x0c, Y11, Y15 - VPSRLD $0x14, Y11, Y11 - VPXOR Y15, Y11, Y11 - VMOVDQA 224(BP), Y15 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - VPADDD Y14, Y0, Y0 - VPADDD Y9, Y5, Y5 - VPADDD Y10, Y6, Y6 - VPADDD Y11, Y7, Y7 - VPXOR Y0, Y4, Y4 - VPXOR Y5, Y1, Y1 - VPXOR Y6, Y2, Y2 - VPXOR Y7, Y3, Y3 - VPSHUFB ·rol8<>+0(SB), Y4, Y4 - VPSHUFB ·rol8<>+0(SB), Y1, Y1 - VPSHUFB ·rol8<>+0(SB), Y2, Y2 - VPSHUFB ·rol8<>+0(SB), Y3, Y3 - ADDQ 16(SI)(CX*1), R10 - ADCQ 24(SI)(CX*1), R11 - ADCQ $0x01, R12 - VPADDD Y4, Y12, Y12 - VPADDD Y1, Y13, Y13 - VPADDD Y2, Y8, Y8 - VPADDD Y3, Y15, Y15 - MOVQ (BP), DX - MOVQ DX, R15 - MULXQ R10, R13, R14 - IMULQ R12, R15 - MULXQ R11, AX, DX - ADDQ AX, R14 - ADCQ DX, R15 - VPXOR Y12, Y14, Y14 - VPXOR Y13, Y9, Y9 - VPXOR Y8, Y10, Y10 - VPXOR Y15, Y11, Y11 - VMOVDQA Y15, 224(BP) - VPSLLD $0x07, Y14, Y15 - VPSRLD $0x19, Y14, Y14 - VPXOR Y15, Y14, Y14 - VPSLLD $0x07, Y9, Y15 - VPSRLD $0x19, Y9, Y9 - VPXOR Y15, Y9, Y9 - VPSLLD $0x07, Y10, Y15 - VPSRLD $0x19, Y10, Y10 - VPXOR Y15, Y10, Y10 - VPSLLD $0x07, Y11, Y15 - VPSRLD $0x19, Y11, Y11 - VPXOR Y15, Y11, Y11 - VMOVDQA 224(BP), Y15 - MOVQ 8(BP), DX - MULXQ R10, R10, AX - ADDQ R10, R14 - MULXQ R11, R11, R8 - ADCQ R11, R15 - ADCQ $0x00, R8 - VPALIGNR $0x04, Y14, Y14, Y14 - VPALIGNR $0x04, Y9, Y9, Y9 - VPALIGNR $0x04, Y10, Y10, Y10 - VPALIGNR $0x04, Y11, Y11, Y11 - VPALIGNR $0x08, Y12, Y12, Y12 - VPALIGNR $0x08, Y13, Y13, Y13 - VPALIGNR $0x08, Y8, Y8, Y8 - VPALIGNR $0x08, Y15, Y15, Y15 - VPALIGNR $0x0c, Y4, Y4, Y4 - VPALIGNR $0x0c, Y1, Y1, Y1 - VPALIGNR $0x0c, Y2, Y2, Y2 - VPALIGNR $0x0c, Y3, Y3, Y3 - VPADDD Y14, Y0, Y0 - VPADDD Y9, Y5, Y5 - VPADDD Y10, Y6, Y6 - VPADDD Y11, Y7, Y7 - IMULQ R12, DX - ADDQ AX, R15 - ADCQ DX, R8 - VPXOR Y0, Y4, Y4 - VPXOR Y5, Y1, Y1 - VPXOR Y6, Y2, Y2 - VPXOR Y7, Y3, Y3 - VPSHUFB ·rol16<>+0(SB), Y4, Y4 - VPSHUFB ·rol16<>+0(SB), Y1, Y1 - VPSHUFB ·rol16<>+0(SB), Y2, Y2 - VPSHUFB ·rol16<>+0(SB), Y3, Y3 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - VPADDD Y4, Y12, Y12 - VPADDD Y1, Y13, Y13 - VPADDD Y2, Y8, Y8 - VPADDD Y3, Y15, Y15 - VPXOR Y12, Y14, Y14 - VPXOR Y13, Y9, Y9 - VPXOR Y8, Y10, Y10 - VPXOR Y15, Y11, Y11 - ADDQ 32(SI)(CX*1), R10 - ADCQ 40(SI)(CX*1), R11 - ADCQ $0x01, R12 - LEAQ 48(CX), CX - VMOVDQA Y15, 224(BP) - VPSLLD $0x0c, Y14, Y15 - VPSRLD $0x14, Y14, Y14 - VPXOR Y15, Y14, Y14 - VPSLLD $0x0c, Y9, Y15 - VPSRLD $0x14, Y9, Y9 - VPXOR Y15, Y9, Y9 - VPSLLD $0x0c, Y10, Y15 - VPSRLD $0x14, Y10, Y10 - VPXOR Y15, Y10, Y10 - VPSLLD $0x0c, Y11, Y15 - VPSRLD $0x14, Y11, Y11 - VPXOR Y15, Y11, Y11 - VMOVDQA 224(BP), Y15 - MOVQ (BP), DX - MOVQ DX, R15 - MULXQ R10, R13, R14 - IMULQ R12, R15 - MULXQ R11, AX, DX - ADDQ AX, R14 - ADCQ DX, R15 - VPADDD Y14, Y0, Y0 - VPADDD Y9, Y5, Y5 - VPADDD Y10, Y6, Y6 - VPADDD Y11, Y7, Y7 - VPXOR Y0, Y4, Y4 - VPXOR Y5, Y1, Y1 - VPXOR Y6, Y2, Y2 - VPXOR Y7, Y3, Y3 - MOVQ 8(BP), DX - MULXQ R10, R10, AX - ADDQ R10, R14 - MULXQ R11, R11, R8 - ADCQ R11, R15 - ADCQ $0x00, R8 - VPSHUFB ·rol8<>+0(SB), Y4, Y4 - VPSHUFB ·rol8<>+0(SB), Y1, Y1 - VPSHUFB ·rol8<>+0(SB), Y2, Y2 - VPSHUFB ·rol8<>+0(SB), Y3, Y3 - VPADDD Y4, Y12, Y12 - VPADDD Y1, Y13, Y13 - VPADDD Y2, Y8, Y8 - VPADDD Y3, Y15, Y15 - IMULQ R12, DX - ADDQ AX, R15 - ADCQ DX, R8 - VPXOR Y12, Y14, Y14 - VPXOR Y13, Y9, Y9 - VPXOR Y8, Y10, Y10 - VPXOR Y15, Y11, Y11 - VMOVDQA Y15, 224(BP) - VPSLLD $0x07, Y14, Y15 - VPSRLD $0x19, Y14, Y14 - VPXOR Y15, Y14, Y14 - VPSLLD $0x07, Y9, Y15 - VPSRLD $0x19, Y9, Y9 - VPXOR Y15, Y9, Y9 - VPSLLD $0x07, Y10, Y15 - VPSRLD $0x19, Y10, Y10 - VPXOR Y15, Y10, Y10 - VPSLLD $0x07, Y11, Y15 - VPSRLD $0x19, Y11, Y11 - VPXOR Y15, Y11, Y11 - VMOVDQA 224(BP), Y15 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - VPALIGNR $0x0c, Y14, Y14, Y14 - VPALIGNR $0x0c, Y9, Y9, Y9 - VPALIGNR $0x0c, Y10, Y10, Y10 - VPALIGNR $0x0c, Y11, Y11, Y11 - VPALIGNR $0x08, Y12, Y12, Y12 - VPALIGNR $0x08, Y13, Y13, Y13 - VPALIGNR $0x08, Y8, Y8, Y8 - VPALIGNR $0x08, Y15, Y15, Y15 - VPALIGNR $0x04, Y4, Y4, Y4 - VPALIGNR $0x04, Y1, Y1, Y1 - VPALIGNR $0x04, Y2, Y2, Y2 - VPALIGNR $0x04, Y3, Y3, Y3 - CMPQ CX, $0x000001e0 - JNE openAVX2InternalLoop - VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 - VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 - VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 - VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 - VPADDD 32(BP), Y14, Y14 - VPADDD 32(BP), Y9, Y9 - VPADDD 32(BP), Y10, Y10 - VPADDD 32(BP), Y11, Y11 - VPADDD 64(BP), Y12, Y12 - VPADDD 64(BP), Y13, Y13 - VPADDD 64(BP), Y8, Y8 - VPADDD 64(BP), Y15, Y15 - VPADDD 96(BP), Y4, Y4 - VPADDD 128(BP), Y1, Y1 - VPADDD 160(BP), Y2, Y2 - VPADDD 192(BP), Y3, Y3 - VMOVDQA Y15, 224(BP) - - // We only hashed 480 of the 512 bytes available - hash the remaining 32 here - ADDQ 480(SI), R10 - ADCQ 488(SI), R11 - ADCQ $0x01, R12 - MOVQ (BP), DX - MOVQ DX, R15 - MULXQ R10, R13, R14 - IMULQ R12, R15 - MULXQ R11, AX, DX - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), DX - MULXQ R10, R10, AX - ADDQ R10, R14 - MULXQ R11, R11, R8 - ADCQ R11, R15 - ADCQ $0x00, R8 - IMULQ R12, DX - ADDQ AX, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - VPERM2I128 $0x02, Y0, Y14, Y15 - VPERM2I128 $0x13, Y0, Y14, Y14 - VPERM2I128 $0x02, Y12, Y4, Y0 - VPERM2I128 $0x13, Y12, Y4, Y12 - VPXOR (SI), Y15, Y15 - VPXOR 32(SI), Y0, Y0 - VPXOR 64(SI), Y14, Y14 - VPXOR 96(SI), Y12, Y12 - VMOVDQU Y15, (DI) - VMOVDQU Y0, 32(DI) - VMOVDQU Y14, 64(DI) - VMOVDQU Y12, 96(DI) - VPERM2I128 $0x02, Y5, Y9, Y0 - VPERM2I128 $0x02, Y13, Y1, Y14 - VPERM2I128 $0x13, Y5, Y9, Y12 - VPERM2I128 $0x13, Y13, Y1, Y4 - VPXOR 128(SI), Y0, Y0 - VPXOR 160(SI), Y14, Y14 - VPXOR 192(SI), Y12, Y12 - VPXOR 224(SI), Y4, Y4 - VMOVDQU Y0, 128(DI) - VMOVDQU Y14, 160(DI) - VMOVDQU Y12, 192(DI) - VMOVDQU Y4, 224(DI) - - // and here - ADDQ 496(SI), R10 - ADCQ 504(SI), R11 - ADCQ $0x01, R12 - MOVQ (BP), DX - MOVQ DX, R15 - MULXQ R10, R13, R14 - IMULQ R12, R15 - MULXQ R11, AX, DX - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), DX - MULXQ R10, R10, AX - ADDQ R10, R14 - MULXQ R11, R11, R8 - ADCQ R11, R15 - ADCQ $0x00, R8 - IMULQ R12, DX - ADDQ AX, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - VPERM2I128 $0x02, Y6, Y10, Y0 - VPERM2I128 $0x02, Y8, Y2, Y14 - VPERM2I128 $0x13, Y6, Y10, Y12 - VPERM2I128 $0x13, Y8, Y2, Y4 - VPXOR 256(SI), Y0, Y0 - VPXOR 288(SI), Y14, Y14 - VPXOR 320(SI), Y12, Y12 - VPXOR 352(SI), Y4, Y4 - VMOVDQU Y0, 256(DI) - VMOVDQU Y14, 288(DI) - VMOVDQU Y12, 320(DI) - VMOVDQU Y4, 352(DI) - VPERM2I128 $0x02, Y7, Y11, Y0 - VPERM2I128 $0x02, 224(BP), Y3, Y14 - VPERM2I128 $0x13, Y7, Y11, Y12 - VPERM2I128 $0x13, 224(BP), Y3, Y4 - VPXOR 384(SI), Y0, Y0 - VPXOR 416(SI), Y14, Y14 - VPXOR 448(SI), Y12, Y12 - VPXOR 480(SI), Y4, Y4 - VMOVDQU Y0, 384(DI) - VMOVDQU Y14, 416(DI) - VMOVDQU Y12, 448(DI) - VMOVDQU Y4, 480(DI) - LEAQ 512(SI), SI - LEAQ 512(DI), DI - SUBQ $0x00000200, BX - JMP openAVX2MainLoop - -openAVX2MainLoopDone: - // Handle the various tail sizes efficiently - TESTQ BX, BX - JE openSSEFinalize - CMPQ BX, $0x80 - JBE openAVX2Tail128 - CMPQ BX, $0x00000100 - JBE openAVX2Tail256 - CMPQ BX, $0x00000180 - JBE openAVX2Tail384 - JMP openAVX2Tail512 - -openAVX2192: - VMOVDQA Y0, Y5 - VMOVDQA Y14, Y9 - VMOVDQA Y12, Y13 - VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 - VMOVDQA Y0, Y6 - VMOVDQA Y14, Y10 - VMOVDQA Y12, Y8 - VMOVDQA Y4, Y2 - VMOVDQA Y1, Y15 - MOVQ $0x0000000a, R9 - -openAVX2192InnerCipherLoop: - VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol16<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x0c, Y14, Y3 - VPSRLD $0x14, Y14, Y14 - VPXOR Y3, Y14, Y14 - VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol8<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x07, Y14, Y3 - VPSRLD $0x19, Y14, Y14 - VPXOR Y3, Y14, Y14 - VPADDD Y9, Y5, Y5 - VPXOR Y5, Y1, Y1 - VPSHUFB ·rol16<>+0(SB), Y1, Y1 - VPADDD Y1, Y13, Y13 - VPXOR Y13, Y9, Y9 - VPSLLD $0x0c, Y9, Y3 - VPSRLD $0x14, Y9, Y9 - VPXOR Y3, Y9, Y9 - VPADDD Y9, Y5, Y5 - VPXOR Y5, Y1, Y1 - VPSHUFB ·rol8<>+0(SB), Y1, Y1 - VPADDD Y1, Y13, Y13 - VPXOR Y13, Y9, Y9 - VPSLLD $0x07, Y9, Y3 - VPSRLD $0x19, Y9, Y9 - VPXOR Y3, Y9, Y9 - VPALIGNR $0x04, Y14, Y14, Y14 - VPALIGNR $0x04, Y9, Y9, Y9 - VPALIGNR $0x08, Y12, Y12, Y12 - VPALIGNR $0x08, Y13, Y13, Y13 - VPALIGNR $0x0c, Y4, Y4, Y4 - VPALIGNR $0x0c, Y1, Y1, Y1 - VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol16<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x0c, Y14, Y3 - VPSRLD $0x14, Y14, Y14 - VPXOR Y3, Y14, Y14 - VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol8<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x07, Y14, Y3 - VPSRLD $0x19, Y14, Y14 - VPXOR Y3, Y14, Y14 - VPADDD Y9, Y5, Y5 - VPXOR Y5, Y1, Y1 - VPSHUFB ·rol16<>+0(SB), Y1, Y1 - VPADDD Y1, Y13, Y13 - VPXOR Y13, Y9, Y9 - VPSLLD $0x0c, Y9, Y3 - VPSRLD $0x14, Y9, Y9 - VPXOR Y3, Y9, Y9 - VPADDD Y9, Y5, Y5 - VPXOR Y5, Y1, Y1 - VPSHUFB ·rol8<>+0(SB), Y1, Y1 - VPADDD Y1, Y13, Y13 - VPXOR Y13, Y9, Y9 - VPSLLD $0x07, Y9, Y3 - VPSRLD $0x19, Y9, Y9 - VPXOR Y3, Y9, Y9 - VPALIGNR $0x0c, Y14, Y14, Y14 - VPALIGNR $0x0c, Y9, Y9, Y9 - VPALIGNR $0x08, Y12, Y12, Y12 - VPALIGNR $0x08, Y13, Y13, Y13 - VPALIGNR $0x04, Y4, Y4, Y4 - VPALIGNR $0x04, Y1, Y1, Y1 - DECQ R9 - JNE openAVX2192InnerCipherLoop - VPADDD Y6, Y0, Y0 - VPADDD Y6, Y5, Y5 - VPADDD Y10, Y14, Y14 - VPADDD Y10, Y9, Y9 - VPADDD Y8, Y12, Y12 - VPADDD Y8, Y13, Y13 - VPADDD Y2, Y4, Y4 - VPADDD Y15, Y1, Y1 - VPERM2I128 $0x02, Y0, Y14, Y3 - - // Clamp and store poly key - VPAND ·polyClampMask<>+0(SB), Y3, Y3 - VMOVDQA Y3, (BP) - - // Stream for up to 192 bytes - VPERM2I128 $0x13, Y0, Y14, Y0 - VPERM2I128 $0x13, Y12, Y4, Y14 - VPERM2I128 $0x02, Y5, Y9, Y12 - VPERM2I128 $0x02, Y13, Y1, Y4 - VPERM2I128 $0x13, Y5, Y9, Y5 - VPERM2I128 $0x13, Y13, Y1, Y9 - -openAVX2ShortOpen: - // Hash - MOVQ ad_len+80(FP), R9 - CALL polyHashADInternal<>(SB) - -openAVX2ShortOpenLoop: - CMPQ BX, $0x20 - JB openAVX2ShortTail32 - SUBQ $0x20, BX - - // Load for hashing - ADDQ (SI), R10 - ADCQ 8(SI), R11 - ADCQ $0x01, R12 - MOVQ (BP), DX - MOVQ DX, R15 - MULXQ R10, R13, R14 - IMULQ R12, R15 - MULXQ R11, AX, DX - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), DX - MULXQ R10, R10, AX - ADDQ R10, R14 - MULXQ R11, R11, R8 - ADCQ R11, R15 - ADCQ $0x00, R8 - IMULQ R12, DX - ADDQ AX, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - ADDQ 16(SI), R10 - ADCQ 24(SI), R11 - ADCQ $0x01, R12 - MOVQ (BP), DX - MOVQ DX, R15 - MULXQ R10, R13, R14 - IMULQ R12, R15 - MULXQ R11, AX, DX - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), DX - MULXQ R10, R10, AX - ADDQ R10, R14 - MULXQ R11, R11, R8 - ADCQ R11, R15 - ADCQ $0x00, R8 - IMULQ R12, DX - ADDQ AX, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - - // Load for decryption - VPXOR (SI), Y0, Y0 - VMOVDQU Y0, (DI) - LEAQ 32(SI), SI - LEAQ 32(DI), DI - - // Shift stream left - VMOVDQA Y14, Y0 - VMOVDQA Y12, Y14 - VMOVDQA Y4, Y12 - VMOVDQA Y5, Y4 - VMOVDQA Y9, Y5 - VMOVDQA Y13, Y9 - VMOVDQA Y1, Y13 - VMOVDQA Y6, Y1 - VMOVDQA Y10, Y6 - JMP openAVX2ShortOpenLoop - -openAVX2ShortTail32: - CMPQ BX, $0x10 - VMOVDQA X0, X1 - JB openAVX2ShortDone - SUBQ $0x10, BX - - // Load for hashing - ADDQ (SI), R10 - ADCQ 8(SI), R11 - ADCQ $0x01, R12 - MOVQ (BP), DX - MOVQ DX, R15 - MULXQ R10, R13, R14 - IMULQ R12, R15 - MULXQ R11, AX, DX - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), DX - MULXQ R10, R10, AX - ADDQ R10, R14 - MULXQ R11, R11, R8 - ADCQ R11, R15 - ADCQ $0x00, R8 - IMULQ R12, DX - ADDQ AX, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - - // Load for decryption - VPXOR (SI), X0, X12 - VMOVDQU X12, (DI) - LEAQ 16(SI), SI - LEAQ 16(DI), DI - VPERM2I128 $0x11, Y0, Y0, Y0 - VMOVDQA X0, X1 - -openAVX2ShortDone: - VZEROUPPER - JMP openSSETail16 - -openAVX2320: - VMOVDQA Y0, Y5 - VMOVDQA Y14, Y9 - VMOVDQA Y12, Y13 - VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 - VMOVDQA Y0, Y6 - VMOVDQA Y14, Y10 - VMOVDQA Y12, Y8 - VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 - VMOVDQA Y14, Y7 - VMOVDQA Y12, Y11 - VMOVDQA Y4, Y15 - MOVQ $0x0000000a, R9 - -openAVX2320InnerCipherLoop: - VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol16<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x0c, Y14, Y3 - VPSRLD $0x14, Y14, Y14 - VPXOR Y3, Y14, Y14 - VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol8<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x07, Y14, Y3 - VPSRLD $0x19, Y14, Y14 - VPXOR Y3, Y14, Y14 - VPADDD Y9, Y5, Y5 - VPXOR Y5, Y1, Y1 - VPSHUFB ·rol16<>+0(SB), Y1, Y1 - VPADDD Y1, Y13, Y13 - VPXOR Y13, Y9, Y9 - VPSLLD $0x0c, Y9, Y3 - VPSRLD $0x14, Y9, Y9 - VPXOR Y3, Y9, Y9 - VPADDD Y9, Y5, Y5 - VPXOR Y5, Y1, Y1 - VPSHUFB ·rol8<>+0(SB), Y1, Y1 - VPADDD Y1, Y13, Y13 - VPXOR Y13, Y9, Y9 - VPSLLD $0x07, Y9, Y3 - VPSRLD $0x19, Y9, Y9 - VPXOR Y3, Y9, Y9 - VPADDD Y10, Y6, Y6 - VPXOR Y6, Y2, Y2 - VPSHUFB ·rol16<>+0(SB), Y2, Y2 - VPADDD Y2, Y8, Y8 - VPXOR Y8, Y10, Y10 - VPSLLD $0x0c, Y10, Y3 - VPSRLD $0x14, Y10, Y10 - VPXOR Y3, Y10, Y10 - VPADDD Y10, Y6, Y6 - VPXOR Y6, Y2, Y2 - VPSHUFB ·rol8<>+0(SB), Y2, Y2 - VPADDD Y2, Y8, Y8 - VPXOR Y8, Y10, Y10 - VPSLLD $0x07, Y10, Y3 - VPSRLD $0x19, Y10, Y10 - VPXOR Y3, Y10, Y10 - VPALIGNR $0x04, Y14, Y14, Y14 - VPALIGNR $0x04, Y9, Y9, Y9 - VPALIGNR $0x04, Y10, Y10, Y10 - VPALIGNR $0x08, Y12, Y12, Y12 - VPALIGNR $0x08, Y13, Y13, Y13 - VPALIGNR $0x08, Y8, Y8, Y8 - VPALIGNR $0x0c, Y4, Y4, Y4 - VPALIGNR $0x0c, Y1, Y1, Y1 - VPALIGNR $0x0c, Y2, Y2, Y2 - VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol16<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x0c, Y14, Y3 - VPSRLD $0x14, Y14, Y14 - VPXOR Y3, Y14, Y14 - VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol8<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x07, Y14, Y3 - VPSRLD $0x19, Y14, Y14 - VPXOR Y3, Y14, Y14 - VPADDD Y9, Y5, Y5 - VPXOR Y5, Y1, Y1 - VPSHUFB ·rol16<>+0(SB), Y1, Y1 - VPADDD Y1, Y13, Y13 - VPXOR Y13, Y9, Y9 - VPSLLD $0x0c, Y9, Y3 - VPSRLD $0x14, Y9, Y9 - VPXOR Y3, Y9, Y9 - VPADDD Y9, Y5, Y5 - VPXOR Y5, Y1, Y1 - VPSHUFB ·rol8<>+0(SB), Y1, Y1 - VPADDD Y1, Y13, Y13 - VPXOR Y13, Y9, Y9 - VPSLLD $0x07, Y9, Y3 - VPSRLD $0x19, Y9, Y9 - VPXOR Y3, Y9, Y9 - VPADDD Y10, Y6, Y6 - VPXOR Y6, Y2, Y2 - VPSHUFB ·rol16<>+0(SB), Y2, Y2 - VPADDD Y2, Y8, Y8 - VPXOR Y8, Y10, Y10 - VPSLLD $0x0c, Y10, Y3 - VPSRLD $0x14, Y10, Y10 - VPXOR Y3, Y10, Y10 - VPADDD Y10, Y6, Y6 - VPXOR Y6, Y2, Y2 - VPSHUFB ·rol8<>+0(SB), Y2, Y2 - VPADDD Y2, Y8, Y8 - VPXOR Y8, Y10, Y10 - VPSLLD $0x07, Y10, Y3 - VPSRLD $0x19, Y10, Y10 - VPXOR Y3, Y10, Y10 - VPALIGNR $0x0c, Y14, Y14, Y14 - VPALIGNR $0x0c, Y9, Y9, Y9 - VPALIGNR $0x0c, Y10, Y10, Y10 - VPALIGNR $0x08, Y12, Y12, Y12 - VPALIGNR $0x08, Y13, Y13, Y13 - VPALIGNR $0x08, Y8, Y8, Y8 - VPALIGNR $0x04, Y4, Y4, Y4 - VPALIGNR $0x04, Y1, Y1, Y1 - VPALIGNR $0x04, Y2, Y2, Y2 - DECQ R9 - JNE openAVX2320InnerCipherLoop - VMOVDQA ·chacha20Constants<>+0(SB), Y3 - VPADDD Y3, Y0, Y0 - VPADDD Y3, Y5, Y5 - VPADDD Y3, Y6, Y6 - VPADDD Y7, Y14, Y14 - VPADDD Y7, Y9, Y9 - VPADDD Y7, Y10, Y10 - VPADDD Y11, Y12, Y12 - VPADDD Y11, Y13, Y13 - VPADDD Y11, Y8, Y8 - VMOVDQA ·avx2IncMask<>+0(SB), Y3 - VPADDD Y15, Y4, Y4 - VPADDD Y3, Y15, Y15 - VPADDD Y15, Y1, Y1 - VPADDD Y3, Y15, Y15 - VPADDD Y15, Y2, Y2 - - // Clamp and store poly key - VPERM2I128 $0x02, Y0, Y14, Y3 - VPAND ·polyClampMask<>+0(SB), Y3, Y3 - VMOVDQA Y3, (BP) - - // Stream for up to 320 bytes - VPERM2I128 $0x13, Y0, Y14, Y0 - VPERM2I128 $0x13, Y12, Y4, Y14 - VPERM2I128 $0x02, Y5, Y9, Y12 - VPERM2I128 $0x02, Y13, Y1, Y4 - VPERM2I128 $0x13, Y5, Y9, Y5 - VPERM2I128 $0x13, Y13, Y1, Y9 - VPERM2I128 $0x02, Y6, Y10, Y13 - VPERM2I128 $0x02, Y8, Y2, Y1 - VPERM2I128 $0x13, Y6, Y10, Y6 - VPERM2I128 $0x13, Y8, Y2, Y10 - JMP openAVX2ShortOpen - -openAVX2Tail128: - // Need to decrypt up to 128 bytes - prepare two blocks - VMOVDQA ·chacha20Constants<>+0(SB), Y5 - VMOVDQA 32(BP), Y9 - VMOVDQA 64(BP), Y13 - VMOVDQA 192(BP), Y1 - VPADDD ·avx2IncMask<>+0(SB), Y1, Y1 - VMOVDQA Y1, Y4 - XORQ R9, R9 - MOVQ BX, CX - ANDQ $-16, CX - TESTQ CX, CX - JE openAVX2Tail128LoopB - -openAVX2Tail128LoopA: - ADDQ (SI)(R9*1), R10 - ADCQ 8(SI)(R9*1), R11 - ADCQ $0x01, R12 - MOVQ (BP), DX - MOVQ DX, R15 - MULXQ R10, R13, R14 - IMULQ R12, R15 - MULXQ R11, AX, DX - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), DX - MULXQ R10, R10, AX - ADDQ R10, R14 - MULXQ R11, R11, R8 - ADCQ R11, R15 - ADCQ $0x00, R8 - IMULQ R12, DX - ADDQ AX, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - -openAVX2Tail128LoopB: - ADDQ $0x10, R9 - VPADDD Y9, Y5, Y5 - VPXOR Y5, Y1, Y1 - VPSHUFB ·rol16<>+0(SB), Y1, Y1 - VPADDD Y1, Y13, Y13 - VPXOR Y13, Y9, Y9 - VPSLLD $0x0c, Y9, Y3 - VPSRLD $0x14, Y9, Y9 - VPXOR Y3, Y9, Y9 - VPADDD Y9, Y5, Y5 - VPXOR Y5, Y1, Y1 - VPSHUFB ·rol8<>+0(SB), Y1, Y1 - VPADDD Y1, Y13, Y13 - VPXOR Y13, Y9, Y9 - VPSLLD $0x07, Y9, Y3 - VPSRLD $0x19, Y9, Y9 - VPXOR Y3, Y9, Y9 - VPALIGNR $0x04, Y9, Y9, Y9 - VPALIGNR $0x08, Y13, Y13, Y13 - VPALIGNR $0x0c, Y1, Y1, Y1 - VPADDD Y9, Y5, Y5 - VPXOR Y5, Y1, Y1 - VPSHUFB ·rol16<>+0(SB), Y1, Y1 - VPADDD Y1, Y13, Y13 - VPXOR Y13, Y9, Y9 - VPSLLD $0x0c, Y9, Y3 - VPSRLD $0x14, Y9, Y9 - VPXOR Y3, Y9, Y9 - VPADDD Y9, Y5, Y5 - VPXOR Y5, Y1, Y1 - VPSHUFB ·rol8<>+0(SB), Y1, Y1 - VPADDD Y1, Y13, Y13 - VPXOR Y13, Y9, Y9 - VPSLLD $0x07, Y9, Y3 - VPSRLD $0x19, Y9, Y9 - VPXOR Y3, Y9, Y9 - VPALIGNR $0x0c, Y9, Y9, Y9 - VPALIGNR $0x08, Y13, Y13, Y13 - VPALIGNR $0x04, Y1, Y1, Y1 - CMPQ R9, CX - JB openAVX2Tail128LoopA - CMPQ R9, $0xa0 - JNE openAVX2Tail128LoopB - VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 - VPADDD 32(BP), Y9, Y9 - VPADDD 64(BP), Y13, Y13 - VPADDD Y4, Y1, Y1 - VPERM2I128 $0x02, Y5, Y9, Y0 - VPERM2I128 $0x02, Y13, Y1, Y14 - VPERM2I128 $0x13, Y5, Y9, Y12 - VPERM2I128 $0x13, Y13, Y1, Y4 - -openAVX2TailLoop: - CMPQ BX, $0x20 - JB openAVX2Tail - SUBQ $0x20, BX - - // Load for decryption - VPXOR (SI), Y0, Y0 - VMOVDQU Y0, (DI) - LEAQ 32(SI), SI - LEAQ 32(DI), DI - VMOVDQA Y14, Y0 - VMOVDQA Y12, Y14 - VMOVDQA Y4, Y12 - JMP openAVX2TailLoop - -openAVX2Tail: - CMPQ BX, $0x10 - VMOVDQA X0, X1 - JB openAVX2TailDone - SUBQ $0x10, BX - - // Load for decryption - VPXOR (SI), X0, X12 - VMOVDQU X12, (DI) - LEAQ 16(SI), SI - LEAQ 16(DI), DI - VPERM2I128 $0x11, Y0, Y0, Y0 - VMOVDQA X0, X1 - -openAVX2TailDone: - VZEROUPPER - JMP openSSETail16 - -openAVX2Tail256: - VMOVDQA ·chacha20Constants<>+0(SB), Y0 - VMOVDQA Y0, Y5 - VMOVDQA 32(BP), Y14 - VMOVDQA Y14, Y9 - VMOVDQA 64(BP), Y12 - VMOVDQA Y12, Y13 - VMOVDQA 192(BP), Y4 - VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 - VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 - VMOVDQA Y4, Y7 - VMOVDQA Y1, Y11 - - // Compute the number of iterations that will hash data - MOVQ BX, 224(BP) - MOVQ BX, CX - SUBQ $0x80, CX - SHRQ $0x04, CX - MOVQ $0x0000000a, R9 - CMPQ CX, $0x0a - CMOVQGT R9, CX - MOVQ SI, BX - XORQ R9, R9 - -openAVX2Tail256LoopA: - ADDQ (BX), R10 - ADCQ 8(BX), R11 - ADCQ $0x01, R12 - MOVQ (BP), DX - MOVQ DX, R15 - MULXQ R10, R13, R14 - IMULQ R12, R15 - MULXQ R11, AX, DX - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), DX - MULXQ R10, R10, AX - ADDQ R10, R14 - MULXQ R11, R11, R8 - ADCQ R11, R15 - ADCQ $0x00, R8 - IMULQ R12, DX - ADDQ AX, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - LEAQ 16(BX), BX - -openAVX2Tail256LoopB: - VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol16<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x0c, Y14, Y3 - VPSRLD $0x14, Y14, Y14 - VPXOR Y3, Y14, Y14 +openAVX2InternalLoop: + ADDQ (SI)(CX*1), R10 + ADCQ 8(SI)(CX*1), R11 + ADCQ $0x01, R12 VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol8<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x07, Y14, Y3 - VPSRLD $0x19, Y14, Y14 - VPXOR Y3, Y14, Y14 VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 - VPSLLD $0x0c, Y9, Y3 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 VPSRLD $0x14, Y9, Y9 - VPXOR Y3, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPADDD Y14, Y0, Y0 VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + ADDQ 16(SI)(CX*1), R10 + ADCQ 24(SI)(CX*1), R11 + ADCQ $0x01, R12 + VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 - VPSLLD $0x07, Y9, Y3 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 VPSRLD $0x19, Y9, Y9 - VPXOR Y3, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 VPALIGNR $0x04, Y14, Y14, Y14 VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x04, Y11, Y11, Y11 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 VPALIGNR $0x0c, Y4, Y4, Y4 VPALIGNR $0x0c, Y1, Y1, Y1 - INCQ R9 - VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol16<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x0c, Y14, Y3 - VPSRLD $0x14, Y14, Y14 - VPXOR Y3, Y14, Y14 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPALIGNR $0x0c, Y3, Y3, Y3 VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol8<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x07, Y14, Y3 - VPSRLD $0x19, Y14, Y14 - VPXOR Y3, Y14, Y14 VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 - VPSLLD $0x0c, Y9, Y3 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + ADDQ 32(SI)(CX*1), R10 + ADCQ 40(SI)(CX*1), R11 + ADCQ $0x01, R12 + LEAQ 48(CX), CX + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 VPSRLD $0x14, Y9, Y9 - VPXOR Y3, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + VPADDD Y14, Y0, Y0 VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 - VPSLLD $0x07, Y9, Y3 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 VPSRLD $0x19, Y9, Y9 - VPXOR Y3, Y9, Y9 - VPALIGNR $0x0c, Y14, Y14, Y14 - VPALIGNR $0x0c, Y9, Y9, Y9 - VPALIGNR $0x08, Y12, Y12, Y12 - VPALIGNR $0x08, Y13, Y13, Y13 - VPALIGNR $0x04, Y4, Y4, Y4 - VPALIGNR $0x04, Y1, Y1, Y1 - CMPQ R9, CX - JB openAVX2Tail256LoopA - CMPQ R9, $0x0a - JNE openAVX2Tail256LoopB - MOVQ BX, R9 - SUBQ SI, BX - MOVQ BX, CX - MOVQ 224(BP), BX - -openAVX2Tail256Hash: - ADDQ $0x10, CX - CMPQ CX, BX - JGT openAVX2Tail256HashEnd - ADDQ (R9), R10 - ADCQ 8(R9), R11 - ADCQ $0x01, R12 - MOVQ (BP), DX - MOVQ DX, R15 - MULXQ R10, R13, R14 - IMULQ R12, R15 - MULXQ R11, AX, DX - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), DX - MULXQ R10, R10, AX - ADDQ R10, R14 - MULXQ R11, R11, R8 - ADCQ R11, R15 - ADCQ $0x00, R8 - IMULQ R12, DX - ADDQ AX, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - LEAQ 16(R9), R9 - JMP openAVX2Tail256Hash + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x0c, Y11, Y11, Y11 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x04, Y2, Y2, Y2 + VPALIGNR $0x04, Y3, Y3, Y3 + CMPQ CX, $0x000001e0 + JNE openAVX2InternalLoop + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 + VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 32(BP), Y10, Y10 + VPADDD 32(BP), Y11, Y11 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD 64(BP), Y8, Y8 + VPADDD 64(BP), Y15, Y15 + VPADDD 96(BP), Y4, Y4 + VPADDD 128(BP), Y1, Y1 + VPADDD 160(BP), Y2, Y2 + VPADDD 192(BP), Y3, Y3 + VMOVDQA Y15, 224(BP) -openAVX2Tail256HashEnd: - VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 - VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 - VPADDD 32(BP), Y14, Y14 - VPADDD 32(BP), Y9, Y9 - VPADDD 64(BP), Y12, Y12 - VPADDD 64(BP), Y13, Y13 - VPADDD Y7, Y4, Y4 - VPADDD Y11, Y1, Y1 - VPERM2I128 $0x02, Y0, Y14, Y6 - VPERM2I128 $0x02, Y12, Y4, Y10 - VPERM2I128 $0x13, Y0, Y14, Y8 - VPERM2I128 $0x13, Y12, Y4, Y2 + // We only hashed 480 of the 512 bytes available - hash the remaining 32 here + ADDQ 480(SI), R10 + ADCQ 488(SI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPERM2I128 $0x02, Y0, Y14, Y15 + VPERM2I128 $0x13, Y0, Y14, Y14 + VPERM2I128 $0x02, Y12, Y4, Y0 + VPERM2I128 $0x13, Y12, Y4, Y12 + VPXOR (SI), Y15, Y15 + VPXOR 32(SI), Y0, Y0 + VPXOR 64(SI), Y14, Y14 + VPXOR 96(SI), Y12, Y12 + VMOVDQU Y15, (DI) + VMOVDQU Y0, 32(DI) + VMOVDQU Y14, 64(DI) + VMOVDQU Y12, 96(DI) VPERM2I128 $0x02, Y5, Y9, Y0 VPERM2I128 $0x02, Y13, Y1, Y14 VPERM2I128 $0x13, Y5, Y9, Y12 VPERM2I128 $0x13, Y13, Y1, Y4 - VPXOR (SI), Y6, Y6 - VPXOR 32(SI), Y10, Y10 - VPXOR 64(SI), Y8, Y8 - VPXOR 96(SI), Y2, Y2 - VMOVDQU Y6, (DI) - VMOVDQU Y10, 32(DI) - VMOVDQU Y8, 64(DI) - VMOVDQU Y2, 96(DI) - LEAQ 128(SI), SI - LEAQ 128(DI), DI - SUBQ $0x80, BX - JMP openAVX2TailLoop + VPXOR 128(SI), Y0, Y0 + VPXOR 160(SI), Y14, Y14 + VPXOR 192(SI), Y12, Y12 + VPXOR 224(SI), Y4, Y4 + VMOVDQU Y0, 128(DI) + VMOVDQU Y14, 160(DI) + VMOVDQU Y12, 192(DI) + VMOVDQU Y4, 224(DI) -openAVX2Tail384: - // Need to decrypt up to 384 bytes - prepare six blocks - VMOVDQA ·chacha20Constants<>+0(SB), Y0 - VMOVDQA Y0, Y5 - VMOVDQA Y0, Y6 - VMOVDQA 32(BP), Y14 - VMOVDQA Y14, Y9 - VMOVDQA Y14, Y10 - VMOVDQA 64(BP), Y12 - VMOVDQA Y12, Y13 - VMOVDQA Y12, Y8 - VMOVDQA 192(BP), Y4 - VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 - VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 - VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 - VMOVDQA Y4, 96(BP) - VMOVDQA Y1, 128(BP) - VMOVDQA Y2, 160(BP) + // and here + ADDQ 496(SI), R10 + ADCQ 504(SI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPERM2I128 $0x02, Y6, Y10, Y0 + VPERM2I128 $0x02, Y8, Y2, Y14 + VPERM2I128 $0x13, Y6, Y10, Y12 + VPERM2I128 $0x13, Y8, Y2, Y4 + VPXOR 256(SI), Y0, Y0 + VPXOR 288(SI), Y14, Y14 + VPXOR 320(SI), Y12, Y12 + VPXOR 352(SI), Y4, Y4 + VMOVDQU Y0, 256(DI) + VMOVDQU Y14, 288(DI) + VMOVDQU Y12, 320(DI) + VMOVDQU Y4, 352(DI) + VPERM2I128 $0x02, Y7, Y11, Y0 + VPERM2I128 $0x02, 224(BP), Y3, Y14 + VPERM2I128 $0x13, Y7, Y11, Y12 + VPERM2I128 $0x13, 224(BP), Y3, Y4 + VPXOR 384(SI), Y0, Y0 + VPXOR 416(SI), Y14, Y14 + VPXOR 448(SI), Y12, Y12 + VPXOR 480(SI), Y4, Y4 + VMOVDQU Y0, 384(DI) + VMOVDQU Y14, 416(DI) + VMOVDQU Y12, 448(DI) + VMOVDQU Y4, 480(DI) + LEAQ 512(SI), SI + LEAQ 512(DI), DI + SUBQ $0x00000200, BX + JMP openAVX2MainLoop - // Compute the number of iterations that will hash two blocks of data - MOVQ BX, 224(BP) - MOVQ BX, CX - SUBQ $0x00000100, CX - SHRQ $0x04, CX - ADDQ $0x06, CX - MOVQ $0x0000000a, R9 - CMPQ CX, $0x0a - CMOVQGT R9, CX - MOVQ SI, BX - XORQ R9, R9 +openAVX2MainLoopDone: + // Handle the various tail sizes efficiently + TESTQ BX, BX + JE openSSEFinalize + CMPQ BX, $0x80 + JBE openAVX2Tail128 + CMPQ BX, $0x00000100 + JBE openAVX2Tail256 + CMPQ BX, $0x00000180 + JBE openAVX2Tail384 + JMP openAVX2Tail512 -openAVX2Tail384LoopB: - ADDQ (BX), R10 - ADCQ 8(BX), R11 +openSSEFinalize: + // Hash in the PT, AAD lengths + ADDQ ad_len+80(FP), R10 + ADCQ src_len+56(FP), R11 ADCQ $0x01, R12 - MOVQ (BP), DX - MOVQ DX, R15 - MULXQ R10, R13, R14 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 IMULQ R12, R15 - MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 - MOVQ 8(BP), DX - MULXQ R10, R10, AX - ADDQ R10, R14 - MULXQ R11, R11, R8 - ADCQ R11, R15 - ADCQ $0x00, R8 - IMULQ R12, DX + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 @@ -4075,174 +797,261 @@ openAVX2Tail384LoopB: ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 - LEAQ 16(BX), BX -openAVX2Tail384LoopA: - VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol16<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x0c, Y14, Y3 - VPSRLD $0x14, Y14, Y14 - VPXOR Y3, Y14, Y14 - VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol8<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x07, Y14, Y3 - VPSRLD $0x19, Y14, Y14 - VPXOR Y3, Y14, Y14 - VPADDD Y9, Y5, Y5 - VPXOR Y5, Y1, Y1 - VPSHUFB ·rol16<>+0(SB), Y1, Y1 - VPADDD Y1, Y13, Y13 - VPXOR Y13, Y9, Y9 - VPSLLD $0x0c, Y9, Y3 - VPSRLD $0x14, Y9, Y9 - VPXOR Y3, Y9, Y9 - VPADDD Y9, Y5, Y5 - VPXOR Y5, Y1, Y1 - VPSHUFB ·rol8<>+0(SB), Y1, Y1 - VPADDD Y1, Y13, Y13 - VPXOR Y13, Y9, Y9 - VPSLLD $0x07, Y9, Y3 - VPSRLD $0x19, Y9, Y9 - VPXOR Y3, Y9, Y9 - VPADDD Y10, Y6, Y6 - VPXOR Y6, Y2, Y2 - VPSHUFB ·rol16<>+0(SB), Y2, Y2 - VPADDD Y2, Y8, Y8 - VPXOR Y8, Y10, Y10 - VPSLLD $0x0c, Y10, Y3 - VPSRLD $0x14, Y10, Y10 - VPXOR Y3, Y10, Y10 - VPADDD Y10, Y6, Y6 - VPXOR Y6, Y2, Y2 - VPSHUFB ·rol8<>+0(SB), Y2, Y2 - VPADDD Y2, Y8, Y8 - VPXOR Y8, Y10, Y10 - VPSLLD $0x07, Y10, Y3 - VPSRLD $0x19, Y10, Y10 - VPXOR Y3, Y10, Y10 - VPALIGNR $0x04, Y14, Y14, Y14 - VPALIGNR $0x04, Y9, Y9, Y9 - VPALIGNR $0x04, Y10, Y10, Y10 - VPALIGNR $0x08, Y12, Y12, Y12 - VPALIGNR $0x08, Y13, Y13, Y13 - VPALIGNR $0x08, Y8, Y8, Y8 - VPALIGNR $0x0c, Y4, Y4, Y4 - VPALIGNR $0x0c, Y1, Y1, Y1 - VPALIGNR $0x0c, Y2, Y2, Y2 - ADDQ (BX), R10 - ADCQ 8(BX), R11 - ADCQ $0x01, R12 - MOVQ (BP), DX - MOVQ DX, R15 - MULXQ R10, R13, R14 - IMULQ R12, R15 - MULXQ R11, AX, DX - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), DX - MULXQ R10, R10, AX - ADDQ R10, R14 - MULXQ R11, R11, R8 - ADCQ R11, R15 - ADCQ $0x00, R8 - IMULQ R12, DX - ADDQ AX, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - LEAQ 16(BX), BX - INCQ R9 - VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol16<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x0c, Y14, Y3 - VPSRLD $0x14, Y14, Y14 - VPXOR Y3, Y14, Y14 - VPADDD Y14, Y0, Y0 - VPXOR Y0, Y4, Y4 - VPSHUFB ·rol8<>+0(SB), Y4, Y4 - VPADDD Y4, Y12, Y12 - VPXOR Y12, Y14, Y14 - VPSLLD $0x07, Y14, Y3 - VPSRLD $0x19, Y14, Y14 - VPXOR Y3, Y14, Y14 - VPADDD Y9, Y5, Y5 - VPXOR Y5, Y1, Y1 - VPSHUFB ·rol16<>+0(SB), Y1, Y1 - VPADDD Y1, Y13, Y13 - VPXOR Y13, Y9, Y9 - VPSLLD $0x0c, Y9, Y3 - VPSRLD $0x14, Y9, Y9 - VPXOR Y3, Y9, Y9 - VPADDD Y9, Y5, Y5 - VPXOR Y5, Y1, Y1 - VPSHUFB ·rol8<>+0(SB), Y1, Y1 - VPADDD Y1, Y13, Y13 - VPXOR Y13, Y9, Y9 - VPSLLD $0x07, Y9, Y3 - VPSRLD $0x19, Y9, Y9 - VPXOR Y3, Y9, Y9 - VPADDD Y10, Y6, Y6 - VPXOR Y6, Y2, Y2 - VPSHUFB ·rol16<>+0(SB), Y2, Y2 - VPADDD Y2, Y8, Y8 - VPXOR Y8, Y10, Y10 - VPSLLD $0x0c, Y10, Y3 - VPSRLD $0x14, Y10, Y10 - VPXOR Y3, Y10, Y10 - VPADDD Y10, Y6, Y6 - VPXOR Y6, Y2, Y2 - VPSHUFB ·rol8<>+0(SB), Y2, Y2 - VPADDD Y2, Y8, Y8 - VPXOR Y8, Y10, Y10 - VPSLLD $0x07, Y10, Y3 - VPSRLD $0x19, Y10, Y10 - VPXOR Y3, Y10, Y10 - VPALIGNR $0x0c, Y14, Y14, Y14 - VPALIGNR $0x0c, Y9, Y9, Y9 - VPALIGNR $0x0c, Y10, Y10, Y10 - VPALIGNR $0x08, Y12, Y12, Y12 - VPALIGNR $0x08, Y13, Y13, Y13 - VPALIGNR $0x08, Y8, Y8, Y8 - VPALIGNR $0x04, Y4, Y4, Y4 - VPALIGNR $0x04, Y1, Y1, Y1 - VPALIGNR $0x04, Y2, Y2, Y2 - CMPQ R9, CX - JB openAVX2Tail384LoopB - CMPQ R9, $0x0a - JNE openAVX2Tail384LoopA - MOVQ BX, R9 - SUBQ SI, BX - MOVQ BX, CX - MOVQ 224(BP), BX + // Final reduce + MOVQ R10, R13 + MOVQ R11, R14 + MOVQ R12, R15 + SUBQ $-5, R10 + SBBQ $-1, R11 + SBBQ $0x03, R12 + CMOVQCS R13, R10 + CMOVQCS R14, R11 + CMOVQCS R15, R12 + + // Add in the "s" part of the key + ADDQ 16(BP), R10 + ADCQ 24(BP), R11 + + // Finally, constant time compare to the tag at the end of the message + XORQ AX, AX + MOVQ $0x00000001, DX + XORQ (SI), R10 + XORQ 8(SI), R11 + ORQ R11, R10 + CMOVQEQ DX, AX + + // Return true iff tags are equal + MOVB AX, ret+96(FP) + RET + +openSSETail16: + TESTQ BX, BX + JE openSSEFinalize + + // We can safely load the CT from the end, because it is padded with the MAC + MOVQ BX, R9 + SHLQ $0x04, R9 + LEAQ ·andMask<>+0(SB), R13 + MOVOU (SI), X12 + ADDQ BX, SI + PAND -16(R13)(R9*1), X12 + MOVO X12, 64(BP) + MOVQ X12, R13 + MOVQ 72(BP), R14 + PXOR X1, X12 + + // We can only store one byte at a time, since plaintext can be shorter than 16 bytes +openSSETail16Store: + MOVQ X12, R8 + MOVB R8, (DI) + PSRLDQ $0x01, X12 + INCQ DI + DECQ BX + JNE openSSETail16Store + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + JMP openSSEFinalize -openAVX2Tail384Hash: - ADDQ $0x10, CX - CMPQ CX, BX - JGT openAVX2Tail384HashEnd - ADDQ (R9), R10 - ADCQ 8(R9), R11 +openAVX2192: + VMOVDQA Y0, Y5 + VMOVDQA Y14, Y9 + VMOVDQA Y12, Y13 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VMOVDQA Y0, Y6 + VMOVDQA Y14, Y10 + VMOVDQA Y12, Y8 + VMOVDQA Y4, Y2 + VMOVDQA Y1, Y15 + MOVQ $0x0000000a, R9 + +openAVX2192InnerCipherLoop: + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + DECQ R9 + JNE openAVX2192InnerCipherLoop + VPADDD Y6, Y0, Y0 + VPADDD Y6, Y5, Y5 + VPADDD Y10, Y14, Y14 + VPADDD Y10, Y9, Y9 + VPADDD Y8, Y12, Y12 + VPADDD Y8, Y13, Y13 + VPADDD Y2, Y4, Y4 + VPADDD Y15, Y1, Y1 + VPERM2I128 $0x02, Y0, Y14, Y3 + + // Clamp and store poly key + VPAND ·polyClampMask<>+0(SB), Y3, Y3 + VMOVDQA Y3, (BP) + + // Stream for up to 192 bytes + VPERM2I128 $0x13, Y0, Y14, Y0 + VPERM2I128 $0x13, Y12, Y4, Y14 + VPERM2I128 $0x02, Y5, Y9, Y12 + VPERM2I128 $0x02, Y13, Y1, Y4 + VPERM2I128 $0x13, Y5, Y9, Y5 + VPERM2I128 $0x13, Y13, Y1, Y9 + +openAVX2ShortOpen: + // Hash + MOVQ ad_len+80(FP), R9 + CALL polyHashADInternal<>(SB) + +openAVX2ShortOpenLoop: + CMPQ BX, $0x20 + JB openAVX2ShortTail32 + SUBQ $0x20, BX + + // Load for hashing + ADDQ (SI), R10 + ADCQ 8(SI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + ADDQ 16(SI), R10 + ADCQ 24(SI), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 @@ -4275,83 +1084,34 @@ openAVX2Tail384Hash: ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 - LEAQ 16(R9), R9 - JMP openAVX2Tail384Hash -openAVX2Tail384HashEnd: - VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 - VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 - VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 - VPADDD 32(BP), Y14, Y14 - VPADDD 32(BP), Y9, Y9 - VPADDD 32(BP), Y10, Y10 - VPADDD 64(BP), Y12, Y12 - VPADDD 64(BP), Y13, Y13 - VPADDD 64(BP), Y8, Y8 - VPADDD 96(BP), Y4, Y4 - VPADDD 128(BP), Y1, Y1 - VPADDD 160(BP), Y2, Y2 - VPERM2I128 $0x02, Y0, Y14, Y3 - VPERM2I128 $0x02, Y12, Y4, Y7 - VPERM2I128 $0x13, Y0, Y14, Y11 - VPERM2I128 $0x13, Y12, Y4, Y15 - VPXOR (SI), Y3, Y3 - VPXOR 32(SI), Y7, Y7 - VPXOR 64(SI), Y11, Y11 - VPXOR 96(SI), Y15, Y15 - VMOVDQU Y3, (DI) - VMOVDQU Y7, 32(DI) - VMOVDQU Y11, 64(DI) - VMOVDQU Y15, 96(DI) - VPERM2I128 $0x02, Y5, Y9, Y3 - VPERM2I128 $0x02, Y13, Y1, Y7 - VPERM2I128 $0x13, Y5, Y9, Y11 - VPERM2I128 $0x13, Y13, Y1, Y15 - VPXOR 128(SI), Y3, Y3 - VPXOR 160(SI), Y7, Y7 - VPXOR 192(SI), Y11, Y11 - VPXOR 224(SI), Y15, Y15 - VMOVDQU Y3, 128(DI) - VMOVDQU Y7, 160(DI) - VMOVDQU Y11, 192(DI) - VMOVDQU Y15, 224(DI) - VPERM2I128 $0x02, Y6, Y10, Y0 - VPERM2I128 $0x02, Y8, Y2, Y14 - VPERM2I128 $0x13, Y6, Y10, Y12 - VPERM2I128 $0x13, Y8, Y2, Y4 - LEAQ 256(SI), SI - LEAQ 256(DI), DI - SUBQ $0x00000100, BX - JMP openAVX2TailLoop + // Load for decryption + VPXOR (SI), Y0, Y0 + VMOVDQU Y0, (DI) + LEAQ 32(SI), SI + LEAQ 32(DI), DI -openAVX2Tail512: - VMOVDQU ·chacha20Constants<>+0(SB), Y0 - VMOVDQA Y0, Y5 - VMOVDQA Y0, Y6 - VMOVDQA Y0, Y7 - VMOVDQA 32(BP), Y14 - VMOVDQA Y14, Y9 - VMOVDQA Y14, Y10 - VMOVDQA Y14, Y11 - VMOVDQA 64(BP), Y12 - VMOVDQA Y12, Y13 - VMOVDQA Y12, Y8 - VMOVDQA Y12, Y15 - VMOVDQA 192(BP), Y4 - VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 - VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 - VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 - VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 - VMOVDQA Y4, 96(BP) - VMOVDQA Y1, 128(BP) - VMOVDQA Y2, 160(BP) - VMOVDQA Y3, 192(BP) - XORQ CX, CX - MOVQ SI, R9 + // Shift stream left + VMOVDQA Y14, Y0 + VMOVDQA Y12, Y14 + VMOVDQA Y4, Y12 + VMOVDQA Y5, Y4 + VMOVDQA Y9, Y5 + VMOVDQA Y13, Y9 + VMOVDQA Y1, Y13 + VMOVDQA Y6, Y1 + VMOVDQA Y10, Y6 + JMP openAVX2ShortOpenLoop + +openAVX2ShortTail32: + CMPQ BX, $0x10 + VMOVDQA X0, X1 + JB openAVX2ShortDone + SUBQ $0x10, BX -openAVX2Tail512LoopB: - ADDQ (R9), R10 - ADCQ 8(R9), R11 + // Load for hashing + ADDQ (SI), R10 + ADCQ 8(SI), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 @@ -4384,252 +1144,202 @@ openAVX2Tail512LoopB: ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 - LEAQ 16(R9), R9 -openAVX2Tail512LoopA: + // Load for decryption + VPXOR (SI), X0, X12 + VMOVDQU X12, (DI) + LEAQ 16(SI), SI + LEAQ 16(DI), DI + VPERM2I128 $0x11, Y0, Y0, Y0 + VMOVDQA X0, X1 + +openAVX2ShortDone: + VZEROUPPER + JMP openSSETail16 + +openAVX2320: + VMOVDQA Y0, Y5 + VMOVDQA Y14, Y9 + VMOVDQA Y12, Y13 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VMOVDQA Y0, Y6 + VMOVDQA Y14, Y10 + VMOVDQA Y12, Y8 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VMOVDQA Y14, Y7 + VMOVDQA Y12, Y11 + VMOVDQA Y4, Y15 + MOVQ $0x0000000a, R9 + +openAVX2320InnerCipherLoop: VPADDD Y14, Y0, Y0 - VPADDD Y9, Y5, Y5 - VPADDD Y10, Y6, Y6 - VPADDD Y11, Y7, Y7 VPXOR Y0, Y4, Y4 - VPXOR Y5, Y1, Y1 - VPXOR Y6, Y2, Y2 - VPXOR Y7, Y3, Y3 VPSHUFB ·rol16<>+0(SB), Y4, Y4 - VPSHUFB ·rol16<>+0(SB), Y1, Y1 - VPSHUFB ·rol16<>+0(SB), Y2, Y2 - VPSHUFB ·rol16<>+0(SB), Y3, Y3 VPADDD Y4, Y12, Y12 - VPADDD Y1, Y13, Y13 - VPADDD Y2, Y8, Y8 - VPADDD Y3, Y15, Y15 VPXOR Y12, Y14, Y14 - VPXOR Y13, Y9, Y9 - VPXOR Y8, Y10, Y10 - VPXOR Y15, Y11, Y11 - VMOVDQA Y15, 224(BP) - VPSLLD $0x0c, Y14, Y15 + VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 - VPXOR Y15, Y14, Y14 - VPSLLD $0x0c, Y9, Y15 - VPSRLD $0x14, Y9, Y9 - VPXOR Y15, Y9, Y9 - VPSLLD $0x0c, Y10, Y15 - VPSRLD $0x14, Y10, Y10 - VPXOR Y15, Y10, Y10 - VPSLLD $0x0c, Y11, Y15 - VPSRLD $0x14, Y11, Y11 - VPXOR Y15, Y11, Y11 - VMOVDQA 224(BP), Y15 - ADDQ (R9), R10 - ADCQ 8(R9), R11 - ADCQ $0x01, R12 - MOVQ (BP), DX - MOVQ DX, R15 - MULXQ R10, R13, R14 - IMULQ R12, R15 - MULXQ R11, AX, DX - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), DX - MULXQ R10, R10, AX - ADDQ R10, R14 - MULXQ R11, R11, R8 - ADCQ R11, R15 - ADCQ $0x00, R8 - IMULQ R12, DX - ADDQ AX, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 + VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 - VPADDD Y9, Y5, Y5 - VPADDD Y10, Y6, Y6 - VPADDD Y11, Y7, Y7 VPXOR Y0, Y4, Y4 - VPXOR Y5, Y1, Y1 - VPXOR Y6, Y2, Y2 - VPXOR Y7, Y3, Y3 VPSHUFB ·rol8<>+0(SB), Y4, Y4 - VPSHUFB ·rol8<>+0(SB), Y1, Y1 - VPSHUFB ·rol8<>+0(SB), Y2, Y2 - VPSHUFB ·rol8<>+0(SB), Y3, Y3 VPADDD Y4, Y12, Y12 - VPADDD Y1, Y13, Y13 - VPADDD Y2, Y8, Y8 - VPADDD Y3, Y15, Y15 VPXOR Y12, Y14, Y14 - VPXOR Y13, Y9, Y9 - VPXOR Y8, Y10, Y10 - VPXOR Y15, Y11, Y11 - VMOVDQA Y15, 224(BP) - VPSLLD $0x07, Y14, Y15 + VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 - VPXOR Y15, Y14, Y14 - VPSLLD $0x07, Y9, Y15 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 VPSRLD $0x19, Y9, Y9 - VPXOR Y15, Y9, Y9 - VPSLLD $0x07, Y10, Y15 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y3 VPSRLD $0x19, Y10, Y10 - VPXOR Y15, Y10, Y10 - VPSLLD $0x07, Y11, Y15 - VPSRLD $0x19, Y11, Y11 - VPXOR Y15, Y11, Y11 - VMOVDQA 224(BP), Y15 + VPXOR Y3, Y10, Y10 VPALIGNR $0x04, Y14, Y14, Y14 VPALIGNR $0x04, Y9, Y9, Y9 VPALIGNR $0x04, Y10, Y10, Y10 - VPALIGNR $0x04, Y11, Y11, Y11 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x08, Y8, Y8, Y8 - VPALIGNR $0x08, Y15, Y15, Y15 VPALIGNR $0x0c, Y4, Y4, Y4 VPALIGNR $0x0c, Y1, Y1, Y1 VPALIGNR $0x0c, Y2, Y2, Y2 - VPALIGNR $0x0c, Y3, Y3, Y3 VPADDD Y14, Y0, Y0 - VPADDD Y9, Y5, Y5 - VPADDD Y10, Y6, Y6 - VPADDD Y11, Y7, Y7 VPXOR Y0, Y4, Y4 - VPXOR Y5, Y1, Y1 - VPXOR Y6, Y2, Y2 - VPXOR Y7, Y3, Y3 VPSHUFB ·rol16<>+0(SB), Y4, Y4 - VPSHUFB ·rol16<>+0(SB), Y1, Y1 - VPSHUFB ·rol16<>+0(SB), Y2, Y2 - VPSHUFB ·rol16<>+0(SB), Y3, Y3 VPADDD Y4, Y12, Y12 - VPADDD Y1, Y13, Y13 - VPADDD Y2, Y8, Y8 - VPADDD Y3, Y15, Y15 VPXOR Y12, Y14, Y14 - VPXOR Y13, Y9, Y9 - VPXOR Y8, Y10, Y10 - VPXOR Y15, Y11, Y11 - ADDQ 16(R9), R10 - ADCQ 24(R9), R11 - ADCQ $0x01, R12 - MOVQ (BP), DX - MOVQ DX, R15 - MULXQ R10, R13, R14 - IMULQ R12, R15 - MULXQ R11, AX, DX - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), DX - MULXQ R10, R10, AX - ADDQ R10, R14 - MULXQ R11, R11, R8 - ADCQ R11, R15 - ADCQ $0x00, R8 - IMULQ R12, DX - ADDQ AX, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - LEAQ 32(R9), R9 - VMOVDQA Y15, 224(BP) - VPSLLD $0x0c, Y14, Y15 + VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 - VPXOR Y15, Y14, Y14 - VPSLLD $0x0c, Y9, Y15 - VPSRLD $0x14, Y9, Y9 - VPXOR Y15, Y9, Y9 - VPSLLD $0x0c, Y10, Y15 - VPSRLD $0x14, Y10, Y10 - VPXOR Y15, Y10, Y10 - VPSLLD $0x0c, Y11, Y15 - VPSRLD $0x14, Y11, Y11 - VPXOR Y15, Y11, Y11 - VMOVDQA 224(BP), Y15 + VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 VPADDD Y10, Y6, Y6 - VPADDD Y11, Y7, Y7 - VPXOR Y0, Y4, Y4 - VPXOR Y5, Y1, Y1 VPXOR Y6, Y2, Y2 - VPXOR Y7, Y3, Y3 - VPSHUFB ·rol8<>+0(SB), Y4, Y4 - VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y2, Y2 - VPSHUFB ·rol8<>+0(SB), Y3, Y3 - VPADDD Y4, Y12, Y12 - VPADDD Y1, Y13, Y13 VPADDD Y2, Y8, Y8 - VPADDD Y3, Y15, Y15 - VPXOR Y12, Y14, Y14 - VPXOR Y13, Y9, Y9 VPXOR Y8, Y10, Y10 - VPXOR Y15, Y11, Y11 - VMOVDQA Y15, 224(BP) - VPSLLD $0x07, Y14, Y15 - VPSRLD $0x19, Y14, Y14 - VPXOR Y15, Y14, Y14 - VPSLLD $0x07, Y9, Y15 - VPSRLD $0x19, Y9, Y9 - VPXOR Y15, Y9, Y9 - VPSLLD $0x07, Y10, Y15 + VPSLLD $0x07, Y10, Y3 VPSRLD $0x19, Y10, Y10 - VPXOR Y15, Y10, Y10 - VPSLLD $0x07, Y11, Y15 - VPSRLD $0x19, Y11, Y11 - VPXOR Y15, Y11, Y11 - VMOVDQA 224(BP), Y15 + VPXOR Y3, Y10, Y10 VPALIGNR $0x0c, Y14, Y14, Y14 VPALIGNR $0x0c, Y9, Y9, Y9 VPALIGNR $0x0c, Y10, Y10, Y10 - VPALIGNR $0x0c, Y11, Y11, Y11 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x08, Y8, Y8, Y8 - VPALIGNR $0x08, Y15, Y15, Y15 VPALIGNR $0x04, Y4, Y4, Y4 VPALIGNR $0x04, Y1, Y1, Y1 VPALIGNR $0x04, Y2, Y2, Y2 - VPALIGNR $0x04, Y3, Y3, Y3 - INCQ CX - CMPQ CX, $0x04 - JLT openAVX2Tail512LoopB - CMPQ CX, $0x0a - JNE openAVX2Tail512LoopA - MOVQ BX, CX - SUBQ $0x00000180, CX - ANDQ $-16, CX + DECQ R9 + JNE openAVX2320InnerCipherLoop + VMOVDQA ·chacha20Constants<>+0(SB), Y3 + VPADDD Y3, Y0, Y0 + VPADDD Y3, Y5, Y5 + VPADDD Y3, Y6, Y6 + VPADDD Y7, Y14, Y14 + VPADDD Y7, Y9, Y9 + VPADDD Y7, Y10, Y10 + VPADDD Y11, Y12, Y12 + VPADDD Y11, Y13, Y13 + VPADDD Y11, Y8, Y8 + VMOVDQA ·avx2IncMask<>+0(SB), Y3 + VPADDD Y15, Y4, Y4 + VPADDD Y3, Y15, Y15 + VPADDD Y15, Y1, Y1 + VPADDD Y3, Y15, Y15 + VPADDD Y15, Y2, Y2 -openAVX2Tail512HashLoop: - TESTQ CX, CX - JE openAVX2Tail512HashEnd - ADDQ (R9), R10 - ADCQ 8(R9), R11 + // Clamp and store poly key + VPERM2I128 $0x02, Y0, Y14, Y3 + VPAND ·polyClampMask<>+0(SB), Y3, Y3 + VMOVDQA Y3, (BP) + + // Stream for up to 320 bytes + VPERM2I128 $0x13, Y0, Y14, Y0 + VPERM2I128 $0x13, Y12, Y4, Y14 + VPERM2I128 $0x02, Y5, Y9, Y12 + VPERM2I128 $0x02, Y13, Y1, Y4 + VPERM2I128 $0x13, Y5, Y9, Y5 + VPERM2I128 $0x13, Y13, Y1, Y9 + VPERM2I128 $0x02, Y6, Y10, Y13 + VPERM2I128 $0x02, Y8, Y2, Y1 + VPERM2I128 $0x13, Y6, Y10, Y6 + VPERM2I128 $0x13, Y8, Y2, Y10 + JMP openAVX2ShortOpen + +openAVX2Tail128: + // Need to decrypt up to 128 bytes - prepare two blocks + VMOVDQA ·chacha20Constants<>+0(SB), Y5 + VMOVDQA 32(BP), Y9 + VMOVDQA 64(BP), Y13 + VMOVDQA 192(BP), Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y1 + VMOVDQA Y1, Y4 + XORQ R9, R9 + MOVQ BX, CX + ANDQ $-16, CX + TESTQ CX, CX + JE openAVX2Tail128LoopB + +openAVX2Tail128LoopA: + ADDQ (SI)(R9*1), R10 + ADCQ 8(SI)(R9*1), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 @@ -4662,884 +1372,137 @@ openAVX2Tail512HashLoop: ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 - LEAQ 16(R9), R9 - SUBQ $0x10, CX - JMP openAVX2Tail512HashLoop -openAVX2Tail512HashEnd: - VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 +openAVX2Tail128LoopB: + ADDQ $0x10, R9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x04, Y1, Y1, Y1 + CMPQ R9, CX + JB openAVX2Tail128LoopA + CMPQ R9, $0xa0 + JNE openAVX2Tail128LoopB VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 - VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 - VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 - VPADDD 32(BP), Y14, Y14 VPADDD 32(BP), Y9, Y9 - VPADDD 32(BP), Y10, Y10 - VPADDD 32(BP), Y11, Y11 - VPADDD 64(BP), Y12, Y12 VPADDD 64(BP), Y13, Y13 - VPADDD 64(BP), Y8, Y8 - VPADDD 64(BP), Y15, Y15 - VPADDD 96(BP), Y4, Y4 - VPADDD 128(BP), Y1, Y1 - VPADDD 160(BP), Y2, Y2 - VPADDD 192(BP), Y3, Y3 - VMOVDQA Y15, 224(BP) - VPERM2I128 $0x02, Y0, Y14, Y15 - VPERM2I128 $0x13, Y0, Y14, Y14 - VPERM2I128 $0x02, Y12, Y4, Y0 - VPERM2I128 $0x13, Y12, Y4, Y12 - VPXOR (SI), Y15, Y15 - VPXOR 32(SI), Y0, Y0 - VPXOR 64(SI), Y14, Y14 - VPXOR 96(SI), Y12, Y12 - VMOVDQU Y15, (DI) - VMOVDQU Y0, 32(DI) - VMOVDQU Y14, 64(DI) - VMOVDQU Y12, 96(DI) - VPERM2I128 $0x02, Y5, Y9, Y0 - VPERM2I128 $0x02, Y13, Y1, Y14 - VPERM2I128 $0x13, Y5, Y9, Y12 - VPERM2I128 $0x13, Y13, Y1, Y4 - VPXOR 128(SI), Y0, Y0 - VPXOR 160(SI), Y14, Y14 - VPXOR 192(SI), Y12, Y12 - VPXOR 224(SI), Y4, Y4 - VMOVDQU Y0, 128(DI) - VMOVDQU Y14, 160(DI) - VMOVDQU Y12, 192(DI) - VMOVDQU Y4, 224(DI) - VPERM2I128 $0x02, Y6, Y10, Y0 - VPERM2I128 $0x02, Y8, Y2, Y14 - VPERM2I128 $0x13, Y6, Y10, Y12 - VPERM2I128 $0x13, Y8, Y2, Y4 - VPXOR 256(SI), Y0, Y0 - VPXOR 288(SI), Y14, Y14 - VPXOR 320(SI), Y12, Y12 - VPXOR 352(SI), Y4, Y4 - VMOVDQU Y0, 256(DI) - VMOVDQU Y14, 288(DI) - VMOVDQU Y12, 320(DI) - VMOVDQU Y4, 352(DI) - VPERM2I128 $0x02, Y7, Y11, Y0 - VPERM2I128 $0x02, 224(BP), Y3, Y14 - VPERM2I128 $0x13, Y7, Y11, Y12 - VPERM2I128 $0x13, 224(BP), Y3, Y4 - LEAQ 384(SI), SI - LEAQ 384(DI), DI - SUBQ $0x00000180, BX - JMP openAVX2TailLoop - -DATA ·chacha20Constants<>+0(SB)/4, $0x61707865 -DATA ·chacha20Constants<>+4(SB)/4, $0x3320646e -DATA ·chacha20Constants<>+8(SB)/4, $0x79622d32 -DATA ·chacha20Constants<>+12(SB)/4, $0x6b206574 -DATA ·chacha20Constants<>+16(SB)/4, $0x61707865 -DATA ·chacha20Constants<>+20(SB)/4, $0x3320646e -DATA ·chacha20Constants<>+24(SB)/4, $0x79622d32 -DATA ·chacha20Constants<>+28(SB)/4, $0x6b206574 -GLOBL ·chacha20Constants<>(SB), RODATA|NOPTR, $32 - -DATA ·polyClampMask<>+0(SB)/8, $0x0ffffffc0fffffff -DATA ·polyClampMask<>+8(SB)/8, $0x0ffffffc0ffffffc -DATA ·polyClampMask<>+16(SB)/8, $0xffffffffffffffff -DATA ·polyClampMask<>+24(SB)/8, $0xffffffffffffffff -GLOBL ·polyClampMask<>(SB), RODATA|NOPTR, $32 - -DATA ·sseIncMask<>+0(SB)/8, $0x0000000000000001 -DATA ·sseIncMask<>+8(SB)/8, $0x0000000000000000 -GLOBL ·sseIncMask<>(SB), RODATA|NOPTR, $16 - -DATA ·andMask<>+0(SB)/8, $0x00000000000000ff -DATA ·andMask<>+8(SB)/8, $0x0000000000000000 -DATA ·andMask<>+16(SB)/8, $0x000000000000ffff -DATA ·andMask<>+24(SB)/8, $0x0000000000000000 -DATA ·andMask<>+32(SB)/8, $0x0000000000ffffff -DATA ·andMask<>+40(SB)/8, $0x0000000000000000 -DATA ·andMask<>+48(SB)/8, $0x00000000ffffffff -DATA ·andMask<>+56(SB)/8, $0x0000000000000000 -DATA ·andMask<>+64(SB)/8, $0x000000ffffffffff -DATA ·andMask<>+72(SB)/8, $0x0000000000000000 -DATA ·andMask<>+80(SB)/8, $0x0000ffffffffffff -DATA ·andMask<>+88(SB)/8, $0x0000000000000000 -DATA ·andMask<>+96(SB)/8, $0x00ffffffffffffff -DATA ·andMask<>+104(SB)/8, $0x0000000000000000 -DATA ·andMask<>+112(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+120(SB)/8, $0x0000000000000000 -DATA ·andMask<>+128(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+136(SB)/8, $0x00000000000000ff -DATA ·andMask<>+144(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+152(SB)/8, $0x000000000000ffff -DATA ·andMask<>+160(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+168(SB)/8, $0x0000000000ffffff -DATA ·andMask<>+176(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+184(SB)/8, $0x00000000ffffffff -DATA ·andMask<>+192(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+200(SB)/8, $0x000000ffffffffff -DATA ·andMask<>+208(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+216(SB)/8, $0x0000ffffffffffff -DATA ·andMask<>+224(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+232(SB)/8, $0x00ffffffffffffff -GLOBL ·andMask<>(SB), RODATA|NOPTR, $240 + VPADDD Y4, Y1, Y1 + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 -DATA ·avx2InitMask<>+0(SB)/8, $0x0000000000000000 -DATA ·avx2InitMask<>+8(SB)/8, $0x0000000000000000 -DATA ·avx2InitMask<>+16(SB)/8, $0x0000000000000001 -DATA ·avx2InitMask<>+24(SB)/8, $0x0000000000000000 -GLOBL ·avx2InitMask<>(SB), RODATA|NOPTR, $32 +openAVX2TailLoop: + CMPQ BX, $0x20 + JB openAVX2Tail + SUBQ $0x20, BX -DATA ·rol16<>+0(SB)/8, $0x0504070601000302 -DATA ·rol16<>+8(SB)/8, $0x0d0c0f0e09080b0a -DATA ·rol16<>+16(SB)/8, $0x0504070601000302 -DATA ·rol16<>+24(SB)/8, $0x0d0c0f0e09080b0a -GLOBL ·rol16<>(SB), RODATA|NOPTR, $32 + // Load for decryption + VPXOR (SI), Y0, Y0 + VMOVDQU Y0, (DI) + LEAQ 32(SI), SI + LEAQ 32(DI), DI + VMOVDQA Y14, Y0 + VMOVDQA Y12, Y14 + VMOVDQA Y4, Y12 + JMP openAVX2TailLoop -DATA ·rol8<>+0(SB)/8, $0x0605040702010003 -DATA ·rol8<>+8(SB)/8, $0x0e0d0c0f0a09080b -DATA ·rol8<>+16(SB)/8, $0x0605040702010003 -DATA ·rol8<>+24(SB)/8, $0x0e0d0c0f0a09080b -GLOBL ·rol8<>(SB), RODATA|NOPTR, $32 +openAVX2Tail: + CMPQ BX, $0x10 + VMOVDQA X0, X1 + JB openAVX2TailDone + SUBQ $0x10, BX -DATA ·avx2IncMask<>+0(SB)/8, $0x0000000000000002 -DATA ·avx2IncMask<>+8(SB)/8, $0x0000000000000000 -DATA ·avx2IncMask<>+16(SB)/8, $0x0000000000000002 -DATA ·avx2IncMask<>+24(SB)/8, $0x0000000000000000 -GLOBL ·avx2IncMask<>(SB), RODATA|NOPTR, $32 + // Load for decryption + VPXOR (SI), X0, X12 + VMOVDQU X12, (DI) + LEAQ 16(SI), SI + LEAQ 16(DI), DI + VPERM2I128 $0x11, Y0, Y0, Y0 + VMOVDQA X0, X1 -// func chacha20Poly1305Seal(dst []byte, key []uint32, src []byte, ad []byte) -// Requires: AVX, AVX2, BMI2, CMOV, SSE2 -TEXT ·chacha20Poly1305Seal(SB), $288-96 - MOVQ SP, BP - ADDQ $0x20, BP - ANDQ $-32, BP - MOVQ dst_base+0(FP), DI - MOVQ key_base+24(FP), R8 - MOVQ src_base+48(FP), SI - MOVQ src_len+56(FP), BX - MOVQ ad_base+72(FP), CX - CMPB ·useAVX2+0(SB), $0x01 - JE chacha20Poly1305Seal_AVX2 +openAVX2TailDone: + VZEROUPPER + JMP openSSETail16 - // Special optimization, for very short buffers - CMPQ BX, $0x80 - JBE sealSSE128 - - // In the seal case - prepare the poly key + 3 blocks of stream in the first iteration - MOVOU ·chacha20Constants<>+0(SB), X0 - MOVOU 16(R8), X3 - MOVOU 32(R8), X6 - MOVOU 48(R8), X9 - - // Store state on stack for future use - MOVO X3, 32(BP) - MOVO X6, 48(BP) - - // Load state, increment counter blocks - MOVO X0, X1 - MOVO X3, X4 - MOVO X6, X7 - MOVO X9, X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X1, X2 - MOVO X4, X5 - MOVO X7, X8 - MOVO X10, X11 - PADDL ·sseIncMask<>+0(SB), X11 - MOVO X2, X12 - MOVO X5, X13 - MOVO X8, X14 - MOVO X11, X15 - PADDL ·sseIncMask<>+0(SB), X15 - - // Store counters - MOVO X9, 80(BP) - MOVO X10, 96(BP) - MOVO X11, 112(BP) - MOVO X15, 128(BP) - MOVQ $0x0000000a, R9 - -sealSSEIntroLoop: - MOVO X14, 64(BP) - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X3 - PXOR X14, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X3 - PXOR X14, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X4 - PXOR X14, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X4 - PXOR X14, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X5 - PXOR X14, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X5 - PXOR X14, X5 - MOVO 64(BP), X14 - MOVO X7, 64(BP) - PADDD X13, X12 - PXOR X12, X15 - ROL16(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x0c, X7 - PSRLL $0x14, X13 - PXOR X7, X13 - PADDD X13, X12 - PXOR X12, X15 - ROL8(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x07, X7 - PSRLL $0x19, X13 - PXOR X7, X13 - MOVO 64(BP), X7 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x0c - MOVO X14, 64(BP) - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X3 - PXOR X14, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X3 - PXOR X14, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X4 - PXOR X14, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X4 - PXOR X14, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X5 - PXOR X14, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X5 - PXOR X14, X5 - MOVO 64(BP), X14 - MOVO X7, 64(BP) - PADDD X13, X12 - PXOR X12, X15 - ROL16(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x0c, X7 - PSRLL $0x14, X13 - PXOR X7, X13 - PADDD X13, X12 - PXOR X12, X15 - ROL8(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x07, X7 - PSRLL $0x19, X13 - PXOR X7, X13 - MOVO 64(BP), X7 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x04 - DECQ R9 - JNE sealSSEIntroLoop - - // Add in the state - PADDD ·chacha20Constants<>+0(SB), X0 - PADDD ·chacha20Constants<>+0(SB), X1 - PADDD ·chacha20Constants<>+0(SB), X2 - PADDD ·chacha20Constants<>+0(SB), X12 - PADDD 32(BP), X3 - PADDD 32(BP), X4 - PADDD 32(BP), X5 - PADDD 32(BP), X13 - PADDD 48(BP), X7 - PADDD 48(BP), X8 - PADDD 48(BP), X14 - PADDD 96(BP), X10 - PADDD 112(BP), X11 - PADDD 128(BP), X15 - - // Clamp and store the key - PAND ·polyClampMask<>+0(SB), X0 - MOVO X0, (BP) - MOVO X3, 16(BP) - - // Hash AAD - MOVQ ad_len+80(FP), R9 - CALL polyHashADInternal<>(SB) - MOVOU (SI), X0 - MOVOU 16(SI), X3 - MOVOU 32(SI), X6 - MOVOU 48(SI), X9 - PXOR X0, X1 - PXOR X3, X4 - PXOR X6, X7 - PXOR X9, X10 - MOVOU X1, (DI) - MOVOU X4, 16(DI) - MOVOU X7, 32(DI) - MOVOU X10, 48(DI) - MOVOU 64(SI), X0 - MOVOU 80(SI), X3 - MOVOU 96(SI), X6 - MOVOU 112(SI), X9 - PXOR X0, X2 - PXOR X3, X5 - PXOR X6, X8 - PXOR X9, X11 - MOVOU X2, 64(DI) - MOVOU X5, 80(DI) - MOVOU X8, 96(DI) - MOVOU X11, 112(DI) - MOVQ $0x00000080, CX - SUBQ $0x80, BX - LEAQ 128(SI), SI - MOVO X12, X1 - MOVO X13, X4 - MOVO X14, X7 - MOVO X15, X10 - CMPQ BX, $0x40 - JBE sealSSE128SealHash - MOVOU (SI), X0 - MOVOU 16(SI), X3 - MOVOU 32(SI), X6 - MOVOU 48(SI), X9 - PXOR X0, X12 - PXOR X3, X13 - PXOR X6, X14 - PXOR X9, X15 - MOVOU X12, 128(DI) - MOVOU X13, 144(DI) - MOVOU X14, 160(DI) - MOVOU X15, 176(DI) - ADDQ $0x40, CX - SUBQ $0x40, BX - LEAQ 64(SI), SI - MOVQ $0x00000002, CX - MOVQ $0x00000008, R9 - CMPQ BX, $0x40 - JBE sealSSETail64 - CMPQ BX, $0x80 - JBE sealSSETail128 - CMPQ BX, $0xc0 - JBE sealSSETail192 - -sealSSEMainLoop: - // Load state, increment counter blocks - MOVO ·chacha20Constants<>+0(SB), X0 - MOVO 32(BP), X3 - MOVO 48(BP), X6 - MOVO 128(BP), X9 - PADDL ·sseIncMask<>+0(SB), X9 - MOVO X0, X1 - MOVO X3, X4 - MOVO X6, X7 - MOVO X9, X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X1, X2 - MOVO X4, X5 - MOVO X7, X8 - MOVO X10, X11 - PADDL ·sseIncMask<>+0(SB), X11 - MOVO X2, X12 - MOVO X5, X13 - MOVO X8, X14 - MOVO X11, X15 - PADDL ·sseIncMask<>+0(SB), X15 - - // Store counters - MOVO X9, 80(BP) - MOVO X10, 96(BP) - MOVO X11, 112(BP) - MOVO X15, 128(BP) - -sealSSEInnerLoop: - MOVO X14, 64(BP) - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X3 - PXOR X14, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X3 - PXOR X14, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X4 - PXOR X14, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X4 - PXOR X14, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X5 - PXOR X14, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X5 - PXOR X14, X5 - MOVO 64(BP), X14 - MOVO X7, 64(BP) - PADDD X13, X12 - PXOR X12, X15 - ROL16(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x0c, X7 - PSRLL $0x14, X13 - PXOR X7, X13 - PADDD X13, X12 - PXOR X12, X15 - ROL8(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x07, X7 - PSRLL $0x19, X13 - PXOR X7, X13 - MOVO 64(BP), X7 - ADDQ (DI), R10 - ADCQ 8(DI), R11 +openAVX2Tail256: + VMOVDQA ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VMOVDQA Y4, Y7 + VMOVDQA Y1, Y11 + + // Compute the number of iterations that will hash data + MOVQ BX, 224(BP) + MOVQ BX, CX + SUBQ $0x80, CX + SHRQ $0x04, CX + MOVQ $0x0000000a, R9 + CMPQ CX, $0x0a + CMOVQGT R9, CX + MOVQ SI, BX + XORQ R9, R9 + +openAVX2Tail256LoopA: + ADDQ (BX), R10 + ADCQ 8(BX), R11 ADCQ $0x01, R12 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x0c - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 IMULQ R12, R15 + MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX ADDQ AX, R15 - ADCQ $0x00, DX - LEAQ 16(DI), DI - MOVO X14, 64(BP) - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X3 - PXOR X14, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X14) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X3 - PXOR X14, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X4 - PXOR X14, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X14) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X4 - PXOR X14, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x0c, X14 - PSRLL $0x14, X5 - PXOR X14, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X14) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X14 - PSLLL $0x07, X14 - PSRLL $0x19, X5 - PXOR X14, X5 - MOVO 64(BP), X14 - MOVO X7, 64(BP) - IMULQ R12, R8 - ADDQ R10, R15 ADCQ DX, R8 - PADDD X13, X12 - PXOR X12, X15 - ROL16(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x0c, X7 - PSRLL $0x14, X13 - PXOR X7, X13 - PADDD X13, X12 - PXOR X12, X15 - ROL8(X15, X7) - PADDD X15, X14 - PXOR X14, X13 - MOVO X13, X7 - PSLLL $0x07, X7 - PSRLL $0x19, X13 - PXOR X7, X13 - MOVO 64(BP), X7 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 @@ -5555,112 +1518,117 @@ sealSSEInnerLoop: ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x04 - DECQ R9 - JGE sealSSEInnerLoop - ADDQ (DI), R10 - ADCQ 8(DI), R11 + LEAQ 16(BX), BX + +openAVX2Tail256LoopB: + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + INCQ R9 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + CMPQ R9, CX + JB openAVX2Tail256LoopA + CMPQ R9, $0x0a + JNE openAVX2Tail256LoopB + MOVQ BX, R9 + SUBQ SI, BX + MOVQ BX, CX + MOVQ 224(BP), BX + +openAVX2Tail256Hash: + ADDQ $0x10, CX + CMPQ CX, BX + JGT openAVX2Tail256HashEnd + ADDQ (R9), R10 + ADCQ 8(R9), R11 ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 IMULQ R12, R15 + MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 @@ -5672,143 +1640,94 @@ sealSSEInnerLoop: SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - LEAQ 16(DI), DI - DECQ CX - JG sealSSEInnerLoop - - // Add in the state - PADDD ·chacha20Constants<>+0(SB), X0 - PADDD ·chacha20Constants<>+0(SB), X1 - PADDD ·chacha20Constants<>+0(SB), X2 - PADDD ·chacha20Constants<>+0(SB), X12 - PADDD 32(BP), X3 - PADDD 32(BP), X4 - PADDD 32(BP), X5 - PADDD 32(BP), X13 - PADDD 48(BP), X6 - PADDD 48(BP), X7 - PADDD 48(BP), X8 - PADDD 48(BP), X14 - PADDD 80(BP), X9 - PADDD 96(BP), X10 - PADDD 112(BP), X11 - PADDD 128(BP), X15 - MOVO X15, 64(BP) - - // Load - xor - store - MOVOU (SI), X15 - PXOR X15, X0 - MOVOU 16(SI), X15 - PXOR X15, X3 - MOVOU 32(SI), X15 - PXOR X15, X6 - MOVOU 48(SI), X15 - PXOR X15, X9 - MOVOU X0, (DI) - MOVOU X3, 16(DI) - MOVOU X6, 32(DI) - MOVOU X9, 48(DI) - MOVO 64(BP), X15 - MOVOU 64(SI), X0 - MOVOU 80(SI), X3 - MOVOU 96(SI), X6 - MOVOU 112(SI), X9 - PXOR X0, X1 - PXOR X3, X4 - PXOR X6, X7 - PXOR X9, X10 - MOVOU X1, 64(DI) - MOVOU X4, 80(DI) - MOVOU X7, 96(DI) - MOVOU X10, 112(DI) - MOVOU 128(SI), X0 - MOVOU 144(SI), X3 - MOVOU 160(SI), X6 - MOVOU 176(SI), X9 - PXOR X0, X2 - PXOR X3, X5 - PXOR X6, X8 - PXOR X9, X11 - MOVOU X2, 128(DI) - MOVOU X5, 144(DI) - MOVOU X8, 160(DI) - MOVOU X11, 176(DI) - ADDQ $0xc0, SI - MOVQ $0x000000c0, CX - SUBQ $0xc0, BX - MOVO X12, X1 - MOVO X13, X4 - MOVO X14, X7 - MOVO X15, X10 - CMPQ BX, $0x40 - JBE sealSSE128SealHash - MOVOU (SI), X0 - MOVOU 16(SI), X3 - MOVOU 32(SI), X6 - MOVOU 48(SI), X9 - PXOR X0, X12 - PXOR X3, X13 - PXOR X6, X14 - PXOR X9, X15 - MOVOU X12, 192(DI) - MOVOU X13, 208(DI) - MOVOU X14, 224(DI) - MOVOU X15, 240(DI) - LEAQ 64(SI), SI - SUBQ $0x40, BX - MOVQ $0x00000006, CX - MOVQ $0x00000004, R9 - CMPQ BX, $0xc0 - JG sealSSEMainLoop - MOVQ BX, CX - TESTQ BX, BX - JE sealSSE128SealHash - MOVQ $0x00000006, CX - CMPQ BX, $0x40 - JBE sealSSETail64 - CMPQ BX, $0x80 - JBE sealSSETail128 - JMP sealSSETail192 - -sealSSETail64: - MOVO ·chacha20Constants<>+0(SB), X1 - MOVO 32(BP), X4 - MOVO 48(BP), X7 - MOVO 128(BP), X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X10, 80(BP) - -sealSSETail64LoopA: - ADDQ (DI), R10 - ADCQ 8(DI), R11 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(R9), R9 + JMP openAVX2Tail256Hash + +openAVX2Tail256HashEnd: + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD Y7, Y4, Y4 + VPADDD Y11, Y1, Y1 + VPERM2I128 $0x02, Y0, Y14, Y6 + VPERM2I128 $0x02, Y12, Y4, Y10 + VPERM2I128 $0x13, Y0, Y14, Y8 + VPERM2I128 $0x13, Y12, Y4, Y2 + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 + VPXOR (SI), Y6, Y6 + VPXOR 32(SI), Y10, Y10 + VPXOR 64(SI), Y8, Y8 + VPXOR 96(SI), Y2, Y2 + VMOVDQU Y6, (DI) + VMOVDQU Y10, 32(DI) + VMOVDQU Y8, 64(DI) + VMOVDQU Y2, 96(DI) + LEAQ 128(SI), SI + LEAQ 128(DI), DI + SUBQ $0x80, BX + JMP openAVX2TailLoop + +openAVX2Tail384: + // Need to decrypt up to 384 bytes - prepare six blocks + VMOVDQA ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA Y0, Y6 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA Y14, Y10 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA Y12, Y8 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VMOVDQA Y4, 96(BP) + VMOVDQA Y1, 128(BP) + VMOVDQA Y2, 160(BP) + + // Compute the number of iterations that will hash two blocks of data + MOVQ BX, 224(BP) + MOVQ BX, CX + SUBQ $0x00000100, CX + SHRQ $0x04, CX + ADDQ $0x06, CX + MOVQ $0x0000000a, R9 + CMPQ CX, $0x0a + CMOVQGT R9, CX + MOVQ SI, BX + XORQ R9, R9 + +openAVX2Tail384LoopB: + ADDQ (BX), R10 + ADCQ 8(BX), R11 ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 IMULQ R12, R15 + MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 @@ -5825,175 +1744,190 @@ sealSSETail64LoopA: ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 - LEAQ 16(DI), DI + LEAQ 16(BX), BX + +openAVX2Tail384LoopA: + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y3 + VPSRLD $0x19, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x0c, Y2, Y2, Y2 + ADDQ (BX), R10 + ADCQ 8(BX), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(BX), BX + INCQ R9 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y3 + VPSRLD $0x19, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x04, Y2, Y2, Y2 + CMPQ R9, CX + JB openAVX2Tail384LoopB + CMPQ R9, $0x0a + JNE openAVX2Tail384LoopA + MOVQ BX, R9 + SUBQ SI, BX + MOVQ BX, CX + MOVQ 224(BP), BX -sealSSETail64LoopB: - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X13) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X13 - PSLLL $0x0c, X13 - PSRLL $0x14, X4 - PXOR X13, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X13) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X13 - PSLLL $0x07, X13 - PSRLL $0x19, X4 - PXOR X13, X4 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X13) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X13 - PSLLL $0x0c, X13 - PSRLL $0x14, X4 - PXOR X13, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X13) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X13 - PSLLL $0x07, X13 - PSRLL $0x19, X4 - PXOR X13, X4 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - ADDQ (DI), R10 - ADCQ 8(DI), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - LEAQ 16(DI), DI - DECQ CX - JG sealSSETail64LoopA - DECQ R9 - JGE sealSSETail64LoopB - PADDL ·chacha20Constants<>+0(SB), X1 - PADDL 32(BP), X4 - PADDL 48(BP), X7 - PADDL 80(BP), X10 - JMP sealSSE128Seal - -sealSSETail128: - MOVO ·chacha20Constants<>+0(SB), X0 - MOVO 32(BP), X3 - MOVO 48(BP), X6 - MOVO 128(BP), X9 - PADDL ·sseIncMask<>+0(SB), X9 - MOVO X9, 80(BP) - MOVO X0, X1 - MOVO X3, X4 - MOVO X6, X7 - MOVO X9, X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X10, 96(BP) - -sealSSETail128LoopA: - ADDQ (DI), R10 - ADCQ 8(DI), R11 +openAVX2Tail384Hash: + ADDQ $0x10, CX + CMPQ CX, BX + JGT openAVX2Tail384HashEnd + ADDQ (R9), R10 + ADCQ 8(R9), R11 ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 IMULQ R12, R15 + MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 @@ -6010,430 +1944,99 @@ sealSSETail128LoopA: ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 - LEAQ 16(DI), DI + LEAQ 16(R9), R9 + JMP openAVX2Tail384Hash -sealSSETail128LoopB: - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - ADDQ (DI), R10 - ADCQ 8(DI), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - LEAQ 16(DI), DI - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - DECQ CX - JG sealSSETail128LoopA - DECQ R9 - JGE sealSSETail128LoopB - PADDL ·chacha20Constants<>+0(SB), X0 - PADDL ·chacha20Constants<>+0(SB), X1 - PADDL 32(BP), X3 - PADDL 32(BP), X4 - PADDL 48(BP), X6 - PADDL 48(BP), X7 - PADDL 80(BP), X9 - PADDL 96(BP), X10 - MOVOU (SI), X12 - MOVOU 16(SI), X13 - MOVOU 32(SI), X14 - MOVOU 48(SI), X15 - PXOR X12, X0 - PXOR X13, X3 - PXOR X14, X6 - PXOR X15, X9 - MOVOU X0, (DI) - MOVOU X3, 16(DI) - MOVOU X6, 32(DI) - MOVOU X9, 48(DI) - MOVQ $0x00000040, CX - LEAQ 64(SI), SI - SUBQ $0x40, BX - JMP sealSSE128SealHash - -sealSSETail192: - MOVO ·chacha20Constants<>+0(SB), X0 - MOVO 32(BP), X3 - MOVO 48(BP), X6 - MOVO 128(BP), X9 - PADDL ·sseIncMask<>+0(SB), X9 - MOVO X9, 80(BP) - MOVO X0, X1 - MOVO X3, X4 - MOVO X6, X7 - MOVO X9, X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X10, 96(BP) - MOVO X1, X2 - MOVO X4, X5 - MOVO X7, X8 - MOVO X10, X11 - PADDL ·sseIncMask<>+0(SB), X11 - MOVO X11, 112(BP) - -sealSSETail192LoopA: - ADDQ (DI), R10 - ADCQ 8(DI), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - LEAQ 16(DI), DI +openAVX2Tail384HashEnd: + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 32(BP), Y10, Y10 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD 64(BP), Y8, Y8 + VPADDD 96(BP), Y4, Y4 + VPADDD 128(BP), Y1, Y1 + VPADDD 160(BP), Y2, Y2 + VPERM2I128 $0x02, Y0, Y14, Y3 + VPERM2I128 $0x02, Y12, Y4, Y7 + VPERM2I128 $0x13, Y0, Y14, Y11 + VPERM2I128 $0x13, Y12, Y4, Y15 + VPXOR (SI), Y3, Y3 + VPXOR 32(SI), Y7, Y7 + VPXOR 64(SI), Y11, Y11 + VPXOR 96(SI), Y15, Y15 + VMOVDQU Y3, (DI) + VMOVDQU Y7, 32(DI) + VMOVDQU Y11, 64(DI) + VMOVDQU Y15, 96(DI) + VPERM2I128 $0x02, Y5, Y9, Y3 + VPERM2I128 $0x02, Y13, Y1, Y7 + VPERM2I128 $0x13, Y5, Y9, Y11 + VPERM2I128 $0x13, Y13, Y1, Y15 + VPXOR 128(SI), Y3, Y3 + VPXOR 160(SI), Y7, Y7 + VPXOR 192(SI), Y11, Y11 + VPXOR 224(SI), Y15, Y15 + VMOVDQU Y3, 128(DI) + VMOVDQU Y7, 160(DI) + VMOVDQU Y11, 192(DI) + VMOVDQU Y15, 224(DI) + VPERM2I128 $0x02, Y6, Y10, Y0 + VPERM2I128 $0x02, Y8, Y2, Y14 + VPERM2I128 $0x13, Y6, Y10, Y12 + VPERM2I128 $0x13, Y8, Y2, Y4 + LEAQ 256(SI), SI + LEAQ 256(DI), DI + SUBQ $0x00000100, BX + JMP openAVX2TailLoop -sealSSETail192LoopB: - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X5 - PXOR X12, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X5 - PXOR X12, X5 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - ADDQ (DI), R10 - ADCQ 8(DI), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 +openAVX2Tail512: + VMOVDQU ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA Y0, Y6 + VMOVDQA Y0, Y7 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA Y14, Y10 + VMOVDQA Y14, Y11 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA Y12, Y8 + VMOVDQA Y12, Y15 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 + VMOVDQA Y4, 96(BP) + VMOVDQA Y1, 128(BP) + VMOVDQA Y2, 160(BP) + VMOVDQA Y3, 192(BP) + XORQ CX, CX + MOVQ SI, R9 + +openAVX2Tail512LoopB: + ADDQ (R9), R10 + ADCQ 8(R9), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 IMULQ R12, R15 + MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 @@ -6450,465 +2053,268 @@ sealSSETail192LoopB: ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 - LEAQ 16(DI), DI - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X5 - PXOR X12, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X5 - PXOR X12, X5 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - DECQ CX - JG sealSSETail192LoopA - DECQ R9 - JGE sealSSETail192LoopB - PADDL ·chacha20Constants<>+0(SB), X0 - PADDL ·chacha20Constants<>+0(SB), X1 - PADDL ·chacha20Constants<>+0(SB), X2 - PADDL 32(BP), X3 - PADDL 32(BP), X4 - PADDL 32(BP), X5 - PADDL 48(BP), X6 - PADDL 48(BP), X7 - PADDL 48(BP), X8 - PADDL 80(BP), X9 - PADDL 96(BP), X10 - PADDL 112(BP), X11 - MOVOU (SI), X12 - MOVOU 16(SI), X13 - MOVOU 32(SI), X14 - MOVOU 48(SI), X15 - PXOR X12, X0 - PXOR X13, X3 - PXOR X14, X6 - PXOR X15, X9 - MOVOU X0, (DI) - MOVOU X3, 16(DI) - MOVOU X6, 32(DI) - MOVOU X9, 48(DI) - MOVOU 64(SI), X12 - MOVOU 80(SI), X13 - MOVOU 96(SI), X14 - MOVOU 112(SI), X15 - PXOR X12, X1 - PXOR X13, X4 - PXOR X14, X7 - PXOR X15, X10 - MOVOU X1, 64(DI) - MOVOU X4, 80(DI) - MOVOU X7, 96(DI) - MOVOU X10, 112(DI) - MOVO X2, X1 - MOVO X5, X4 - MOVO X8, X7 - MOVO X11, X10 - MOVQ $0x00000080, CX - LEAQ 128(SI), SI - SUBQ $0x80, BX - JMP sealSSE128SealHash - -sealSSE128: - MOVOU ·chacha20Constants<>+0(SB), X0 - MOVOU 16(R8), X3 - MOVOU 32(R8), X6 - MOVOU 48(R8), X9 - MOVO X0, X1 - MOVO X3, X4 - MOVO X6, X7 - MOVO X9, X10 - PADDL ·sseIncMask<>+0(SB), X10 - MOVO X1, X2 - MOVO X4, X5 - MOVO X7, X8 - MOVO X10, X11 - PADDL ·sseIncMask<>+0(SB), X11 - MOVO X3, X13 - MOVO X6, X14 - MOVO X10, X15 - MOVQ $0x0000000a, R9 - -sealSSE128InnerCipherLoop: - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X5 - PXOR X12, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X5 - PXOR X12, X5 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x04 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x0c - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - PADDD X3, X0 - PXOR X0, X9 - ROL16(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X3 - PXOR X12, X3 - PADDD X3, X0 - PXOR X0, X9 - ROL8(X9, X12) - PADDD X9, X6 - PXOR X6, X3 - MOVO X3, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X3 - PXOR X12, X3 - PADDD X4, X1 - PXOR X1, X10 - ROL16(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X4 - PXOR X12, X4 - PADDD X4, X1 - PXOR X1, X10 - ROL8(X10, X12) - PADDD X10, X7 - PXOR X7, X4 - MOVO X4, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X4 - PXOR X12, X4 - PADDD X5, X2 - PXOR X2, X11 - ROL16(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x0c, X12 - PSRLL $0x14, X5 - PXOR X12, X5 - PADDD X5, X2 - PXOR X2, X11 - ROL8(X11, X12) - PADDD X11, X8 - PXOR X8, X5 - MOVO X5, X12 - PSLLL $0x07, X12 - PSRLL $0x19, X5 - PXOR X12, X5 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xe4 - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xed - BYTE $0x0c - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xf6 - BYTE $0x08 - BYTE $0x66 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xff - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc0 - BYTE $0x08 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xc9 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xd2 - BYTE $0x04 - BYTE $0x66 - BYTE $0x45 - BYTE $0x0f - BYTE $0x3a - BYTE $0x0f - BYTE $0xdb - BYTE $0x04 - DECQ R9 - JNE sealSSE128InnerCipherLoop - - // A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded - PADDL ·chacha20Constants<>+0(SB), X0 - PADDL ·chacha20Constants<>+0(SB), X1 - PADDL ·chacha20Constants<>+0(SB), X2 - PADDL X13, X3 - PADDL X13, X4 - PADDL X13, X5 - PADDL X14, X7 - PADDL X14, X8 - PADDL X15, X10 - PADDL ·sseIncMask<>+0(SB), X15 - PADDL X15, X11 - PAND ·polyClampMask<>+0(SB), X0 - MOVOU X0, (BP) - MOVOU X3, 16(BP) + LEAQ 16(R9), R9 - // Hash - MOVQ ad_len+80(FP), R9 - CALL polyHashADInternal<>(SB) - XORQ CX, CX +openAVX2Tail512LoopA: + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + ADDQ (R9), R10 + ADCQ 8(R9), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x04, Y11, Y11, Y11 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPALIGNR $0x0c, Y3, Y3, Y3 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + ADDQ 16(R9), R10 + ADCQ 24(R9), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 32(R9), R9 + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x0c, Y11, Y11, Y11 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x04, Y2, Y2, Y2 + VPALIGNR $0x04, Y3, Y3, Y3 + INCQ CX + CMPQ CX, $0x04 + JLT openAVX2Tail512LoopB + CMPQ CX, $0x0a + JNE openAVX2Tail512LoopA + MOVQ BX, CX + SUBQ $0x00000180, CX + ANDQ $-16, CX -sealSSE128SealHash: - CMPQ CX, $0x10 - JB sealSSE128Seal - ADDQ (DI), R10 - ADCQ 8(DI), R11 +openAVX2Tail512HashLoop: + TESTQ CX, CX + JE openAVX2Tail512HashEnd + ADDQ (R9), R10 + ADCQ 8(R9), R11 ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 IMULQ R12, R15 + MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 @@ -6925,238 +2331,162 @@ sealSSE128SealHash: ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 + LEAQ 16(R9), R9 SUBQ $0x10, CX - ADDQ $0x10, DI - JMP sealSSE128SealHash - -sealSSE128Seal: - CMPQ BX, $0x10 - JB sealSSETail - SUBQ $0x10, BX - - // Load for decryption - MOVOU (SI), X12 - PXOR X12, X1 - MOVOU X1, (DI) - LEAQ 16(SI), SI - LEAQ 16(DI), DI - - // Extract for hashing - MOVQ X1, R13 - PSRLDQ $0x08, X1 - MOVQ X1, R14 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - - // Shift the stream "left" - MOVO X4, X1 - MOVO X7, X4 - MOVO X10, X7 - MOVO X2, X10 - MOVO X5, X2 - MOVO X8, X5 - MOVO X11, X8 - JMP sealSSE128Seal + JMP openAVX2Tail512HashLoop -sealSSETail: - TESTQ BX, BX - JE sealSSEFinalize +openAVX2Tail512HashEnd: + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 + VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 32(BP), Y10, Y10 + VPADDD 32(BP), Y11, Y11 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD 64(BP), Y8, Y8 + VPADDD 64(BP), Y15, Y15 + VPADDD 96(BP), Y4, Y4 + VPADDD 128(BP), Y1, Y1 + VPADDD 160(BP), Y2, Y2 + VPADDD 192(BP), Y3, Y3 + VMOVDQA Y15, 224(BP) + VPERM2I128 $0x02, Y0, Y14, Y15 + VPERM2I128 $0x13, Y0, Y14, Y14 + VPERM2I128 $0x02, Y12, Y4, Y0 + VPERM2I128 $0x13, Y12, Y4, Y12 + VPXOR (SI), Y15, Y15 + VPXOR 32(SI), Y0, Y0 + VPXOR 64(SI), Y14, Y14 + VPXOR 96(SI), Y12, Y12 + VMOVDQU Y15, (DI) + VMOVDQU Y0, 32(DI) + VMOVDQU Y14, 64(DI) + VMOVDQU Y12, 96(DI) + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 + VPXOR 128(SI), Y0, Y0 + VPXOR 160(SI), Y14, Y14 + VPXOR 192(SI), Y12, Y12 + VPXOR 224(SI), Y4, Y4 + VMOVDQU Y0, 128(DI) + VMOVDQU Y14, 160(DI) + VMOVDQU Y12, 192(DI) + VMOVDQU Y4, 224(DI) + VPERM2I128 $0x02, Y6, Y10, Y0 + VPERM2I128 $0x02, Y8, Y2, Y14 + VPERM2I128 $0x13, Y6, Y10, Y12 + VPERM2I128 $0x13, Y8, Y2, Y4 + VPXOR 256(SI), Y0, Y0 + VPXOR 288(SI), Y14, Y14 + VPXOR 320(SI), Y12, Y12 + VPXOR 352(SI), Y4, Y4 + VMOVDQU Y0, 256(DI) + VMOVDQU Y14, 288(DI) + VMOVDQU Y12, 320(DI) + VMOVDQU Y4, 352(DI) + VPERM2I128 $0x02, Y7, Y11, Y0 + VPERM2I128 $0x02, 224(BP), Y3, Y14 + VPERM2I128 $0x13, Y7, Y11, Y12 + VPERM2I128 $0x13, 224(BP), Y3, Y4 + LEAQ 384(SI), SI + LEAQ 384(DI), DI + SUBQ $0x00000180, BX + JMP openAVX2TailLoop - // We can only load the PT one byte at a time to avoid read after end of buffer - MOVQ BX, R9 - SHLQ $0x04, R9 - LEAQ ·andMask<>+0(SB), R13 - MOVQ BX, CX - LEAQ -1(SI)(BX*1), SI - XORQ R15, R15 - XORQ R8, R8 - XORQ AX, AX +DATA ·chacha20Constants<>+0(SB)/4, $0x61707865 +DATA ·chacha20Constants<>+4(SB)/4, $0x3320646e +DATA ·chacha20Constants<>+8(SB)/4, $0x79622d32 +DATA ·chacha20Constants<>+12(SB)/4, $0x6b206574 +DATA ·chacha20Constants<>+16(SB)/4, $0x61707865 +DATA ·chacha20Constants<>+20(SB)/4, $0x3320646e +DATA ·chacha20Constants<>+24(SB)/4, $0x79622d32 +DATA ·chacha20Constants<>+28(SB)/4, $0x6b206574 +GLOBL ·chacha20Constants<>(SB), RODATA|NOPTR, $32 -sealSSETailLoadLoop: - SHLQ $0x08, R15, R8 - SHLQ $0x08, R15 - MOVB (SI), AX - XORQ AX, R15 - LEAQ -1(SI), SI - DECQ CX - JNE sealSSETailLoadLoop - MOVQ R15, 64(BP) - MOVQ R8, 72(BP) - PXOR 64(BP), X1 - MOVOU X1, (DI) - MOVOU -16(R13)(R9*1), X12 - PAND X12, X1 - MOVQ X1, R13 - PSRLDQ $0x08, X1 - MOVQ X1, R14 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 - ADDQ BX, DI +DATA ·avx2InitMask<>+0(SB)/8, $0x0000000000000000 +DATA ·avx2InitMask<>+8(SB)/8, $0x0000000000000000 +DATA ·avx2InitMask<>+16(SB)/8, $0x0000000000000001 +DATA ·avx2InitMask<>+24(SB)/8, $0x0000000000000000 +GLOBL ·avx2InitMask<>(SB), RODATA|NOPTR, $32 -sealSSEFinalize: - // Hash in the buffer lengths - ADDQ ad_len+80(FP), R10 - ADCQ src_len+56(FP), R11 - ADCQ $0x01, R12 - MOVQ (BP), AX - MOVQ AX, R15 - MULQ R10 - MOVQ AX, R13 - MOVQ DX, R14 - MOVQ (BP), AX - MULQ R11 - IMULQ R12, R15 - ADDQ AX, R14 - ADCQ DX, R15 - MOVQ 8(BP), AX - MOVQ AX, R8 - MULQ R10 - ADDQ AX, R14 - ADCQ $0x00, DX - MOVQ DX, R10 - MOVQ 8(BP), AX - MULQ R11 - ADDQ AX, R15 - ADCQ $0x00, DX - IMULQ R12, R8 - ADDQ R10, R15 - ADCQ DX, R8 - MOVQ R13, R10 - MOVQ R14, R11 - MOVQ R15, R12 - ANDQ $0x03, R12 - MOVQ R15, R13 - ANDQ $-4, R13 - MOVQ R8, R14 - SHRQ $0x02, R8, R15 - SHRQ $0x02, R8 - ADDQ R13, R10 - ADCQ R14, R11 - ADCQ $0x00, R12 - ADDQ R15, R10 - ADCQ R8, R11 - ADCQ $0x00, R12 +DATA ·rol16<>+0(SB)/8, $0x0504070601000302 +DATA ·rol16<>+8(SB)/8, $0x0d0c0f0e09080b0a +DATA ·rol16<>+16(SB)/8, $0x0504070601000302 +DATA ·rol16<>+24(SB)/8, $0x0d0c0f0e09080b0a +GLOBL ·rol16<>(SB), RODATA|NOPTR, $32 - // Final reduce - MOVQ R10, R13 - MOVQ R11, R14 - MOVQ R12, R15 - SUBQ $-5, R10 - SBBQ $-1, R11 - SBBQ $0x03, R12 - CMOVQCS R13, R10 - CMOVQCS R14, R11 - CMOVQCS R15, R12 +DATA ·rol8<>+0(SB)/8, $0x0605040702010003 +DATA ·rol8<>+8(SB)/8, $0x0e0d0c0f0a09080b +DATA ·rol8<>+16(SB)/8, $0x0605040702010003 +DATA ·rol8<>+24(SB)/8, $0x0e0d0c0f0a09080b +GLOBL ·rol8<>(SB), RODATA|NOPTR, $32 - // Add in the "s" part of the key - ADDQ 16(BP), R10 - ADCQ 24(BP), R11 +DATA ·polyClampMask<>+0(SB)/8, $0x0ffffffc0fffffff +DATA ·polyClampMask<>+8(SB)/8, $0x0ffffffc0ffffffc +DATA ·polyClampMask<>+16(SB)/8, $0xffffffffffffffff +DATA ·polyClampMask<>+24(SB)/8, $0xffffffffffffffff +GLOBL ·polyClampMask<>(SB), RODATA|NOPTR, $32 - // Finally store the tag at the end of the message - MOVQ R10, (DI) - MOVQ R11, 8(DI) - RET +DATA ·avx2IncMask<>+0(SB)/8, $0x0000000000000002 +DATA ·avx2IncMask<>+8(SB)/8, $0x0000000000000000 +DATA ·avx2IncMask<>+16(SB)/8, $0x0000000000000002 +DATA ·avx2IncMask<>+24(SB)/8, $0x0000000000000000 +GLOBL ·avx2IncMask<>(SB), RODATA|NOPTR, $32 + +DATA ·andMask<>+0(SB)/8, $0x00000000000000ff +DATA ·andMask<>+8(SB)/8, $0x0000000000000000 +DATA ·andMask<>+16(SB)/8, $0x000000000000ffff +DATA ·andMask<>+24(SB)/8, $0x0000000000000000 +DATA ·andMask<>+32(SB)/8, $0x0000000000ffffff +DATA ·andMask<>+40(SB)/8, $0x0000000000000000 +DATA ·andMask<>+48(SB)/8, $0x00000000ffffffff +DATA ·andMask<>+56(SB)/8, $0x0000000000000000 +DATA ·andMask<>+64(SB)/8, $0x000000ffffffffff +DATA ·andMask<>+72(SB)/8, $0x0000000000000000 +DATA ·andMask<>+80(SB)/8, $0x0000ffffffffffff +DATA ·andMask<>+88(SB)/8, $0x0000000000000000 +DATA ·andMask<>+96(SB)/8, $0x00ffffffffffffff +DATA ·andMask<>+104(SB)/8, $0x0000000000000000 +DATA ·andMask<>+112(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+120(SB)/8, $0x0000000000000000 +DATA ·andMask<>+128(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+136(SB)/8, $0x00000000000000ff +DATA ·andMask<>+144(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+152(SB)/8, $0x000000000000ffff +DATA ·andMask<>+160(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+168(SB)/8, $0x0000000000ffffff +DATA ·andMask<>+176(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+184(SB)/8, $0x00000000ffffffff +DATA ·andMask<>+192(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+200(SB)/8, $0x000000ffffffffff +DATA ·andMask<>+208(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+216(SB)/8, $0x0000ffffffffffff +DATA ·andMask<>+224(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+232(SB)/8, $0x00ffffffffffffff +GLOBL ·andMask<>(SB), RODATA|NOPTR, $240 -chacha20Poly1305Seal_AVX2: +// func chacha20Poly1305Seal(dst []byte, key []uint32, src []byte, ad []byte) +// Requires: AVX, AVX2, BMI2, CMOV, SSE2 +TEXT ·chacha20Poly1305Seal(SB), $288-96 + MOVQ SP, BP + ADDQ $0x20, BP + ANDQ $-32, BP + MOVQ dst_base+0(FP), DI + MOVQ key_base+24(FP), R8 + MOVQ src_base+48(FP), SI + MOVQ src_len+56(FP), BX + MOVQ ad_base+72(FP), CX VZEROUPPER - VMOVDQU ·chacha20Constants<>+0(SB), Y0 - BYTE $0xc4 - BYTE $0x42 - BYTE $0x7d - BYTE $0x5a - BYTE $0x70 - BYTE $0x10 - BYTE $0xc4 - BYTE $0x42 - BYTE $0x7d - BYTE $0x5a - BYTE $0x60 - BYTE $0x20 - BYTE $0xc4 - BYTE $0xc2 - BYTE $0x7d - BYTE $0x5a - BYTE $0x60 - BYTE $0x30 - VPADDD ·avx2InitMask<>+0(SB), Y4, Y4 + VMOVDQU ·chacha20Constants<>+0(SB), Y0 + VBROADCASTI128 16(R8), Y14 + VBROADCASTI128 32(R8), Y12 + VBROADCASTI128 48(R8), Y4 + VPADDD ·avx2InitMask<>+0(SB), Y4, Y4 // Special optimizations, for very short buffers CMPQ BX, $0x000000c0 @@ -8170,6 +3500,144 @@ sealAVX2InternalLoopStart: JBE sealAVX2Tail384 JMP sealAVX2Tail512 +sealSSETail: + TESTQ BX, BX + JE sealSSEFinalize + + // We can only load the PT one byte at a time to avoid read after end of buffer + MOVQ BX, R9 + SHLQ $0x04, R9 + LEAQ ·andMask<>+0(SB), R13 + MOVQ BX, CX + LEAQ -1(SI)(BX*1), SI + XORQ R15, R15 + XORQ R8, R8 + XORQ AX, AX + +sealSSETailLoadLoop: + SHLQ $0x08, R15, R8 + SHLQ $0x08, R15 + MOVB (SI), AX + XORQ AX, R15 + LEAQ -1(SI), SI + DECQ CX + JNE sealSSETailLoadLoop + MOVQ R15, 64(BP) + MOVQ R8, 72(BP) + PXOR 64(BP), X1 + MOVOU X1, (DI) + MOVOU -16(R13)(R9*1), X12 + PAND X12, X1 + MOVQ X1, R13 + PSRLDQ $0x08, X1 + MOVQ X1, R14 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + ADDQ BX, DI + +sealSSEFinalize: + // Hash in the buffer lengths + ADDQ ad_len+80(FP), R10 + ADCQ src_len+56(FP), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + + // Final reduce + MOVQ R10, R13 + MOVQ R11, R14 + MOVQ R12, R15 + SUBQ $-5, R10 + SBBQ $-1, R11 + SBBQ $0x03, R12 + CMOVQCS R13, R10 + CMOVQCS R14, R11 + CMOVQCS R15, R12 + + // Add in the "s" part of the key + ADDQ 16(BP), R10 + ADCQ 24(BP), R11 + + // Finally store the tag at the end of the message + MOVQ R10, (DI) + MOVQ R11, 8(DI) + RET + seal192AVX2: VMOVDQA Y0, Y5 VMOVDQA Y14, Y9 diff --git a/vendor/golang.org/x/crypto/pkcs12/crypto.go b/vendor/golang.org/x/crypto/pkcs12/crypto.go index 212538cb5a871..3f307323cc397 100644 --- a/vendor/golang.org/x/crypto/pkcs12/crypto.go +++ b/vendor/golang.org/x/crypto/pkcs12/crypto.go @@ -80,6 +80,10 @@ func pbDecrypterFor(algorithm pkix.AlgorithmIdentifier, password []byte) (cipher return nil, 0, err } + if params.Iterations < 0 || params.Iterations > maxIterations { + return nil, 0, NotImplementedError("iteration count is invalid or too high") + } + key := cipherType.deriveKey(params.Salt, password, params.Iterations) iv := cipherType.deriveIV(params.Salt, password, params.Iterations) diff --git a/vendor/golang.org/x/crypto/pkcs12/mac.go b/vendor/golang.org/x/crypto/pkcs12/mac.go index 5f38aa7de83ce..84039cee1e5da 100644 --- a/vendor/golang.org/x/crypto/pkcs12/mac.go +++ b/vendor/golang.org/x/crypto/pkcs12/mac.go @@ -27,11 +27,19 @@ var ( oidSHA1 = asn1.ObjectIdentifier([]int{1, 3, 14, 3, 2, 26}) ) +// maxIterations is a safety limit to prevent CPU exhaustion from +// crafted PKCS#12 files with unreasonable iteration counts. +const maxIterations = 1 << 20 // ~1 million + func verifyMac(macData *macData, message, password []byte) error { if !macData.Mac.Algorithm.Algorithm.Equal(oidSHA1) { return NotImplementedError("unknown digest algorithm: " + macData.Mac.Algorithm.Algorithm.String()) } + if macData.Iterations < 0 || macData.Iterations > maxIterations { + return NotImplementedError("iteration count is invalid or too high") + } + key := pbkdf(sha1Sum, 20, 64, macData.MacSalt, password, macData.Iterations, 3, 20) mac := hmac.New(sha1.New, key) diff --git a/vendor/golang.org/x/crypto/ssh/certs.go b/vendor/golang.org/x/crypto/ssh/certs.go index 139fa31e1ba3d..6f75d77ec1a5e 100644 --- a/vendor/golang.org/x/crypto/ssh/certs.go +++ b/vendor/golang.org/x/crypto/ssh/certs.go @@ -348,6 +348,9 @@ func (c *CertChecker) CheckHostKey(addr string, remote net.Addr, key PublicKey) if cert.CertType != HostCert { return fmt.Errorf("ssh: certificate presented as a host key has type %d", cert.CertType) } + if c.IsHostAuthority == nil { + return errors.New("ssh: cannot verify certificate, IsHostAuthority not set") + } if !c.IsHostAuthority(cert.SignatureKey, addr) { return fmt.Errorf("ssh: no authorities for hostname: %v", addr) } @@ -375,6 +378,9 @@ func (c *CertChecker) Authenticate(conn ConnMetadata, pubKey PublicKey) (*Permis if cert.CertType != UserCert { return nil, fmt.Errorf("ssh: cert has type %d", cert.CertType) } + if c.IsUserAuthority == nil { + return nil, errors.New("ssh: cannot verify certificate, IsUserAuthority not set") + } if !c.IsUserAuthority(cert.SignatureKey) { return nil, fmt.Errorf("ssh: certificate signed by unrecognized authority") } @@ -438,7 +444,17 @@ func (c *CertChecker) CheckCert(principal string, cert *Certificate) error { if before := int64(cert.ValidBefore); cert.ValidBefore != uint64(CertTimeInfinity) && (unixNow >= before || before < 0) { return fmt.Errorf("ssh: cert has expired") } - if err := cert.SignatureKey.Verify(cert.bytesForSigning(), cert.Signature); err != nil { + // Match OpenSSH: the SK user-presence flag is never enforced on a + // certificate's CA signature. OpenSSH calls sshkey_verify with + // detailsp==NULL in sshkey.c:cert_parse, so the UP/UV flags are + // not even extracted. The UP bit on a CA signature reflects the + // CA operator's presence at signing time, which has no bearing on + // whether the user being authenticated is present now; enforcing + // it here would only break interop with certificates issued by + // non-interactive SK CAs. skKeyWithoutUP is a no-op for non-SK + // keys (the common case). + caKey := skKeyWithoutUP(cert.SignatureKey) + if err := caKey.Verify(cert.bytesForSigning(), cert.Signature); err != nil { return fmt.Errorf("ssh: certificate signature does not verify") } diff --git a/vendor/golang.org/x/crypto/ssh/channel.go b/vendor/golang.org/x/crypto/ssh/channel.go index cc0bb7ab64814..afc9aef1853a4 100644 --- a/vendor/golang.org/x/crypto/ssh/channel.go +++ b/vendor/golang.org/x/crypto/ssh/channel.go @@ -11,6 +11,7 @@ import ( "io" "log" "sync" + "sync/atomic" ) const ( @@ -131,11 +132,17 @@ func (r RejectionReason) String() string { return fmt.Sprintf("unknown reason %d", int(r)) } -func min(a uint32, b int) uint32 { - if a < uint32(b) { - return a +// minPayloadSize returns min(limit, length) clamped to a uint32. It is used +// to compute the size of the next channel data packet from the remaining +// payload. The comparison is done in int64 because length is an int — on +// 64-bit systems len(data) can exceed 2^32, and a direct uint32(length) +// cast would silently truncate to 0 at every multiple of 2^32, causing +// WriteExtended's loop to spin without making progress. +func minPayloadSize(limit uint32, length int) uint32 { + if int64(length) > int64(limit) { + return limit } - return uint32(b) + return uint32(length) } type channelDirection uint8 @@ -177,6 +184,12 @@ type channel struct { // with WantReply=true outstanding. This lock is held by a // goroutine that has such an outgoing request pending. sentRequestMu sync.Mutex + // sentRequestPending is set to true while a SendRequest call with + // WantReply=true is in flight. handlePacket uses it as a gate: responses + // arriving while no request is pending are dropped to prevent a + // misbehaving peer from stalling the mux read loop by filling ch.msg + // with unsolicited channelRequestSuccess/Failure messages. + sentRequestPending atomic.Bool incomingRequests chan *Request @@ -251,7 +264,7 @@ func (ch *channel) WriteExtended(data []byte, extendedCode uint32) (n int, err e ch.writeMu.Unlock() for len(data) > 0 { - space := min(ch.maxRemotePayload, len(data)) + space := minPayloadSize(ch.maxRemotePayload, len(data)) if space, err = ch.remoteWin.reserve(space); err != nil { return n, err } @@ -460,6 +473,18 @@ func (ch *channel) handlePacket(packet []byte) error { } ch.incomingRequests <- &req + case *channelRequestSuccessMsg, *channelRequestFailureMsg: + // Drop responses that arrive when no SendRequest is waiting, to + // prevent a malicious peer from filling ch.msg and stalling the + // mux read loop. The non-blocking send additionally protects the + // loop if a well-behaved caller is slow to read. + if !ch.sentRequestPending.Load() { + return nil + } + select { + case ch.msg <- msg: + default: + } default: ch.msg <- msg } @@ -530,7 +555,17 @@ func (ch *channel) Reject(reason RejectionReason, message string) error { Language: "en", } ch.decided = true - return ch.sendMessage(reject) + err := ch.sendMessage(reject) + + // Remove the channel from the mux to prevent memory leaks. + // Do not call ch.close() here: no goroutine holds a reference to a + // rejected channel's internal channels (msg, incomingRequests), so + // removing it from chanList is sufficient for GC. Calling close() + // would race with the mux loop goroutine (handlePacket or dropAll), + // causing a panic from closing an already-closed channel. + ch.mux.chanList.remove(ch.localId) + + return err } func (ch *channel) Read(data []byte) (int, error) { @@ -586,6 +621,27 @@ func (ch *channel) SendRequest(name string, wantReply bool, payload []byte) (boo if wantReply { ch.sentRequestMu.Lock() defer ch.sentRequestMu.Unlock() + + // Open the gate so that responses arriving while this request is in + // flight are allowed to reach ch.msg. Responses arriving while no + // request is pending are dropped by handlePacket. + ch.sentRequestPending.Store(true) + defer ch.sentRequestPending.Store(false) + + // Drain any spurious responses that may have been buffered. This + // prevents a previously buffered unexpected response from being + // consumed instead of the actual response for this request. + drain: + for { + select { + case _, ok := <-ch.msg: + if !ok { + break drain + } + default: + break drain + } + } } msg := channelRequestMsg{ diff --git a/vendor/golang.org/x/crypto/ssh/cipher.go b/vendor/golang.org/x/crypto/ssh/cipher.go index ad2b37057831d..48d019954599a 100644 --- a/vendor/golang.org/x/crypto/ssh/cipher.go +++ b/vendor/golang.org/x/crypto/ssh/cipher.go @@ -407,7 +407,7 @@ func (c *gcmCipher) readCipherPacket(seqNum uint32, r io.Reader) ([]byte, error) return nil, fmt.Errorf("ssh: illegal padding %d", padding) } - if int(padding+1) >= len(plain) { + if int(padding)+1 >= len(plain) { return nil, fmt.Errorf("ssh: padding %d too large", padding) } plain = plain[1 : length-uint32(padding)] diff --git a/vendor/golang.org/x/crypto/ssh/client.go b/vendor/golang.org/x/crypto/ssh/client.go index 33079789bce1b..89f0def9f690d 100644 --- a/vendor/golang.org/x/crypto/ssh/client.go +++ b/vendor/golang.org/x/crypto/ssh/client.go @@ -88,6 +88,32 @@ func NewClientConn(c net.Conn, addr string, config *ClientConfig) (Conn, <-chan return conn, conn.mux.incomingChannels, conn.mux.incomingRequests, nil } +// NewControlClientConn establishes an SSH connection over an OpenSSH +// ControlMaster socket c in proxy mode. +// +// Note that this package only implements the client side of the multiplexing +// protocol. The provided net.Conn must be a local, secure connection (such as a +// Unix domain socket) connected to an already-running OpenSSH process acting as +// the ControlMaster. +// +// WARNING: Because proxy mode bypasses the standard cryptographic handshake +// passing a standard network connection (e.g., TCP) will result in plaintext +// data leakage. +// +// The Request and NewChannel channels must be serviced or the connection +// will hang. +func NewControlClientConn(c net.Conn) (Conn, <-chan NewChannel, <-chan *Request, error) { + conn := &connection{ + sshConn: sshConn{conn: c}, + } + var err error + if conn.transport, err = handshakeControlProxy(c); err != nil { + return nil, nil, nil, fmt.Errorf("ssh: control proxy handshake failed: %w", err) + } + conn.mux = newMux(conn.transport) + return conn, conn.mux.incomingChannels, conn.mux.incomingRequests, nil +} + // clientHandshake performs the client side key exchange. See RFC 4253 Section // 7. func (c *connection) clientHandshake(dialAddress string, config *ClientConfig) error { @@ -197,6 +223,59 @@ type HostKeyCallback func(hostname string, remote net.Addr, key PublicKey) error // the server. A BannerCallback receives the message sent by the remote server. type BannerCallback func(message string) error +// ClientAuthContext contains information about the current state of the +// authentication process, passed to [ClientAuthCallback]. +type ClientAuthContext struct { + // Metadata contains the connection metadata. + Metadata ConnMetadata + + // Algorithms contains the negotiated algorithms. + Algorithms NegotiatedAlgorithms + + // AllowedMethods lists the authentication methods currently accepted + // by the server. These are the protocol-level names defined in RFC 4252 + // such as "publickey", "password". + AllowedMethods []string + + // PartialSuccessMethods lists the authentication methods that have already + // succeeded, indicating a multi-step authentication flow. This list + // represents the exact sequence of partial successes and may contain + // duplicates if the same method succeeded multiple times. + PartialSuccessMethods []string + + // TriedMethods lists the methods that have already been attempted and + // failed during this session. This list represents the exact sequence of + // failures and may contain duplicates. This allows the callback to also + // track the number of failed attempts for a specific method. + TriedMethods []string +} + +// ClientAuthCallback is a hook invoked before each authentication attempt. It +// allows the client to dynamically select an authentication method based on the +// current context, server capabilities, or previous failures. +// +// The callback is invoked after the initial "none" authentication method, once +// the server's supported authentication methods are known. +// +// Return values: +// - (AuthMethod, nil): The client will attempt this specific method next. +// The returned method does NOT need to be present in [ClientConfig.Auth]. +// This allows for dynamic authentication strategies (e.g., prompting +// for a password only if public key auth fails). Callers should inspect +// [ClientAuthContext.TriedMethods] to avoid repeatedly returning the +// same failing method. +// - (nil, nil): The client selects from [ClientConfig.Auth] the first +// instance of a method that has not been tried yet, or aborts if none +// are left. If authentication is not successful, the callback is invoked +// again before the following attempt. +// - (nil, error): The authentication process is aborted immediately, +// causing the ongoing SSH handshake to fail with the provided error. +// +// To bound resource use, the client caps the total number of authentication +// attempts (failures and partial successes combined) at 64. If the cap is +// exceeded the handshake aborts with an error. +type ClientAuthCallback func(ctx *ClientAuthContext) (AuthMethod, error) + // A ClientConfig structure is used to configure a Client. It must not be // modified after having been passed to an SSH function. type ClientConfig struct { @@ -210,6 +289,9 @@ type ClientConfig struct { // Auth contains possible authentication methods to use with the // server. Only the first instance of a particular RFC 4252 method will // be used during authentication. + // + // If AuthCallback is set, these AuthMethod are only used if the + // callback returns nil. Auth []AuthMethod // HostKeyCallback is called during the cryptographic @@ -240,6 +322,9 @@ type ClientConfig struct { // // A Timeout of zero means no timeout. Timeout time.Duration + + // AuthCallback, if non-nil, is invoked before each authentication attempt. + AuthCallback ClientAuthCallback } // InsecureIgnoreHostKey returns a function that can be used for diff --git a/vendor/golang.org/x/crypto/ssh/client_auth.go b/vendor/golang.org/x/crypto/ssh/client_auth.go index 4f2f75c367409..60af2fc4dd4fa 100644 --- a/vendor/golang.org/x/crypto/ssh/client_auth.go +++ b/vendor/golang.org/x/crypto/ssh/client_auth.go @@ -21,6 +21,12 @@ const ( authSuccess ) +// maxAuthClientTried bounds the total number of authentication attempts +// (failures and partial successes combined) the client makes before +// aborting the loop, to prevent unbounded growth when an AuthCallback +// keeps supplying methods. +const maxAuthClientTried = 64 + // clientAuthenticate authenticates with the remote server. See RFC 4252. func (c *connection) clientAuthenticate(config *ClientConfig) error { // initiate user auth session @@ -67,32 +73,62 @@ func (c *connection) clientAuthenticate(config *ClientConfig) error { // then any untried methods suggested by the server. var tried []string var lastMethods []string + var partialSuccess []string sessionID := c.transport.getSessionID() for auth := AuthMethod(new(noneAuth)); auth != nil; { ok, methods, err := auth.auth(sessionID, config.User, c.transport, config.Rand, extensions) if err != nil { // On disconnect, return error immediately - if _, ok := err.(*disconnectMsg); ok { + if _, isDisconnect := err.(*disconnectMsg); isDisconnect { return err } - // We return the error later if there is no other method left to - // try. + // We return the error later if there is no other method + // left to try. ok = authFailure } - if ok == authSuccess { - // success + + switch ok { + case authSuccess: return nil - } else if ok == authFailure { - if m := auth.method(); !slices.Contains(tried, m) { - tried = append(tried, m) - } + case authPartialSuccess: + partialSuccess = append(partialSuccess, auth.method()) + case authFailure: + tried = append(tried, auth.method()) } + if len(partialSuccess)+len(tried) > maxAuthClientTried { + return fmt.Errorf("ssh: too many authentication attempts (%d), aborting", + len(partialSuccess)+len(tried)) + } + if methods == nil { methods = lastMethods } lastMethods = methods + // If AuthCallback is set it takes precedence: it picks the next + // AuthMethod dynamically. The returned method need not be in + // config.Auth. If the callback returns (nil, nil) we fall back to + // selecting the next untried method from config.Auth below; on + // (nil, error) the handshake aborts. + if config.AuthCallback != nil { + ctx := &ClientAuthContext{ + Metadata: c, + Algorithms: c.Algorithms(), + AllowedMethods: slices.Clone(methods), + PartialSuccessMethods: slices.Clone(partialSuccess), + TriedMethods: slices.Clone(tried), + } + altAuth, cbErr := config.AuthCallback(ctx) + if cbErr != nil { + return cbErr + } + if altAuth != nil { + auth = altAuth + continue + } + } + auth = nil findNext: @@ -377,11 +413,11 @@ func (cb publicKeyCallback) auth(session []byte, user string, c packetConn, rand return authFailure, nil, err } - // If authentication succeeds or the list of available methods does not - // contain the "publickey" method, do not attempt to authenticate with any - // other keys. According to RFC 4252 Section 7, the latter can occur when - // additional authentication methods are required. - if success == authSuccess || !slices.Contains(methods, cb.method()) { + // If authentication succeeds or partially succeeds, return immediately + // so the caller can select the next auth method. According to RFC 4252 + // Section 7, if the server no longer lists "publickey" among its + // allowed methods, do not attempt to authenticate with any other keys. + if success == authSuccess || success == authPartialSuccess || !slices.Contains(methods, cb.method()) { return success, methods, err } } diff --git a/vendor/golang.org/x/crypto/ssh/connection.go b/vendor/golang.org/x/crypto/ssh/connection.go index 613a71a7b34e6..378f6407b5344 100644 --- a/vendor/golang.org/x/crypto/ssh/connection.go +++ b/vendor/golang.org/x/crypto/ssh/connection.go @@ -91,9 +91,17 @@ func DiscardRequests(in <-chan *Request) { } } +// A connTransport represents the transport for a connection. +type connTransport interface { + packetConn + getAlgorithms() NegotiatedAlgorithms + getSessionID() []byte + waitSession() error +} + // A connection represents an incoming connection. type connection struct { - transport *handshakeTransport + transport connTransport sshConn // The connection protocol. diff --git a/vendor/golang.org/x/crypto/ssh/control.go b/vendor/golang.org/x/crypto/ssh/control.go new file mode 100644 index 0000000000000..9b14e4cafabc5 --- /dev/null +++ b/vendor/golang.org/x/crypto/ssh/control.go @@ -0,0 +1,155 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ssh + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + + "golang.org/x/crypto/cryptobyte" +) + +const ( + muxProtocolVersion = 4 + + muxMsgHello = 0x00000001 + muxCProxy = 0x1000000f + muxSProxy = 0x8000000f +) + +const controlProxyRequestID = 0 + +// handshakeControlProxy attempts to establish a transport connection with an +// OpenSSH ControlMaster socket in proxy mode. For details see: +// https://github.com/openssh/openssh-portable/blob/master/PROTOCOL.mux +func handshakeControlProxy(rw io.ReadWriteCloser) (connTransport, error) { + if err := controlProxyWritePacket(rw, func(b *cryptobyte.Builder) { + b.AddUint32(muxMsgHello) + b.AddUint32(muxProtocolVersion) + }); err != nil { + return nil, fmt.Errorf("mux hello write failed: %w", err) + } + if err := controlProxyWritePacket(rw, func(b *cryptobyte.Builder) { + b.AddUint32(muxCProxy) + b.AddUint32(controlProxyRequestID) + }); err != nil { + return nil, fmt.Errorf("mux client proxy write failed: %w", err) + } + + messageType, body, err := controlProxyReadMessage(rw) + if err != nil { + return nil, fmt.Errorf("mux hello read failed: %w", err) + } + if messageType != muxMsgHello { + return nil, fmt.Errorf("expected hello response, got %v", messageType) + } + var v uint32 + if !body.ReadUint32(&v) { + return nil, errors.New("EOF reading mux protocol version") + } + if v != muxProtocolVersion { + return nil, fmt.Errorf("mux server has unsupported version %v", v) + } + messageType, body, err = controlProxyReadMessage(rw) + if err != nil { + return nil, fmt.Errorf("mux server proxy read failed: %w", err) + } + if messageType != muxSProxy { + return nil, fmt.Errorf("expected server proxy response, got %v", messageType) + } + var reqID uint32 + if !body.ReadUint32(&reqID) { + return nil, errors.New("EOF reading request id") + } + if reqID != controlProxyRequestID { + return nil, fmt.Errorf("expected request id %v, got %v", controlProxyRequestID, reqID) + } + return &controlProxyTransport{rw}, nil +} + +// controlProxyTransport implements the connTransport interface for +// ControlMaster connections. Each controlMessage has zero length padding and +// no MAC. +type controlProxyTransport struct { + rw io.ReadWriteCloser +} + +func (p *controlProxyTransport) Close() error { + return p.rw.Close() +} + +func (p *controlProxyTransport) writePacket(controlMessage []byte) error { + return controlProxyWritePacket(p.rw, func(b *cryptobyte.Builder) { + b.AddUint8(0) // Padding length. + b.AddBytes(controlMessage) + }) +} + +func (p *controlProxyTransport) readPacket() ([]byte, error) { + buf, err := controlProxyReadPacket(p.rw) + if err != nil { + return nil, fmt.Errorf("ssh: error reading control message: %w", err) + } + // Discard the padding length. + if len(buf) < 1 { + return nil, errors.New("ssh: EOF reading padding length") + } + if buf[0] != 0 { + return nil, errors.New("ssh: unexpected non-zero padding in control message") + } + return buf[1:], nil +} + +func (p *controlProxyTransport) getAlgorithms() NegotiatedAlgorithms { + return NegotiatedAlgorithms{} +} + +func (p *controlProxyTransport) getSessionID() []byte { + return nil +} + +func (p *controlProxyTransport) waitSession() error { + return nil +} + +func controlProxyWritePacket(w io.Writer, f cryptobyte.BuilderContinuation) error { + var buf []byte + b := cryptobyte.NewBuilder(buf) + b.AddUint32LengthPrefixed(f) + out, err := b.Bytes() + if err != nil { + return err + } + _, err = w.Write(out) + return err +} + +func controlProxyReadPacket(r io.Reader) (cryptobyte.String, error) { + var l uint32 + if err := binary.Read(r, binary.BigEndian, &l); err != nil { + return nil, err + } + if l > maxPacket { + return nil, fmt.Errorf("message length %v exceeds maximum %v", l, maxPacket) + } + buf := make([]byte, l) + if _, err := io.ReadFull(r, buf); err != nil { + return nil, err + } + return buf, nil +} + +func controlProxyReadMessage(r io.Reader) (messageType uint32, body cryptobyte.String, err error) { + body, err = controlProxyReadPacket(r) + if err != nil { + return 0, nil, fmt.Errorf("error reading message body: %w", err) + } + if !body.ReadUint32(&messageType) { + return 0, nil, errors.New("EOF reading message type") + } + return messageType, body, nil +} diff --git a/vendor/golang.org/x/crypto/ssh/kex.go b/vendor/golang.org/x/crypto/ssh/kex.go index 5f7fdd85142f7..91b771c4aeb7d 100644 --- a/vendor/golang.org/x/crypto/ssh/kex.go +++ b/vendor/golang.org/x/crypto/ssh/kex.go @@ -16,6 +16,7 @@ import ( "io" "math/big" "slices" + "sync" "golang.org/x/crypto/curve25519" ) @@ -718,15 +719,9 @@ func (gex *dhGEXSHA) Server(c packetConn, randSource io.Reader, magics *handshak kexDHGexRequest.MaxBits, kexDHGexRequest.PreferredBits) } - var p *big.Int - // We hardcode sending Oakley Group 14 (2048 bits), Oakley Group 15 (3072 - // bits) or Oakley Group 16 (4096 bits), based on the requested max size. - if kexDHGexRequest.MaxBits < 3072 { - p, _ = new(big.Int).SetString(oakleyGroup14, 16) - } else if kexDHGexRequest.MaxBits < 4096 { - p, _ = new(big.Int).SetString(oakleyGroup15, 16) - } else { - p, _ = new(big.Int).SetString(oakleyGroup16, 16) + p, err := chooseDH(kexDHGexRequest) + if err != nil { + return nil, err } g := big.NewInt(2) @@ -805,3 +800,65 @@ func (gex *dhGEXSHA) Server(c packetConn, randSource io.Reader, magics *handshak Hash: gex.hashFunc, }, err } + +type dhKEXGroup struct { + size int + p *big.Int +} + +// supportedDHKEXGroups returns the DH groups the server is willing to offer +// for diffie-hellman-group-exchange-* key exchanges. The list is built lazily +// on first use to keep the hex-to-big.Int parse out of package initialization. +var supportedDHKEXGroups = sync.OnceValue(func() []dhKEXGroup { + specs := []struct { + size int + hex string + }{ + {2048, oakleyGroup14}, + {3072, oakleyGroup15}, + {4096, oakleyGroup16}, + } + out := make([]dhKEXGroup, 0, len(specs)) + for _, s := range specs { + p, _ := new(big.Int).SetString(s.hex, 16) + out = append(out, dhKEXGroup{size: s.size, p: p}) + } + return out +}) + +// chooseDH picks a DH group for the given client request, mirroring the +// algorithm used by OpenSSH's choose_dh in dh.c: prefer the smallest known +// group larger than or equal to the client's PreferredBits, and otherwise pick +// the largest group within the accepted [MinBits, MaxBits] range. +func chooseDH(req kexDHGexRequestMsg) (*big.Int, error) { + var best *big.Int + bestSize := 0 + wantBits := int(req.PreferredBits) + + for _, group := range supportedDHKEXGroups() { + if uint32(group.size) < req.MinBits || uint32(group.size) > req.MaxBits { + continue + } + + if bestSize == 0 { + best = group.p + bestSize = group.size + continue + } + + closerFromAbove := group.size >= wantBits && group.size < bestSize + closerFromBelow := group.size > bestSize && bestSize < wantBits + + if closerFromAbove || closerFromBelow { + best = group.p + bestSize = group.size + } + } + + if bestSize == 0 { + return nil, fmt.Errorf("ssh: no suitable DH group found for request min: %d, preferred: %d, max: %d", + req.MinBits, req.PreferredBits, req.MaxBits) + } + + return best, nil +} diff --git a/vendor/golang.org/x/crypto/ssh/keys.go b/vendor/golang.org/x/crypto/ssh/keys.go index 47a07539d90ce..334861b7f1e70 100644 --- a/vendor/golang.org/x/crypto/ssh/keys.go +++ b/vendor/golang.org/x/crypto/ssh/keys.go @@ -76,7 +76,7 @@ func parsePubKey(in []byte, algo string) (pubKey PublicKey, rest []byte, err err case InsecureKeyAlgoDSA: return parseDSA(in) case KeyAlgoECDSA256, KeyAlgoECDSA384, KeyAlgoECDSA521: - return parseECDSA(in) + return parseECDSA(in, algo) case KeyAlgoSKECDSA256: return parseSKECDSA(in) case KeyAlgoED25519: @@ -469,6 +469,12 @@ func parseRSA(in []byte) (out PublicKey, rest []byte, err error) { return nil, nil, err } + // 8192 bits is also the maximum RSA key size accepted by crypto/tls for + // signature verification: + // https://github.com/golang/go/blob/69801b25/src/crypto/tls/handshake_client.go#L1096 + if w.N.BitLen() > 8192 { + return nil, nil, errors.New("ssh: rsa modulus too large") + } if w.E.BitLen() > 24 { return nil, nil, errors.New("ssh: exponent too large") } @@ -574,6 +580,24 @@ func checkDSAParams(param *dsa.Parameters) error { return fmt.Errorf("ssh: unsupported DSA key size %d", l) } + // FIPS 186-2 specifies that Q must be exactly 160 bits. We must enforce + // this to prevent DoS attacks where an attacker sends a huge Q which makes + // verification slow. + if l := param.Q.BitLen(); l != 160 { + return fmt.Errorf("ssh: unsupported DSA sub-prime size %d", l) + } + + // The generator G is an element of the group, so it must be strictly less + // than the modulus P. + if param.G.Cmp(param.P) >= 0 { + return errors.New("ssh: DSA generator larger than modulus") + } + + // G must be positive. + if param.G.Sign() <= 0 { + return errors.New("ssh: DSA generator must be positive") + } + return nil } @@ -596,6 +620,14 @@ func parseDSA(in []byte) (out PublicKey, rest []byte, err error) { return nil, nil, err } + // The public value Y must be a non-zero element of the group, i.e. + // strictly between 0 and P. crypto/dsa.Verify does not range-check Y, + // so we reject out-of-range values here to prevent a maliciously + // oversized Y from slowing verification. + if w.Y.Sign() <= 0 || w.Y.Cmp(w.P) >= 0 { + return nil, nil, errors.New("ssh: DSA public value Y out of range") + } + key := &dsaPublicKey{ Parameters: param, Y: w.Y, @@ -774,7 +806,7 @@ func supportedEllipticCurve(curve elliptic.Curve) bool { } // parseECDSA parses an ECDSA key according to RFC 5656, section 3.1. -func parseECDSA(in []byte) (out PublicKey, rest []byte, err error) { +func parseECDSA(in []byte, expectedType string) (out PublicKey, rest []byte, err error) { var w struct { Curve string KeyBytes []byte @@ -785,6 +817,12 @@ func parseECDSA(in []byte) (out PublicKey, rest []byte, err error) { return nil, nil, err } + actualType := "ecdsa-sha2-" + w.Curve + if expectedType != actualType { + return nil, nil, fmt.Errorf("ssh: algorithm type mismatch: expected %q, found curve %q (type %q)", + expectedType, w.Curve, actualType) + } + key := new(ecdsa.PublicKey) switch w.Curve { @@ -869,11 +907,25 @@ type skFields struct { Counter uint32 } +// flagUserPresence is the "user present" bit (UP) in the SK signature +// flags, matching the FIDO CTAP2 authenticatorData UP flag. See +// openssh/PROTOCOL.u2f. +const flagUserPresence = 0x01 + +// errSKMissingUserPresence is returned by SK key Verify methods when +// the signature does not assert user presence and the key was not +// marked as no-touch-required. +var errSKMissingUserPresence = errors.New("ssh: signature missing required user presence flag") + type skECDSAPublicKey struct { // application is a URL-like string, typically "ssh:" for SSH. // see openssh/PROTOCOL.u2f for details. application string ecdsa.PublicKey + // noTouchRequired, when true, disables the default user-presence + // check in Verify. It is set by skKeyWithoutUP on a clone of the + // key, never on an instance shared across authentication attempts. + noTouchRequired bool } func (k *skECDSAPublicKey) Type() string { @@ -959,6 +1011,10 @@ func (k *skECDSAPublicKey) Verify(data []byte, sig *Signature) error { return err } + if skf.Flags&flagUserPresence == 0 && !k.noTouchRequired { + return errSKMissingUserPresence + } + blob := struct { ApplicationDigest []byte `ssh:"rest"` Flags byte @@ -992,6 +1048,10 @@ type skEd25519PublicKey struct { // see openssh/PROTOCOL.u2f for details. application string ed25519.PublicKey + // noTouchRequired, when true, disables the default user-presence + // check in Verify. It is set by skKeyWithoutUP on a clone of the + // key, never on an instance shared across authentication attempts. + noTouchRequired bool } func (k *skEd25519PublicKey) Type() string { @@ -1066,6 +1126,10 @@ func (k *skEd25519PublicKey) Verify(data []byte, sig *Signature) error { return err } + if skf.Flags&flagUserPresence == 0 && !k.noTouchRequired { + return errSKMissingUserPresence + } + blob := struct { ApplicationDigest []byte `ssh:"rest"` Flags byte @@ -1408,6 +1472,17 @@ func passphraseProtectedOpenSSHKey(passphrase []byte) openSSHDecryptFunc { return nil, err } + // OpenSSH does not impose an upper bound on the bcrypt round count + // stored in the key file, but bcrypt_pbkdf cost is linear in rounds: + // the default is 16, ssh-keygen lets users pick anything up to + // INT_MAX. Cap at 2048 (128x the default, a few seconds of CPU) so + // that an oversized value in the file cannot tie up the caller for + // months. + const maxRounds = 1 << 11 + if opts.Rounds > maxRounds { + return nil, fmt.Errorf("ssh: bcrypt KDF rounds %d exceed maximum %d", opts.Rounds, maxRounds) + } + k, err := bcrypt_pbkdf.Key(passphrase, []byte(opts.Salt), int(opts.Rounds), 32+16) if err != nil { return nil, err @@ -1577,10 +1652,28 @@ func parseOpenSSHPrivateKey(key []byte, decrypt openSSHDecryptFunc) (crypto.Priv return nil, err } + // Mirror the validation done in parseRSA for public keys: cap the + // modulus at the same limit enforced by crypto/tls, reject oversized + // or invalid exponents, and additionally bound the prime factors to + // avoid the expensive CRT coefficient recomputation in pk.Precompute. + if key.N.BitLen() > 8192 { + return nil, errors.New("ssh: rsa modulus too large") + } + if key.P.BitLen() > 4096 || key.Q.BitLen() > 4096 { + return nil, errors.New("ssh: rsa prime too large") + } + if key.E.BitLen() > 24 { + return nil, errors.New("ssh: exponent too large") + } + e := key.E.Int64() + if e < 3 || e&1 == 0 { + return nil, errors.New("ssh: incorrect exponent") + } + pk := &rsa.PrivateKey{ PublicKey: rsa.PublicKey{ N: key.N, - E: int(key.E.Int64()), + E: int(e), }, D: key.D, Primes: []*big.Int{key.P, key.Q}, diff --git a/vendor/golang.org/x/crypto/ssh/mux.go b/vendor/golang.org/x/crypto/ssh/mux.go index d2d24c635d32a..5775881c6a2ed 100644 --- a/vendor/golang.org/x/crypto/ssh/mux.go +++ b/vendor/golang.org/x/crypto/ssh/mux.go @@ -91,9 +91,10 @@ type mux struct { incomingChannels chan NewChannel - globalSentMu sync.Mutex - globalResponses chan interface{} - incomingRequests chan *Request + globalSentMu sync.Mutex + globalSentPending atomic.Bool + globalResponses chan interface{} + incomingRequests chan *Request errCond *sync.Cond err error @@ -141,6 +142,27 @@ func (m *mux) SendRequest(name string, wantReply bool, payload []byte) (bool, [] if wantReply { m.globalSentMu.Lock() defer m.globalSentMu.Unlock() + + // Open the gate so that responses arriving while this request is in + // flight are allowed to reach globalResponses. Any response arriving + // while no request is pending is dropped by handleGlobalPacket. + m.globalSentPending.Store(true) + defer m.globalSentPending.Store(false) + + // Drain any spurious responses that may have been buffered. This prevents + // a previously buffered unexpected response from being consumed instead + // of the actual response for this request. + drain: + for { + select { + case _, ok := <-m.globalResponses: + if !ok { + break drain + } + default: + break drain + } + } } if err := m.sendMessage(globalRequestMsg{ @@ -267,7 +289,16 @@ func (m *mux) handleGlobalPacket(packet []byte) error { mux: m, } case *globalRequestSuccessMsg, *globalRequestFailureMsg: - m.globalResponses <- msg + // Drop responses that arrive when no SendRequest is waiting, to + // prevent a malicious peer from staging responses for a future + // caller. + if !m.globalSentPending.Load() { + return nil + } + select { + case m.globalResponses <- msg: + default: + } default: panic(fmt.Sprintf("not a global message %#v", msg)) } diff --git a/vendor/golang.org/x/crypto/ssh/server.go b/vendor/golang.org/x/crypto/ssh/server.go index 064dcbaf5aef4..3c0fcc953eaa7 100644 --- a/vendor/golang.org/x/crypto/ssh/server.go +++ b/vendor/golang.org/x/crypto/ssh/server.go @@ -34,21 +34,29 @@ type Permissions struct { // or not supported. CriticalOptions map[string]string - // Extensions are extra functionality that the server may - // offer on authenticated connections. Lack of support for an - // extension does not preclude authenticating a user. Common - // extensions are "permit-agent-forwarding", - // "permit-X11-forwarding". The Go SSH library currently does - // not act on any extension, and it is up to server - // implementations to honor them. Extensions can be used to - // pass data from the authentication callbacks to the server - // application layer. + // Extensions are extra functionality that the server may offer on + // authenticated connections. Lack of support for an extension does not + // preclude authenticating a user. Common extensions are + // "permit-agent-forwarding", "permit-X11-forwarding". In general the Go + // SSH library does not act on extensions and it is up to server + // implementations to honor them; extensions can also be used to pass data + // from the authentication callbacks to the server application layer. + // + // The one extension acted upon by this library is "no-touch-required", + // which applies only to security-key public keys + // (sk-ecdsa-sha2-nistp256@openssh.com and sk-ssh-ed25519@openssh.com). + // When present, it waives the default requirement that SK signatures + // assert user presence (i.e. a physical touch of the authenticator) + // during signature verification. Extensions map[string]string // ExtraData allows to store user defined data. ExtraData map[any]any } +// GSSAPIWithMICConfig includes the server callbacks for gssapi-with-mic +// authentication. If either field is nil, gssapi-with-mic is considered not +// configured. type GSSAPIWithMICConfig struct { // AllowLogin, must be set, is called when gssapi-with-mic // authentication is selected (RFC 4462 section 3). The srcName is from the @@ -63,6 +71,10 @@ type GSSAPIWithMICConfig struct { Server GSSAPIServer } +func gssapiWithMICConfigured(config *GSSAPIWithMICConfig) bool { + return config != nil && config.AllowLogin != nil && config.Server != nil +} + // SendAuthBanner implements [ServerPreAuthConn]. func (s *connection) SendAuthBanner(msg string) error { return s.transport.writePacket(Marshal(&userAuthBannerMsg{ @@ -84,6 +96,79 @@ type ServerPreAuthConn interface { SendAuthBanner(string) error } +// noTouchRequiredExtension is the extension name used by OpenSSH in +// authorized_keys options and certificate extensions to mark keys +// whose signatures do not need to assert user presence (touch). See +// ssh-keygen(1) and sshd(8). +const noTouchRequiredExtension = "no-touch-required" + +// noTouchAllowed reports whether the user presence requirement on +// SK signatures should be waived for this authentication attempt. The +// requirement is waived when the "no-touch-required" extension is +// present either in the Permissions returned by the auth callback +// (authorized_keys-level opt-out) or in the certificate's own +// Extensions (CA-level opt-out), matching OpenSSH behavior. OpenSSH +// reads the per-key opt-out only from cert Extensions and +// authorized_keys options (never from CriticalOptions); we follow the +// same rule. +func noTouchAllowed(pubKey PublicKey, perms *Permissions) bool { + if perms != nil { + if _, ok := perms.Extensions[noTouchRequiredExtension]; ok { + return true + } + } + if cert, ok := pubKey.(*Certificate); ok { + if _, ok := cert.Extensions[noTouchRequiredExtension]; ok { + return true + } + } + return false +} + +// skKeyWithoutUP returns a PublicKey equivalent to pubKey but whose +// Verify accepts SK signatures with the user-presence flag clear. If +// pubKey is not (and does not wrap) an SK key, pubKey is returned +// unchanged. The returned value never mutates pubKey: for SK keys a +// shallow copy is made so that the noTouchRequired flag is set only on +// the clone. +// +// The implementation is iterative rather than recursive. When pubKey +// is a *Certificate we unwrap exactly one level to look at the inner +// key. The SSH cert format forbids Certificate.Key from being another +// Certificate (parseCert rejects it), but nothing stops callers from +// constructing such a value directly in Go; a recursive descent could +// otherwise be driven to unbounded depth by a hand-crafted or cyclic +// Certificate. A malformed input of that shape simply returns +// unchanged here. +func skKeyWithoutUP(pubKey PublicKey) PublicKey { + cert, isCert := pubKey.(*Certificate) + target := pubKey + if isCert { + target = cert.Key + } + var cloned PublicKey + switch k := target.(type) { + case *skECDSAPublicKey: + c := *k + c.noTouchRequired = true + cloned = &c + case *skEd25519PublicKey: + c := *k + c.noTouchRequired = true + cloned = &c + default: + // Not an SK key (or a pathological *Certificate wrapping + // another *Certificate): pubKey is already usable for Verify. + return pubKey + } + if !isCert { + return cloned + } + c := *cert + c.Key = cloned + return &c +} + // ServerConfig holds server specific configuration data. type ServerConfig struct { // Config contains configuration shared between client and server. @@ -242,8 +327,10 @@ func (c *pubKeyCache) add(candidate cachedPubKey) { type ServerConn struct { Conn - // If the succeeding authentication callback returned a - // non-nil Permissions pointer, it is stored here. + // If the succeeding authentication callback returned a non-nil Permissions + // pointer, it is stored here. These are the permissions from the final, + // successful authentication method. Permissions returned by callbacks that + // return PartialSuccessError are not preserved and must be nil. Permissions *Permissions } @@ -302,8 +389,7 @@ func (s *connection) serverHandshake(config *ServerConfig) (*Permissions, error) } if !config.NoClientAuth && config.PasswordCallback == nil && config.PublicKeyCallback == nil && - config.KeyboardInteractiveCallback == nil && (config.GSSAPIWithMICConfig == nil || - config.GSSAPIWithMICConfig.AllowLogin == nil || config.GSSAPIWithMICConfig.Server == nil) { + config.KeyboardInteractiveCallback == nil && !gssapiWithMICConfigured(config.GSSAPIWithMICConfig) { return nil, errors.New("ssh: no authentication methods configured but NoClientAuth is also false") } @@ -527,6 +613,15 @@ func (b *BannerError) Error() string { return b.Err.Error() } +// maxAuthServerAttempts caps the total number of SSH_MSG_USERAUTH_REQUEST +// messages the server will process on a single connection, regardless of +// outcome (failure, partial success, public key query, or none). It is a +// backstop against clients that drive the authentication loop indefinitely +// without ever incurring a real failure — for example by repeatedly +// triggering PartialSuccessError or by spamming public key offer queries — +// neither of which increment the MaxAuthTries failure counter. +const maxAuthServerAttempts = 128 + func (s *connection) serverAuthenticate(config *ServerConfig) (*Permissions, error) { if config.PreAuthConnCallback != nil { config.PreAuthConnCallback(s) @@ -537,6 +632,7 @@ func (s *connection) serverAuthenticate(config *ServerConfig) (*Permissions, err var perms *Permissions authFailures := 0 + authAttempts := 0 noneAuthCount := 0 var authErrs []error var calledBannerCallback bool @@ -565,6 +661,19 @@ userAuthLoop: return nil, &ServerAuthError{Errors: authErrs} } + if authAttempts >= maxAuthServerAttempts { + discMsg := &disconnectMsg{ + Reason: 2, + Message: "too many authentication attempts", + } + if err := s.transport.writePacket(Marshal(discMsg)); err != nil { + return nil, err + } + authErrs = append(authErrs, discMsg) + return nil, &ServerAuthError{Errors: authErrs} + } + authAttempts++ + var userAuthReq userAuthRequestMsg if packet, err := s.transport.readPacket(); err != nil { if err == io.EOF { @@ -737,8 +846,15 @@ userAuthLoop: } signedData := buildDataSignedForAuth(sessionID, userAuthReq, algo, pubKeyData) - - if err := pubKey.Verify(signedData, sig); err != nil { + // pubKey is reused below for VerifiedPublicKeyCallback and + // must remain the key as presented by the client; derive a + // separate value for Verify that carries any applicable + // no-touch-required opt-out. + pubKeyForVerify := pubKey + if noTouchAllowed(pubKey, candidate.perms) { + pubKeyForVerify = skKeyWithoutUP(pubKey) + } + if err := pubKeyForVerify.Verify(signedData, sig); err != nil { return nil, err } @@ -750,9 +866,16 @@ userAuthLoop: // considered verified and the callback must not run. perms, authErr = config.VerifiedPublicKeyCallback(s, pubKey, perms, algo) } + if authErr == nil && perms != nil && perms.CriticalOptions != nil { + if saco := perms.CriticalOptions[sourceAddressCriticalOption]; saco != "" { + if err := checkSourceAddress(s.RemoteAddr(), saco); err != nil { + authErr = err + } + } + } } case "gssapi-with-mic": - if authConfig.GSSAPIWithMICConfig == nil { + if !gssapiWithMICConfigured(authConfig.GSSAPIWithMICConfig) { authErr = errors.New("ssh: gssapi-with-mic auth not configured") break } @@ -824,6 +947,13 @@ userAuthLoop: var failureMsg userAuthFailureMsg if partialSuccess, ok := authErr.(*PartialSuccessError); ok { + // Permissions are not preserved between authentication steps. To + // avoid confusion about the final state of the connection, we + // disallow returning non-nil Permissions combined with + // PartialSuccessError. + if perms != nil { + return nil, errors.New("ssh: permissions must be nil when returning PartialSuccessError") + } // After a partial success error we don't allow changing the user // name and execute the NoClientAuthCallback. partialSuccessReturned = true @@ -878,8 +1008,7 @@ userAuthLoop: if authConfig.KeyboardInteractiveCallback != nil { failureMsg.Methods = append(failureMsg.Methods, "keyboard-interactive") } - if authConfig.GSSAPIWithMICConfig != nil && authConfig.GSSAPIWithMICConfig.Server != nil && - authConfig.GSSAPIWithMICConfig.AllowLogin != nil { + if gssapiWithMICConfigured(authConfig.GSSAPIWithMICConfig) { failureMsg.Methods = append(failureMsg.Methods, "gssapi-with-mic") } diff --git a/vendor/golang.org/x/crypto/ssh/session.go b/vendor/golang.org/x/crypto/ssh/session.go index acef62259fdee..ac62955788191 100644 --- a/vendor/golang.org/x/crypto/ssh/session.go +++ b/vendor/golang.org/x/crypto/ssh/session.go @@ -423,6 +423,9 @@ func (s *Session) wait(reqs <-chan *Request) error { for msg := range reqs { switch msg.Type { case "exit-status": + if len(msg.Payload) < 4 { + return errors.New("ssh: malformed exit-status request") + } wm.status = int(binary.BigEndian.Uint32(msg.Payload)) case "exit-signal": var sigval struct { diff --git a/vendor/golang.org/x/net/http2/server.go b/vendor/golang.org/x/net/http2/server.go index fbb145115a7f3..a7d2053b6c796 100644 --- a/vendor/golang.org/x/net/http2/server.go +++ b/vendor/golang.org/x/net/http2/server.go @@ -2657,21 +2657,6 @@ func (rws *responseWriterState) writeChunk(p []byte) (n int, err error) { return len(p), nil } -// TrailerPrefix is a magic prefix for ResponseWriter.Header map keys -// that, if present, signals that the map entry is actually for -// the response trailers, and not the response headers. The prefix -// is stripped after the ServeHTTP call finishes and the values are -// sent in the trailers. -// -// This mechanism is intended only for trailers that are not known -// prior to the headers being written. If the set of trailers is fixed -// or known before the header is written, the normal Go trailers mechanism -// is preferred: -// -// https://golang.org/pkg/net/http/#ResponseWriter -// https://golang.org/pkg/net/http/#example_ResponseWriter_trailers -const TrailerPrefix = "Trailer:" - // promoteUndeclaredTrailers permits http.Handlers to set trailers // after the header has already been flushed. Because the Go // ResponseWriter interface has no way to set Trailers (only the @@ -2948,12 +2933,6 @@ func (w *responseWriter) handlerDone() { responseWriterStatePool.Put(rws) } -// Push errors. -var ( - ErrRecursivePush = errors.New("http2: recursive push not allowed") - ErrPushLimitReached = errors.New("http2: push would exceed peer's SETTINGS_MAX_CONCURRENT_STREAMS") -) - var _ http.Pusher = (*responseWriter)(nil) func (w *responseWriter) Push(target string, opts *http.PushOptions) error { diff --git a/vendor/golang.org/x/net/http2/server_common.go b/vendor/golang.org/x/net/http2/server_common.go index e2faeb9b6ad8a..449538c861a6e 100644 --- a/vendor/golang.org/x/net/http2/server_common.go +++ b/vendor/golang.org/x/net/http2/server_common.go @@ -6,11 +6,33 @@ package http2 import ( "context" + "errors" "net" "net/http" "time" ) +// TrailerPrefix is a magic prefix for ResponseWriter.Header map keys +// that, if present, signals that the map entry is actually for +// the response trailers, and not the response headers. The prefix +// is stripped after the ServeHTTP call finishes and the values are +// sent in the trailers. +// +// This mechanism is intended only for trailers that are not known +// prior to the headers being written. If the set of trailers is fixed +// or known before the header is written, the normal Go trailers mechanism +// is preferred: +// +// https://golang.org/pkg/net/http/#ResponseWriter +// https://golang.org/pkg/net/http/#example_ResponseWriter_trailers +const TrailerPrefix = "Trailer:" + +// Push errors. +var ( + ErrRecursivePush = errors.New("http2: recursive push not allowed") + ErrPushLimitReached = errors.New("http2: push would exceed peer's SETTINGS_MAX_CONCURRENT_STREAMS") +) + // ConfigureServer adds HTTP/2 support to a net/http Server. // // The configuration conf may be nil. diff --git a/vendor/golang.org/x/net/http2/server_wrap.go b/vendor/golang.org/x/net/http2/server_wrap.go index 9e6003b895abb..a7a09551c4320 100644 --- a/vendor/golang.org/x/net/http2/server_wrap.go +++ b/vendor/golang.org/x/net/http2/server_wrap.go @@ -159,3 +159,43 @@ type FrameWriteRequest struct { // to avoid duplicating an exported symbol across two files, // but the changes required to make this work are fairly large. } + +func (wr FrameWriteRequest) StreamID() uint32 { + return 0 +} + +func (wr FrameWriteRequest) DataSize() int { + return 0 +} + +func (wr FrameWriteRequest) Consume(n int32) (FrameWriteRequest, FrameWriteRequest, int) { + return FrameWriteRequest{}, FrameWriteRequest{}, 0 +} + +func (wr FrameWriteRequest) String() string { + return "" +} + +// NewPriorityWriteScheduler is deprecated. +// +// Deprecated: User-provided write schedulers are deprecated. +func NewPriorityWriteScheduler(cfg *PriorityWriteSchedulerConfig) WriteScheduler { + return unsupportedWriteScheduler{} +} + +// NewRandomWriteScheduler is deprecated. +// +// Deprecated: User-provided write schedulers are deprecated. +func NewRandomWriteScheduler() WriteScheduler { + return unsupportedWriteScheduler{} +} + +type unsupportedWriteScheduler struct{} + +func (unsupportedWriteScheduler) OpenStream(streamID uint32, options OpenStreamOptions) {} +func (unsupportedWriteScheduler) CloseStream(streamID uint32) {} +func (unsupportedWriteScheduler) AdjustStream(streamID uint32, priority PriorityParam) {} +func (unsupportedWriteScheduler) Push(wr FrameWriteRequest) {} +func (unsupportedWriteScheduler) Pop() (wr FrameWriteRequest, ok bool) { + return FrameWriteRequest{}, false +} diff --git a/vendor/golang.org/x/net/http2/transport.go b/vendor/golang.org/x/net/http2/transport.go index 882a926945dd9..08ac409b0e0ab 100644 --- a/vendor/golang.org/x/net/http2/transport.go +++ b/vendor/golang.org/x/net/http2/transport.go @@ -399,27 +399,6 @@ func (sew stickyErrWriter) Write(p []byte) (n int, err error) { return n, err } -// noCachedConnError is the concrete type of ErrNoCachedConn, which -// needs to be detected by net/http regardless of whether it's its -// bundled version (in h2_bundle.go with a rewritten type name) or -// from a user's x/net/http2. As such, as it has a unique method name -// (IsHTTP2NoCachedConnError) that net/http sniffs for via func -// isNoCachedConnError. -type noCachedConnError struct{} - -func (noCachedConnError) IsHTTP2NoCachedConnError() {} -func (noCachedConnError) Error() string { return "http2: no cached connection was available" } - -// isNoCachedConnError reports whether err is of type noCachedConnError -// or its equivalent renamed type in net/http2's h2_bundle.go. Both types -// may coexist in the same running program. -func isNoCachedConnError(err error) bool { - _, ok := err.(interface{ IsHTTP2NoCachedConnError() }) - return ok -} - -var ErrNoCachedConn error = noCachedConnError{} - func (t *Transport) roundTripOpt(req *http.Request, opt RoundTripOpt) (*http.Response, error) { switch req.URL.Scheme { case "https": @@ -1786,19 +1765,6 @@ func (cc *ClientConn) readLoop() { } } -// GoAwayError is returned by the Transport when the server closes the -// TCP connection after sending a GOAWAY frame. -type GoAwayError struct { - LastStreamID uint32 - ErrCode ErrCode - DebugData string -} - -func (e GoAwayError) Error() string { - return fmt.Sprintf("http2: server sent GOAWAY and closed the connection; LastStreamID=%v, ErrCode=%v, debug=%q", - e.LastStreamID, e.ErrCode, e.DebugData) -} - func isEOFOrNetReadError(err error) bool { if err == io.EOF { return true diff --git a/vendor/golang.org/x/net/http2/transport_common.go b/vendor/golang.org/x/net/http2/transport_common.go index f7f85b3ad5cd8..b9f52932e92ab 100644 --- a/vendor/golang.org/x/net/http2/transport_common.go +++ b/vendor/golang.org/x/net/http2/transport_common.go @@ -411,3 +411,37 @@ func (t *Transport) dialTLSWithContext(ctx context.Context, network, addr string tlsCn := cn.(*tls.Conn) // DialContext comment promises this will always succeed return tlsCn, nil } + +// GoAwayError is returned by the Transport when the server closes the +// TCP connection after sending a GOAWAY frame. +type GoAwayError struct { + LastStreamID uint32 + ErrCode ErrCode + DebugData string +} + +func (e GoAwayError) Error() string { + return fmt.Sprintf("http2: server sent GOAWAY and closed the connection; LastStreamID=%v, ErrCode=%v, debug=%q", + e.LastStreamID, e.ErrCode, e.DebugData) +} + +// noCachedConnError is the concrete type of ErrNoCachedConn, which +// needs to be detected by net/http regardless of whether it's its +// bundled version (in h2_bundle.go with a rewritten type name) or +// from a user's x/net/http2. As such, as it has a unique method name +// (IsHTTP2NoCachedConnError) that net/http sniffs for via func +// isNoCachedConnError. +type noCachedConnError struct{} + +func (noCachedConnError) IsHTTP2NoCachedConnError() {} +func (noCachedConnError) Error() string { return "http2: no cached connection was available" } + +// isNoCachedConnError reports whether err is of type noCachedConnError +// or its equivalent renamed type in net/http2's h2_bundle.go. Both types +// may coexist in the same running program. +func isNoCachedConnError(err error) bool { + _, ok := err.(interface{ IsHTTP2NoCachedConnError() }) + return ok +} + +var ErrNoCachedConn error = noCachedConnError{} diff --git a/vendor/golang.org/x/net/http2/writesched_common.go b/vendor/golang.org/x/net/http2/writesched_common.go index 957bc659e712b..75354c1ff8fc3 100644 --- a/vendor/golang.org/x/net/http2/writesched_common.go +++ b/vendor/golang.org/x/net/http2/writesched_common.go @@ -47,3 +47,44 @@ type OpenStreamOptions struct { // priority is used to set the priority of the newly opened stream. priority PriorityParam } + +// PriorityWriteSchedulerConfig configures a priorityWriteScheduler. +// +// Deprecated: User-provided write schedulers are deprecated. +type PriorityWriteSchedulerConfig struct { + // MaxClosedNodesInTree controls the maximum number of closed streams to + // retain in the priority tree. Setting this to zero saves a small amount + // of memory at the cost of performance. + // + // See RFC 7540, Section 5.3.4: + // "It is possible for a stream to become closed while prioritization + // information ... is in transit. ... This potentially creates suboptimal + // prioritization, since the stream could be given a priority that is + // different from what is intended. To avoid these problems, an endpoint + // SHOULD retain stream prioritization state for a period after streams + // become closed. The longer state is retained, the lower the chance that + // streams are assigned incorrect or default priority values." + MaxClosedNodesInTree int + + // MaxIdleNodesInTree controls the maximum number of idle streams to + // retain in the priority tree. Setting this to zero saves a small amount + // of memory at the cost of performance. + // + // See RFC 7540, Section 5.3.4: + // Similarly, streams that are in the "idle" state can be assigned + // priority or become a parent of other streams. This allows for the + // creation of a grouping node in the dependency tree, which enables + // more flexible expressions of priority. Idle streams begin with a + // default priority (Section 5.3.5). + MaxIdleNodesInTree int + + // ThrottleOutOfOrderWrites enables write throttling to help ensure that + // data is delivered in priority order. This works around a race where + // stream B depends on stream A and both streams are about to call Write + // to queue DATA frames. If B wins the race, a naive scheduler would eagerly + // write as much data from B as possible, but this is suboptimal because A + // is a higher-priority stream. With throttling enabled, we write a small + // amount of data from B to minimize the amount of bandwidth that B can + // steal from A. + ThrottleOutOfOrderWrites bool +} diff --git a/vendor/golang.org/x/net/http2/writesched_priority_rfc7540.go b/vendor/golang.org/x/net/http2/writesched_priority_rfc7540.go index ccd1afef2ab27..10e67f7ceebce 100644 --- a/vendor/golang.org/x/net/http2/writesched_priority_rfc7540.go +++ b/vendor/golang.org/x/net/http2/writesched_priority_rfc7540.go @@ -15,47 +15,6 @@ import ( // RFC 7540, Section 5.3.5: the default weight is 16. const priorityDefaultWeightRFC7540 = 15 // 16 = 15 + 1 -// PriorityWriteSchedulerConfig configures a priorityWriteScheduler. -// -// Deprecated: User-provided write schedulers are deprecated. -type PriorityWriteSchedulerConfig struct { - // MaxClosedNodesInTree controls the maximum number of closed streams to - // retain in the priority tree. Setting this to zero saves a small amount - // of memory at the cost of performance. - // - // See RFC 7540, Section 5.3.4: - // "It is possible for a stream to become closed while prioritization - // information ... is in transit. ... This potentially creates suboptimal - // prioritization, since the stream could be given a priority that is - // different from what is intended. To avoid these problems, an endpoint - // SHOULD retain stream prioritization state for a period after streams - // become closed. The longer state is retained, the lower the chance that - // streams are assigned incorrect or default priority values." - MaxClosedNodesInTree int - - // MaxIdleNodesInTree controls the maximum number of idle streams to - // retain in the priority tree. Setting this to zero saves a small amount - // of memory at the cost of performance. - // - // See RFC 7540, Section 5.3.4: - // Similarly, streams that are in the "idle" state can be assigned - // priority or become a parent of other streams. This allows for the - // creation of a grouping node in the dependency tree, which enables - // more flexible expressions of priority. Idle streams begin with a - // default priority (Section 5.3.5). - MaxIdleNodesInTree int - - // ThrottleOutOfOrderWrites enables write throttling to help ensure that - // data is delivered in priority order. This works around a race where - // stream B depends on stream A and both streams are about to call Write - // to queue DATA frames. If B wins the race, a naive scheduler would eagerly - // write as much data from B as possible, but this is suboptimal because A - // is a higher-priority stream. With throttling enabled, we write a small - // amount of data from B to minimize the amount of bandwidth that B can - // steal from A. - ThrottleOutOfOrderWrites bool -} - // NewPriorityWriteScheduler constructs a WriteScheduler that schedules // frames by following HTTP/2 priorities as described in RFC 7540 Section 5.3. // If cfg is nil, default options are used. diff --git a/vendor/golang.org/x/sync/errgroup/errgroup.go b/vendor/golang.org/x/sync/errgroup/errgroup.go index f69fd754685a4..c261a8ebbddb1 100644 --- a/vendor/golang.org/x/sync/errgroup/errgroup.go +++ b/vendor/golang.org/x/sync/errgroup/errgroup.go @@ -109,7 +109,7 @@ func (g *Group) TryGo(f func() error) bool { if g.sem != nil { select { case g.sem <- token{}: - // Note: this allows barging iff channels in general allow barging. + // Note: this allows barging if and only if channels in general allow barging. default: return false } diff --git a/vendor/golang.org/x/sync/semaphore/semaphore.go b/vendor/golang.org/x/sync/semaphore/semaphore.go index b618162aab648..040c5bc50917b 100644 --- a/vendor/golang.org/x/sync/semaphore/semaphore.go +++ b/vendor/golang.org/x/sync/semaphore/semaphore.go @@ -83,7 +83,7 @@ func (s *Weighted) Acquire(ctx context.Context, n int64) error { default: isFront := s.waiters.Front() == elem s.waiters.Remove(elem) - // If we're at the front and there're extra tokens left, notify other waiters. + // If we're at the front and there are extra tokens left, notify other waiters. if isFront && s.size > s.cur { s.notifyWaiters() } @@ -139,15 +139,15 @@ func (s *Weighted) notifyWaiters() { w := next.Value.(waiter) if s.size-s.cur < w.n { - // Not enough tokens for the next waiter. We could keep going (to try to + // Not enough tokens for the next waiter. We could keep going (to try to // find a waiter with a smaller request), but under load that could cause // starvation for large requests; instead, we leave all remaining waiters // blocked. // // Consider a semaphore used as a read-write lock, with N tokens, N - // readers, and one writer. Each reader can Acquire(1) to obtain a read - // lock. The writer can Acquire(N) to obtain a write lock, excluding all - // of the readers. If we allow the readers to jump ahead in the queue, + // readers, and one writer. Each reader can Acquire(1) to obtain a read + // lock. The writer can Acquire(N) to obtain a write lock, excluding all + // of the readers. If we allow the readers to jump ahead in the queue, // the writer will starve — there is always one token available for every // reader. break diff --git a/vendor/golang.org/x/sync/singleflight/singleflight.go b/vendor/golang.org/x/sync/singleflight/singleflight.go index 90ca138af31f4..32e40bdc8717c 100644 --- a/vendor/golang.org/x/sync/singleflight/singleflight.go +++ b/vendor/golang.org/x/sync/singleflight/singleflight.go @@ -15,12 +15,12 @@ import ( "sync" ) -// errGoexit indicates the runtime.Goexit was called in -// the user given function. +// errGoexit indicates runtime.Goexit was called in +// the user-given function. var errGoexit = errors.New("runtime.Goexit was called") // A panicError is an arbitrary value recovered from a panic -// with the stack trace during the execution of given function. +// with the stack trace during the execution of the given function. type panicError struct { value any stack []byte @@ -204,7 +204,7 @@ func (g *Group) doCall(c *call, key string, fn func() (any, error)) { } } -// Forget tells the singleflight to forget about a key. Future calls +// Forget tells the singleflight to forget about a key. Future calls // to Do for this key will call the function rather than waiting for // an earlier call to complete. func (g *Group) Forget(key string) { diff --git a/vendor/golang.org/x/sys/cpu/cpu.go b/vendor/golang.org/x/sys/cpu/cpu.go index 63541994ef030..f1ce515d5c9a2 100644 --- a/vendor/golang.org/x/sys/cpu/cpu.go +++ b/vendor/golang.org/x/sys/cpu/cpu.go @@ -152,13 +152,17 @@ var ARM struct { // The booleans in Loong64 contain the correspondingly named cpu feature bit. // The struct is padded to avoid false sharing. var Loong64 struct { - _ CacheLinePad - HasLSX bool // support 128-bit vector extension - HasLASX bool // support 256-bit vector extension - HasCRC32 bool // support CRC instruction - HasLAM_BH bool // support AM{SWAP/ADD}[_DB].{B/H} instruction - HasLAMCAS bool // support AMCAS[_DB].{B/H/W/D} instruction - _ CacheLinePad + _ CacheLinePad + HasLSX bool // support 128-bit vector extension + HasLASX bool // support 256-bit vector extension + HasCRC32 bool // support CRC instruction + HasLAMCAS bool // support AMCAS[_DB].{B/H/W/D} + HasLAM_BH bool // support AM{SWAP/ADD}[_DB].{B/H} instruction + HasLLACQ_SCREL bool // support LLACQ.{W/D}, SCREL.{W/D} instruction + HasSCQ bool // support SC.Q instruction + HasDBAR_HINTS bool // supports finer-grained DBAR hints + + _ CacheLinePad } // MIPS64X contains the supported CPU features of the current mips64/mips64le @@ -232,6 +236,7 @@ var RISCV64 struct { HasZba bool // Address generation instructions extension HasZbb bool // Basic bit-manipulation extension HasZbs bool // Single-bit instructions extension + HasZbc bool // Carryless multiplication extension HasZvbb bool // Vector Basic Bit-manipulation HasZvbc bool // Vector Carryless Multiplication HasZvkb bool // Vector Cryptography Bit-manipulation diff --git a/vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go b/vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go index ad741536f395b..f4fb52ee966ca 100644 --- a/vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go +++ b/vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go @@ -58,6 +58,7 @@ const ( riscv_HWPROBE_EXT_ZBA = 0x8 riscv_HWPROBE_EXT_ZBB = 0x10 riscv_HWPROBE_EXT_ZBS = 0x20 + riscv_HWPROBE_EXT_ZBC = 0x80 riscv_HWPROBE_EXT_ZVBB = 0x20000 riscv_HWPROBE_EXT_ZVBC = 0x40000 riscv_HWPROBE_EXT_ZVKB = 0x80000 @@ -108,6 +109,7 @@ func doinit() { RISCV64.HasZba = isSet(v, riscv_HWPROBE_EXT_ZBA) RISCV64.HasZbb = isSet(v, riscv_HWPROBE_EXT_ZBB) RISCV64.HasZbs = isSet(v, riscv_HWPROBE_EXT_ZBS) + RISCV64.HasZbc = isSet(v, riscv_HWPROBE_EXT_ZBC) RISCV64.HasZvbb = isSet(v, riscv_HWPROBE_EXT_ZVBB) RISCV64.HasZvbc = isSet(v, riscv_HWPROBE_EXT_ZVBC) RISCV64.HasZvkb = isSet(v, riscv_HWPROBE_EXT_ZVKB) diff --git a/vendor/golang.org/x/sys/cpu/cpu_loong64.go b/vendor/golang.org/x/sys/cpu/cpu_loong64.go index 45ecb29ae78a5..8c234b4415ef2 100644 --- a/vendor/golang.org/x/sys/cpu/cpu_loong64.go +++ b/vendor/golang.org/x/sys/cpu/cpu_loong64.go @@ -15,8 +15,13 @@ const ( cpucfg1_CRC32 = 1 << 25 // CPUCFG2 bits - cpucfg2_LAM_BH = 1 << 27 - cpucfg2_LAMCAS = 1 << 28 + cpucfg2_LAM_BH = 1 << 27 + cpucfg2_LAMCAS = 1 << 28 + cpucfg2_LLACQ_SCREL = 1 << 29 + cpucfg2_SCQ = 1 << 30 + + // CPUCFG3 bits + cpucfg3_DBAR_HINTS = 1 << 17 ) func initOptions() { @@ -26,6 +31,9 @@ func initOptions() { {Name: "crc32", Feature: &Loong64.HasCRC32}, {Name: "lam_bh", Feature: &Loong64.HasLAM_BH}, {Name: "lamcas", Feature: &Loong64.HasLAMCAS}, + {Name: "llacq_screl", Feature: &Loong64.HasLLACQ_SCREL}, + {Name: "scq", Feature: &Loong64.HasSCQ}, + {Name: "dbar_hints", Feature: &Loong64.HasDBAR_HINTS}, } // The CPUCFG data on Loong64 only reflects the hardware capabilities, @@ -37,10 +45,14 @@ func initOptions() { // through CPUCFG cfg1 := get_cpucfg(1) cfg2 := get_cpucfg(2) + cfg3 := get_cpucfg(3) Loong64.HasCRC32 = cfgIsSet(cfg1, cpucfg1_CRC32) Loong64.HasLAMCAS = cfgIsSet(cfg2, cpucfg2_LAMCAS) Loong64.HasLAM_BH = cfgIsSet(cfg2, cpucfg2_LAM_BH) + Loong64.HasLLACQ_SCREL = cfgIsSet(cfg2, cpucfg2_LLACQ_SCREL) + Loong64.HasSCQ = cfgIsSet(cfg2, cpucfg2_SCQ) + Loong64.HasDBAR_HINTS = cfgIsSet(cfg3, cpucfg3_DBAR_HINTS) } func get_cpucfg(reg uint32) uint32 diff --git a/vendor/golang.org/x/sys/cpu/cpu_riscv64.go b/vendor/golang.org/x/sys/cpu/cpu_riscv64.go index 0f617aef541b5..d4e9885f31475 100644 --- a/vendor/golang.org/x/sys/cpu/cpu_riscv64.go +++ b/vendor/golang.org/x/sys/cpu/cpu_riscv64.go @@ -16,6 +16,7 @@ func initOptions() { {Name: "zba", Feature: &RISCV64.HasZba}, {Name: "zbb", Feature: &RISCV64.HasZbb}, {Name: "zbs", Feature: &RISCV64.HasZbs}, + {Name: "zbc", Feature: &RISCV64.HasZbc}, // RISC-V Cryptography Extensions {Name: "zvbb", Feature: &RISCV64.HasZvbb}, {Name: "zvbc", Feature: &RISCV64.HasZvbc}, diff --git a/vendor/golang.org/x/sys/unix/mkerrors.sh b/vendor/golang.org/x/sys/unix/mkerrors.sh index fd39be4efdc6a..fa74cfe9e8a48 100644 --- a/vendor/golang.org/x/sys/unix/mkerrors.sh +++ b/vendor/golang.org/x/sys/unix/mkerrors.sh @@ -354,6 +354,9 @@ struct ltchars { // Renamed in v6.16, commit c6d732c38f93 ("net: ethtool: remove duplicate defines for family info") #define ETHTOOL_FAMILY_NAME ETHTOOL_GENL_NAME #define ETHTOOL_FAMILY_VERSION ETHTOOL_GENL_VERSION + +// Removed in v6.17, commit 760e6f7befba ("futex: Remove support for IMMUTABLE") +#define PR_FUTEX_HASH_GET_IMMUTABLE 3 ' includes_NetBSD=' diff --git a/vendor/golang.org/x/sys/unix/readv_unix.go b/vendor/golang.org/x/sys/unix/readv_unix.go new file mode 100644 index 0000000000000..38a2be937ea77 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/readv_unix.go @@ -0,0 +1,103 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin || linux || openbsd + +package unix + +import "unsafe" + +// minIovec is the size of the small initial allocation used by +// Readv, Writev, etc. +// +// This small allocation gets stack allocated, which lets the +// common use case of len(iovs) <= minIovec avoid more expensive +// heap allocations. +const minIovec = 8 + +// appendBytes converts bs to Iovecs and appends them to vecs. +func appendBytes(vecs []Iovec, bs [][]byte) []Iovec { + for _, b := range bs { + var v Iovec + v.SetLen(len(b)) + if len(b) > 0 { + v.Base = &b[0] + } else { + v.Base = (*byte)(unsafe.Pointer(&_zero)) + } + vecs = append(vecs, v) + } + return vecs +} + +// writevRaceDetect tells the race detector that the program +// has read the first n bytes stored in iovecs. +func writevRaceDetect(iovecs []Iovec, n int) { + if !raceenabled { + return + } + for i := 0; n > 0 && i < len(iovecs); i++ { + m := min(int(iovecs[i].Len), n) + n -= m + if m > 0 { + raceReadRange(unsafe.Pointer(iovecs[i].Base), m) + } + } +} + +// readvRaceDetect tells the race detector that the program +// has written to the first n bytes stored in iovecs. +func readvRaceDetect(iovecs []Iovec, n int, err error) { + if !raceenabled { + return + } + for i := 0; n > 0 && i < len(iovecs); i++ { + m := min(int(iovecs[i].Len), n) + n -= m + if m > 0 { + raceWriteRange(unsafe.Pointer(iovecs[i].Base), m) + } + } + if err == nil { + raceAcquire(unsafe.Pointer(&ioSync)) + } +} + +func Readv(fd int, iovs [][]byte) (n int, err error) { + iovecs := make([]Iovec, 0, minIovec) + iovecs = appendBytes(iovecs, iovs) + n, err = readv(fd, iovecs) + readvRaceDetect(iovecs, n, err) + return n, err +} + +func Preadv(fd int, iovs [][]byte, offset int64) (n int, err error) { + iovecs := make([]Iovec, 0, minIovec) + iovecs = appendBytes(iovecs, iovs) + n, err = preadv(fd, iovecs, offset) + readvRaceDetect(iovecs, n, err) + return n, err +} + +func Writev(fd int, iovs [][]byte) (n int, err error) { + iovecs := make([]Iovec, 0, minIovec) + iovecs = appendBytes(iovecs, iovs) + if raceenabled { + raceReleaseMerge(unsafe.Pointer(&ioSync)) + } + n, err = writev(fd, iovecs) + writevRaceDetect(iovecs, n) + return n, err +} + +func Pwritev(fd int, iovs [][]byte, offset int64) (n int, err error) { + iovecs := make([]Iovec, 0, minIovec) + iovecs = appendBytes(iovecs, iovs) + if raceenabled { + raceReleaseMerge(unsafe.Pointer(&ioSync)) + } + n, err = pwritev(fd, iovecs, offset) + writevRaceDetect(iovecs, n) + return n, err +} diff --git a/vendor/golang.org/x/sys/unix/syscall_darwin.go b/vendor/golang.org/x/sys/unix/syscall_darwin.go index 7838ca5db200b..38590ca81dd1a 100644 --- a/vendor/golang.org/x/sys/unix/syscall_darwin.go +++ b/vendor/golang.org/x/sys/unix/syscall_darwin.go @@ -602,95 +602,6 @@ func Connectx(fd int, srcIf uint32, srcAddr, dstAddr Sockaddr, associd SaeAssocI return } -const minIovec = 8 - -func Readv(fd int, iovs [][]byte) (n int, err error) { - iovecs := make([]Iovec, 0, minIovec) - iovecs = appendBytes(iovecs, iovs) - n, err = readv(fd, iovecs) - readvRacedetect(iovecs, n, err) - return n, err -} - -func Preadv(fd int, iovs [][]byte, offset int64) (n int, err error) { - iovecs := make([]Iovec, 0, minIovec) - iovecs = appendBytes(iovecs, iovs) - n, err = preadv(fd, iovecs, offset) - readvRacedetect(iovecs, n, err) - return n, err -} - -func Writev(fd int, iovs [][]byte) (n int, err error) { - iovecs := make([]Iovec, 0, minIovec) - iovecs = appendBytes(iovecs, iovs) - if raceenabled { - raceReleaseMerge(unsafe.Pointer(&ioSync)) - } - n, err = writev(fd, iovecs) - writevRacedetect(iovecs, n) - return n, err -} - -func Pwritev(fd int, iovs [][]byte, offset int64) (n int, err error) { - iovecs := make([]Iovec, 0, minIovec) - iovecs = appendBytes(iovecs, iovs) - if raceenabled { - raceReleaseMerge(unsafe.Pointer(&ioSync)) - } - n, err = pwritev(fd, iovecs, offset) - writevRacedetect(iovecs, n) - return n, err -} - -func appendBytes(vecs []Iovec, bs [][]byte) []Iovec { - for _, b := range bs { - var v Iovec - v.SetLen(len(b)) - if len(b) > 0 { - v.Base = &b[0] - } else { - v.Base = (*byte)(unsafe.Pointer(&_zero)) - } - vecs = append(vecs, v) - } - return vecs -} - -func writevRacedetect(iovecs []Iovec, n int) { - if !raceenabled { - return - } - for i := 0; n > 0 && i < len(iovecs); i++ { - m := int(iovecs[i].Len) - if m > n { - m = n - } - n -= m - if m > 0 { - raceReadRange(unsafe.Pointer(iovecs[i].Base), m) - } - } -} - -func readvRacedetect(iovecs []Iovec, n int, err error) { - if !raceenabled { - return - } - for i := 0; n > 0 && i < len(iovecs); i++ { - m := int(iovecs[i].Len) - if m > n { - m = n - } - n -= m - if m > 0 { - raceWriteRange(unsafe.Pointer(iovecs[i].Base), m) - } - } - if err == nil { - raceAcquire(unsafe.Pointer(&ioSync)) - } -} - //sys connectx(fd int, endpoints *SaEndpoints, associd SaeAssocID, flags uint32, iov []Iovec, n *uintptr, connid *SaeConnID) (err error) //sys sendfile(infd int, outfd int, offset int64, len *int64, hdtr unsafe.Pointer, flags int) (err error) diff --git a/vendor/golang.org/x/sys/unix/syscall_linux.go b/vendor/golang.org/x/sys/unix/syscall_linux.go index f7b82bccac3ed..ce4d7ab1ed8a4 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux.go @@ -2150,33 +2150,10 @@ func Signalfd(fd int, sigmask *Sigset_t, flags int) (newfd int, err error) { //sys exitThread(code int) (err error) = SYS_EXIT //sys readv(fd int, iovs []Iovec) (n int, err error) = SYS_READV //sys writev(fd int, iovs []Iovec) (n int, err error) = SYS_WRITEV -//sys preadv(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr) (n int, err error) = SYS_PREADV -//sys pwritev(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr) (n int, err error) = SYS_PWRITEV -//sys preadv2(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr, flags int) (n int, err error) = SYS_PREADV2 -//sys pwritev2(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr, flags int) (n int, err error) = SYS_PWRITEV2 - -// minIovec is the size of the small initial allocation used by -// Readv, Writev, etc. -// -// This small allocation gets stack allocated, which lets the -// common use case of len(iovs) <= minIovs avoid more expensive -// heap allocations. -const minIovec = 8 - -// appendBytes converts bs to Iovecs and appends them to vecs. -func appendBytes(vecs []Iovec, bs [][]byte) []Iovec { - for _, b := range bs { - var v Iovec - v.SetLen(len(b)) - if len(b) > 0 { - v.Base = &b[0] - } else { - v.Base = (*byte)(unsafe.Pointer(&_zero)) - } - vecs = append(vecs, v) - } - return vecs -} +//sys preadvSyscall(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr) (n int, err error) = SYS_PREADV +//sys pwritevSyscall(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr) (n int, err error) = SYS_PWRITEV +//sys preadv2Syscall(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr, flags int) (n int, err error) = SYS_PREADV2 +//sys pwritev2Syscall(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr, flags int) (n int, err error) = SYS_PWRITEV2 // offs2lohi splits offs into its low and high order bits. func offs2lohi(offs int64) (lo, hi uintptr) { @@ -2184,69 +2161,23 @@ func offs2lohi(offs int64) (lo, hi uintptr) { return uintptr(offs), uintptr(uint64(offs) >> (longBits - 1) >> 1) // two shifts to avoid false positive in vet } -func Readv(fd int, iovs [][]byte) (n int, err error) { - iovecs := make([]Iovec, 0, minIovec) - iovecs = appendBytes(iovecs, iovs) - n, err = readv(fd, iovecs) - readvRacedetect(iovecs, n, err) - return n, err -} - -func Preadv(fd int, iovs [][]byte, offset int64) (n int, err error) { - iovecs := make([]Iovec, 0, minIovec) - iovecs = appendBytes(iovecs, iovs) +func preadv(fd int, iovecs []Iovec, offset int64) (n int, err error) { lo, hi := offs2lohi(offset) - n, err = preadv(fd, iovecs, lo, hi) - readvRacedetect(iovecs, n, err) - return n, err + return preadvSyscall(fd, iovecs, lo, hi) } func Preadv2(fd int, iovs [][]byte, offset int64, flags int) (n int, err error) { iovecs := make([]Iovec, 0, minIovec) iovecs = appendBytes(iovecs, iovs) lo, hi := offs2lohi(offset) - n, err = preadv2(fd, iovecs, lo, hi, flags) - readvRacedetect(iovecs, n, err) - return n, err -} - -func readvRacedetect(iovecs []Iovec, n int, err error) { - if !raceenabled { - return - } - for i := 0; n > 0 && i < len(iovecs); i++ { - m := min(int(iovecs[i].Len), n) - n -= m - if m > 0 { - raceWriteRange(unsafe.Pointer(iovecs[i].Base), m) - } - } - if err == nil { - raceAcquire(unsafe.Pointer(&ioSync)) - } -} - -func Writev(fd int, iovs [][]byte) (n int, err error) { - iovecs := make([]Iovec, 0, minIovec) - iovecs = appendBytes(iovecs, iovs) - if raceenabled { - raceReleaseMerge(unsafe.Pointer(&ioSync)) - } - n, err = writev(fd, iovecs) - writevRacedetect(iovecs, n) + n, err = preadv2Syscall(fd, iovecs, lo, hi, flags) + readvRaceDetect(iovecs, n, err) return n, err } -func Pwritev(fd int, iovs [][]byte, offset int64) (n int, err error) { - iovecs := make([]Iovec, 0, minIovec) - iovecs = appendBytes(iovecs, iovs) - if raceenabled { - raceReleaseMerge(unsafe.Pointer(&ioSync)) - } +func pwritev(fd int, iovecs []Iovec, offset int64) (n int, err error) { lo, hi := offs2lohi(offset) - n, err = pwritev(fd, iovecs, lo, hi) - writevRacedetect(iovecs, n) - return n, err + return pwritevSyscall(fd, iovecs, lo, hi) } func Pwritev2(fd int, iovs [][]byte, offset int64, flags int) (n int, err error) { @@ -2256,24 +2187,11 @@ func Pwritev2(fd int, iovs [][]byte, offset int64, flags int) (n int, err error) raceReleaseMerge(unsafe.Pointer(&ioSync)) } lo, hi := offs2lohi(offset) - n, err = pwritev2(fd, iovecs, lo, hi, flags) - writevRacedetect(iovecs, n) + n, err = pwritev2Syscall(fd, iovecs, lo, hi, flags) + writevRaceDetect(iovecs, n) return n, err } -func writevRacedetect(iovecs []Iovec, n int) { - if !raceenabled { - return - } - for i := 0; n > 0 && i < len(iovecs); i++ { - m := min(int(iovecs[i].Len), n) - n -= m - if m > 0 { - raceReadRange(unsafe.Pointer(iovecs[i].Base), m) - } - } -} - // mmap varies by architecture; see syscall_linux_*.go. //sys munmap(addr uintptr, length uintptr) (err error) //sys mremap(oldaddr uintptr, oldlength uintptr, newlength uintptr, flags int, newaddr uintptr) (xaddr uintptr, err error) diff --git a/vendor/golang.org/x/sys/unix/syscall_openbsd.go b/vendor/golang.org/x/sys/unix/syscall_openbsd.go index b86ded549c662..7b0ef8e123984 100644 --- a/vendor/golang.org/x/sys/unix/syscall_openbsd.go +++ b/vendor/golang.org/x/sys/unix/syscall_openbsd.go @@ -300,6 +300,10 @@ func Uname(uname *Utsname) error { //sys Pathconf(path string, name int) (val int, err error) //sys pread(fd int, p []byte, offset int64) (n int, err error) //sys pwrite(fd int, p []byte, offset int64) (n int, err error) +//sys readv(fd int, iovecs []Iovec) (n int, err error) +//sys writev(fd int, iovecs []Iovec) (n int, err error) +//sys preadv(fd int, iovecs []Iovec, offset int64) (n int, err error) +//sys pwritev(fd int, iovecs []Iovec, offset int64) (n int, err error) //sys read(fd int, p []byte) (n int, err error) //sys Readlink(path string, buf []byte) (n int, err error) //sys Readlinkat(dirfd int, path string, buf []byte) (n int, err error) diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux.go b/vendor/golang.org/x/sys/unix/zerrors_linux.go index 120a7b35d1d4e..9d72a6b73a936 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux.go @@ -353,8 +353,10 @@ const ( AUDIT_MAC_IPSEC_EVENT = 0x587 AUDIT_MAC_MAP_ADD = 0x581 AUDIT_MAC_MAP_DEL = 0x582 + AUDIT_MAC_OBJ_CONTEXTS = 0x592 AUDIT_MAC_POLICY_LOAD = 0x57b AUDIT_MAC_STATUS = 0x57c + AUDIT_MAC_TASK_CONTEXTS = 0x591 AUDIT_MAC_UNLBL_ALLOW = 0x57e AUDIT_MAC_UNLBL_STCADD = 0x588 AUDIT_MAC_UNLBL_STCDEL = 0x589 @@ -591,8 +593,13 @@ const ( CAN_CTRLMODE_LOOPBACK = 0x1 CAN_CTRLMODE_ONE_SHOT = 0x8 CAN_CTRLMODE_PRESUME_ACK = 0x40 + CAN_CTRLMODE_RESTRICTED = 0x800 CAN_CTRLMODE_TDC_AUTO = 0x200 CAN_CTRLMODE_TDC_MANUAL = 0x400 + CAN_CTRLMODE_XL = 0x1000 + CAN_CTRLMODE_XL_TDC_AUTO = 0x2000 + CAN_CTRLMODE_XL_TDC_MANUAL = 0x4000 + CAN_CTRLMODE_XL_TMS = 0x8000 CAN_EFF_FLAG = 0x80000000 CAN_EFF_ID_BITS = 0x1d CAN_EFF_MASK = 0x1fffffff @@ -800,6 +807,8 @@ const ( DEVLINK_PORT_FN_CAP_IPSEC_PACKET = 0x8 DEVLINK_PORT_FN_CAP_MIGRATABLE = 0x2 DEVLINK_PORT_FN_CAP_ROCE = 0x1 + DEVLINK_RATE_TCS_MAX = 0x8 + DEVLINK_RATE_TC_INDEX_MAX = 0x7 DEVLINK_SB_THRESHOLD_TO_ALPHA_MAX = 0x14 DEVLINK_SUPPORTED_FLASH_OVERWRITE_SECTIONS = 0x3 DEVMEM_MAGIC = 0x454d444d @@ -1186,6 +1195,7 @@ const ( ETH_P_MPLS_UC = 0x8847 ETH_P_MRP = 0x88e3 ETH_P_MVRP = 0x88f5 + ETH_P_MXLGSW = 0x88c3 ETH_P_NCSI = 0x88f8 ETH_P_NSH = 0x894f ETH_P_PAE = 0x888e @@ -1218,6 +1228,7 @@ const ( ETH_P_WCCP = 0x883e ETH_P_X25 = 0x805 ETH_P_XDSA = 0xf8 + ETH_P_YT921X = 0x9988 ET_CORE = 0x4 ET_DYN = 0x3 ET_EXEC = 0x2 @@ -1258,6 +1269,7 @@ const ( FALLOC_FL_NO_HIDE_STALE = 0x4 FALLOC_FL_PUNCH_HOLE = 0x2 FALLOC_FL_UNSHARE_RANGE = 0x40 + FALLOC_FL_WRITE_ZEROES = 0x80 FALLOC_FL_ZERO_RANGE = 0x10 FANOTIFY_METADATA_VERSION = 0x3 FAN_ACCESS = 0x1 @@ -1477,6 +1489,7 @@ const ( GRND_INSECURE = 0x4 GRND_NONBLOCK = 0x1 GRND_RANDOM = 0x2 + GUEST_MEMFD_MAGIC = 0x474d454d HDIO_DRIVE_CMD = 0x31f HDIO_DRIVE_CMD_AEB = 0x31e HDIO_DRIVE_CMD_HDR_SIZE = 0x4 @@ -1517,6 +1530,7 @@ const ( HDIO_SET_XFER = 0x306 HDIO_TRISTATE_HWIF = 0x31b HDIO_UNREGISTER_HWIF = 0x32a + HIDIOCTL_LAST = 0xd HID_MAX_DESCRIPTOR_SIZE = 0x1000 HOSTFS_SUPER_MAGIC = 0xc0ffee HPFS_SUPER_MAGIC = 0xf995e849 @@ -1809,6 +1823,8 @@ const ( KEXEC_ARCH_X86_64 = 0x3e0000 KEXEC_CRASH_HOTPLUG_SUPPORT = 0x8 KEXEC_FILE_DEBUG = 0x8 + KEXEC_FILE_FORCE_DTB = 0x20 + KEXEC_FILE_NO_CMA = 0x10 KEXEC_FILE_NO_INITRAMFS = 0x4 KEXEC_FILE_ON_CRASH = 0x2 KEXEC_FILE_UNLOAD = 0x1 @@ -1905,6 +1921,7 @@ const ( LANDLOCK_RESTRICT_SELF_LOG_NEW_EXEC_ON = 0x2 LANDLOCK_RESTRICT_SELF_LOG_SAME_EXEC_OFF = 0x1 LANDLOCK_RESTRICT_SELF_LOG_SUBDOMAINS_OFF = 0x4 + LANDLOCK_RESTRICT_SELF_TSYNC = 0x8 LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET = 0x1 LANDLOCK_SCOPE_SIGNAL = 0x2 LINUX_REBOOT_CMD_CAD_OFF = 0x0 @@ -2412,6 +2429,7 @@ const ( NN_PRXFPREG = "LINUX" NN_RISCV_CSR = "LINUX" NN_RISCV_TAGGED_ADDR_CTRL = "LINUX" + NN_RISCV_USER_CFI = "LINUX" NN_RISCV_VECTOR = "LINUX" NN_S390_CTRS = "LINUX" NN_S390_GS_BC = "LINUX" @@ -2493,6 +2511,7 @@ const ( NT_PRXFPREG = 0x46e62b7f NT_RISCV_CSR = 0x900 NT_RISCV_TAGGED_ADDR_CTRL = 0x902 + NT_RISCV_USER_CFI = 0x903 NT_RISCV_VECTOR = 0x901 NT_S390_CTRS = 0x304 NT_S390_GS_BC = 0x30c @@ -2515,6 +2534,7 @@ const ( NT_X86_SHSTK = 0x204 NT_X86_XSAVE_LAYOUT = 0x205 NT_X86_XSTATE = 0x202 + NULL_FS_MAGIC = 0x4e554c4c OCFS2_SUPER_MAGIC = 0x7461636f OCRNL = 0x8 OFDEL = 0x80 @@ -2594,6 +2614,7 @@ const ( PERF_ATTR_SIZE_VER6 = 0x78 PERF_ATTR_SIZE_VER7 = 0x80 PERF_ATTR_SIZE_VER8 = 0x88 + PERF_ATTR_SIZE_VER9 = 0x90 PERF_AUX_FLAG_COLLISION = 0x8 PERF_AUX_FLAG_CORESIGHT_FORMAT_CORESIGHT = 0x0 PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW = 0x100 @@ -2629,6 +2650,7 @@ const ( PERF_MEM_LVLNUM_ANY_CACHE = 0xb PERF_MEM_LVLNUM_CXL = 0x9 PERF_MEM_LVLNUM_IO = 0xa + PERF_MEM_LVLNUM_L0 = 0x7 PERF_MEM_LVLNUM_L1 = 0x1 PERF_MEM_LVLNUM_L2 = 0x2 PERF_MEM_LVLNUM_L2_MHB = 0x5 @@ -2662,6 +2684,23 @@ const ( PERF_MEM_OP_PFETCH = 0x8 PERF_MEM_OP_SHIFT = 0x0 PERF_MEM_OP_STORE = 0x4 + PERF_MEM_REGION_L_NON_SHARE = 0x3 + PERF_MEM_REGION_L_SHARE = 0x2 + PERF_MEM_REGION_MEM0 = 0x8 + PERF_MEM_REGION_MEM1 = 0x9 + PERF_MEM_REGION_MEM2 = 0xa + PERF_MEM_REGION_MEM3 = 0xb + PERF_MEM_REGION_MEM4 = 0xc + PERF_MEM_REGION_MEM5 = 0xd + PERF_MEM_REGION_MEM6 = 0xe + PERF_MEM_REGION_MEM7 = 0xf + PERF_MEM_REGION_MMIO = 0x7 + PERF_MEM_REGION_NA = 0x0 + PERF_MEM_REGION_O_IO = 0x4 + PERF_MEM_REGION_O_NON_SHARE = 0x6 + PERF_MEM_REGION_O_SHARE = 0x5 + PERF_MEM_REGION_RSVD = 0x1 + PERF_MEM_REGION_SHIFT = 0x2e PERF_MEM_REMOTE_REMOTE = 0x1 PERF_MEM_REMOTE_SHIFT = 0x25 PERF_MEM_SNOOPX_FWD = 0x1 @@ -2776,6 +2815,10 @@ const ( PR_CAP_AMBIENT_IS_SET = 0x1 PR_CAP_AMBIENT_LOWER = 0x3 PR_CAP_AMBIENT_RAISE = 0x2 + PR_CFI_BRANCH_LANDING_PADS = 0x0 + PR_CFI_DISABLE = 0x2 + PR_CFI_ENABLE = 0x1 + PR_CFI_LOCK = 0x4 PR_ENDIAN_BIG = 0x0 PR_ENDIAN_LITTLE = 0x1 PR_ENDIAN_PPC_LITTLE = 0x2 @@ -2798,6 +2841,7 @@ const ( PR_FUTEX_HASH_GET_SLOTS = 0x2 PR_FUTEX_HASH_SET_SLOTS = 0x1 PR_GET_AUXV = 0x41555856 + PR_GET_CFI = 0x50 PR_GET_CHILD_SUBREAPER = 0x25 PR_GET_DUMPABLE = 0x3 PR_GET_ENDIAN = 0x13 @@ -2834,6 +2878,7 @@ const ( PR_MDWE_REFUSE_EXEC_GAIN = 0x1 PR_MPX_DISABLE_MANAGEMENT = 0x2c PR_MPX_ENABLE_MANAGEMENT = 0x2b + PR_MTE_STORE_ONLY = 0x80000 PR_MTE_TAG_MASK = 0x7fff8 PR_MTE_TAG_SHIFT = 0x3 PR_MTE_TCF_ASYNC = 0x4 @@ -2877,6 +2922,10 @@ const ( PR_RISCV_V_VSTATE_CTRL_NEXT_MASK = 0xc PR_RISCV_V_VSTATE_CTRL_OFF = 0x1 PR_RISCV_V_VSTATE_CTRL_ON = 0x2 + PR_RSEQ_SLICE_EXTENSION = 0x4f + PR_RSEQ_SLICE_EXTENSION_GET = 0x1 + PR_RSEQ_SLICE_EXTENSION_SET = 0x2 + PR_RSEQ_SLICE_EXT_ENABLE = 0x1 PR_SCHED_CORE = 0x3e PR_SCHED_CORE_CREATE = 0x1 PR_SCHED_CORE_GET = 0x0 @@ -2886,6 +2935,7 @@ const ( PR_SCHED_CORE_SCOPE_THREAD_GROUP = 0x1 PR_SCHED_CORE_SHARE_FROM = 0x3 PR_SCHED_CORE_SHARE_TO = 0x2 + PR_SET_CFI = 0x51 PR_SET_CHILD_SUBREAPER = 0x24 PR_SET_DUMPABLE = 0x4 PR_SET_ENDIAN = 0x14 @@ -2951,11 +3001,14 @@ const ( PR_SVE_SET_VL_ONEXEC = 0x40000 PR_SVE_VL_INHERIT = 0x20000 PR_SVE_VL_LEN_MASK = 0xffff + PR_SYS_DISPATCH_EXCLUSIVE_ON = 0x1 + PR_SYS_DISPATCH_INCLUSIVE_ON = 0x2 PR_SYS_DISPATCH_OFF = 0x0 PR_SYS_DISPATCH_ON = 0x1 PR_TAGGED_ADDR_ENABLE = 0x1 PR_TASK_PERF_EVENTS_DISABLE = 0x1f PR_TASK_PERF_EVENTS_ENABLE = 0x20 + PR_THP_DISABLE_EXCEPT_ADVISED = 0x2 PR_TIMER_CREATE_RESTORE_IDS = 0x4d PR_TIMER_CREATE_RESTORE_IDS_GET = 0x2 PR_TIMER_CREATE_RESTORE_IDS_OFF = 0x0 @@ -2987,8 +3040,10 @@ const ( PTP_STRICT_FLAGS = 0x8 PTP_SYS_OFFSET_EXTENDED = 0xc4c03d09 PTP_SYS_OFFSET_EXTENDED2 = 0xc4c03d12 + PTP_SYS_OFFSET_EXTENDED_CYCLES = 0xc4c03d16 PTP_SYS_OFFSET_PRECISE = 0xc0403d08 PTP_SYS_OFFSET_PRECISE2 = 0xc0403d11 + PTP_SYS_OFFSET_PRECISE_CYCLES = 0xc0403d15 PTRACE_ATTACH = 0x10 PTRACE_CONT = 0x7 PTRACE_DETACH = 0x11 @@ -3330,8 +3385,9 @@ const ( RWF_DSYNC = 0x2 RWF_HIPRI = 0x1 RWF_NOAPPEND = 0x20 + RWF_NOSIGNAL = 0x100 RWF_NOWAIT = 0x8 - RWF_SUPPORTED = 0xff + RWF_SUPPORTED = 0x1ff RWF_SYNC = 0x4 RWF_WRITE_LIFE_NOT_SET = 0x0 SCHED_BATCH = 0x3 @@ -3714,7 +3770,7 @@ const ( TASKSTATS_GENL_NAME = "TASKSTATS" TASKSTATS_GENL_VERSION = 0x1 TASKSTATS_TYPE_MAX = 0x6 - TASKSTATS_VERSION = 0x10 + TASKSTATS_VERSION = 0x11 TCIFLUSH = 0x0 TCIOFF = 0x2 TCIOFLUSH = 0x2 @@ -4052,6 +4108,7 @@ const ( XDP_FLAGS_REPLACE = 0x10 XDP_FLAGS_SKB_MODE = 0x2 XDP_FLAGS_UPDATE_IF_NOEXIST = 0x1 + XDP_MAX_TX_SKB_BUDGET = 0x9 XDP_MMAP_OFFSETS = 0x1 XDP_OPTIONS = 0x8 XDP_OPTIONS_ZEROCOPY = 0x1 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_386.go b/vendor/golang.org/x/sys/unix/zerrors_linux_386.go index 97a61fc5b8459..c0a8ea1de0e57 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_386.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_386.go @@ -159,6 +159,7 @@ const ( NFDBITS = 0x20 NLDLY = 0x100 NOFLSH = 0x80 + NS_GET_ID = 0x8008b70d NS_GET_MNTNS_ID = 0x8008b705 NS_GET_NSTYPE = 0xb703 NS_GET_OWNER_UID = 0xb704 @@ -305,6 +306,7 @@ const ( RTC_WKALM_SET = 0x4028700f SCM_DEVMEM_DMABUF = 0x4f SCM_DEVMEM_LINEAR = 0x4e + SCM_INQ = 0x54 SCM_TIMESTAMPING = 0x25 SCM_TIMESTAMPING_OPT_STATS = 0x36 SCM_TIMESTAMPING_PKTINFO = 0x3a @@ -352,6 +354,7 @@ const ( SO_ERROR = 0x4 SO_INCOMING_CPU = 0x31 SO_INCOMING_NAPI_ID = 0x38 + SO_INQ = 0x54 SO_KEEPALIVE = 0x9 SO_LINGER = 0xd SO_LOCK_FILTER = 0x2c @@ -596,6 +599,8 @@ const ( EDESTADDRREQ = syscall.Errno(0x59) EDOTDOT = syscall.Errno(0x49) EDQUOT = syscall.Errno(0x7a) + EFSBADCRC = syscall.Errno(0x4a) + EFSCORRUPTED = syscall.Errno(0x75) EHOSTDOWN = syscall.Errno(0x70) EHOSTUNREACH = syscall.Errno(0x71) EHWPOISON = syscall.Errno(0x85) @@ -819,7 +824,7 @@ var errorList = [...]struct { {114, "EALREADY", "operation already in progress"}, {115, "EINPROGRESS", "operation now in progress"}, {116, "ESTALE", "stale file handle"}, - {117, "EUCLEAN", "structure needs cleaning"}, + {117, "EFSCORRUPTED", "structure needs cleaning"}, {118, "ENOTNAM", "not a XENIX named type file"}, {119, "ENAVAIL", "no XENIX semaphores available"}, {120, "EISNAM", "is a named type file"}, diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go index a0d6d498c4b7e..ff927c830100f 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go @@ -159,6 +159,7 @@ const ( NFDBITS = 0x40 NLDLY = 0x100 NOFLSH = 0x80 + NS_GET_ID = 0x8008b70d NS_GET_MNTNS_ID = 0x8008b705 NS_GET_NSTYPE = 0xb703 NS_GET_OWNER_UID = 0xb704 @@ -306,6 +307,7 @@ const ( RTC_WKALM_SET = 0x4028700f SCM_DEVMEM_DMABUF = 0x4f SCM_DEVMEM_LINEAR = 0x4e + SCM_INQ = 0x54 SCM_TIMESTAMPING = 0x25 SCM_TIMESTAMPING_OPT_STATS = 0x36 SCM_TIMESTAMPING_PKTINFO = 0x3a @@ -353,6 +355,7 @@ const ( SO_ERROR = 0x4 SO_INCOMING_CPU = 0x31 SO_INCOMING_NAPI_ID = 0x38 + SO_INQ = 0x54 SO_KEEPALIVE = 0x9 SO_LINGER = 0xd SO_LOCK_FILTER = 0x2c @@ -596,6 +599,8 @@ const ( EDESTADDRREQ = syscall.Errno(0x59) EDOTDOT = syscall.Errno(0x49) EDQUOT = syscall.Errno(0x7a) + EFSBADCRC = syscall.Errno(0x4a) + EFSCORRUPTED = syscall.Errno(0x75) EHOSTDOWN = syscall.Errno(0x70) EHOSTUNREACH = syscall.Errno(0x71) EHWPOISON = syscall.Errno(0x85) @@ -819,7 +824,7 @@ var errorList = [...]struct { {114, "EALREADY", "operation already in progress"}, {115, "EINPROGRESS", "operation now in progress"}, {116, "ESTALE", "stale file handle"}, - {117, "EUCLEAN", "structure needs cleaning"}, + {117, "EFSCORRUPTED", "structure needs cleaning"}, {118, "ENOTNAM", "not a XENIX named type file"}, {119, "ENAVAIL", "no XENIX semaphores available"}, {120, "EISNAM", "is a named type file"}, diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go b/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go index dd9c903f9adcd..55294eda5bce6 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go @@ -156,6 +156,7 @@ const ( NFDBITS = 0x20 NLDLY = 0x100 NOFLSH = 0x80 + NS_GET_ID = 0x8008b70d NS_GET_MNTNS_ID = 0x8008b705 NS_GET_NSTYPE = 0xb703 NS_GET_OWNER_UID = 0xb704 @@ -311,6 +312,7 @@ const ( RTC_WKALM_SET = 0x4028700f SCM_DEVMEM_DMABUF = 0x4f SCM_DEVMEM_LINEAR = 0x4e + SCM_INQ = 0x54 SCM_TIMESTAMPING = 0x25 SCM_TIMESTAMPING_OPT_STATS = 0x36 SCM_TIMESTAMPING_PKTINFO = 0x3a @@ -358,6 +360,7 @@ const ( SO_ERROR = 0x4 SO_INCOMING_CPU = 0x31 SO_INCOMING_NAPI_ID = 0x38 + SO_INQ = 0x54 SO_KEEPALIVE = 0x9 SO_LINGER = 0xd SO_LOCK_FILTER = 0x2c @@ -601,6 +604,8 @@ const ( EDESTADDRREQ = syscall.Errno(0x59) EDOTDOT = syscall.Errno(0x49) EDQUOT = syscall.Errno(0x7a) + EFSBADCRC = syscall.Errno(0x4a) + EFSCORRUPTED = syscall.Errno(0x75) EHOSTDOWN = syscall.Errno(0x70) EHOSTUNREACH = syscall.Errno(0x71) EHWPOISON = syscall.Errno(0x85) @@ -824,7 +829,7 @@ var errorList = [...]struct { {114, "EALREADY", "operation already in progress"}, {115, "EINPROGRESS", "operation now in progress"}, {116, "ESTALE", "stale file handle"}, - {117, "EUCLEAN", "structure needs cleaning"}, + {117, "EFSCORRUPTED", "structure needs cleaning"}, {118, "ENOTNAM", "not a XENIX named type file"}, {119, "ENAVAIL", "no XENIX semaphores available"}, {120, "EISNAM", "is a named type file"}, diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go index 384c61ca3a85e..5dac54c35cce5 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go @@ -161,6 +161,7 @@ const ( NFDBITS = 0x40 NLDLY = 0x100 NOFLSH = 0x80 + NS_GET_ID = 0x8008b70d NS_GET_MNTNS_ID = 0x8008b705 NS_GET_NSTYPE = 0xb703 NS_GET_OWNER_UID = 0xb704 @@ -304,6 +305,7 @@ const ( RTC_WKALM_SET = 0x4028700f SCM_DEVMEM_DMABUF = 0x4f SCM_DEVMEM_LINEAR = 0x4e + SCM_INQ = 0x54 SCM_TIMESTAMPING = 0x25 SCM_TIMESTAMPING_OPT_STATS = 0x36 SCM_TIMESTAMPING_PKTINFO = 0x3a @@ -351,6 +353,7 @@ const ( SO_ERROR = 0x4 SO_INCOMING_CPU = 0x31 SO_INCOMING_NAPI_ID = 0x38 + SO_INQ = 0x54 SO_KEEPALIVE = 0x9 SO_LINGER = 0xd SO_LOCK_FILTER = 0x2c @@ -598,6 +601,8 @@ const ( EDESTADDRREQ = syscall.Errno(0x59) EDOTDOT = syscall.Errno(0x49) EDQUOT = syscall.Errno(0x7a) + EFSBADCRC = syscall.Errno(0x4a) + EFSCORRUPTED = syscall.Errno(0x75) EHOSTDOWN = syscall.Errno(0x70) EHOSTUNREACH = syscall.Errno(0x71) EHWPOISON = syscall.Errno(0x85) @@ -821,7 +826,7 @@ var errorList = [...]struct { {114, "EALREADY", "operation already in progress"}, {115, "EINPROGRESS", "operation now in progress"}, {116, "ESTALE", "stale file handle"}, - {117, "EUCLEAN", "structure needs cleaning"}, + {117, "EFSCORRUPTED", "structure needs cleaning"}, {118, "ENOTNAM", "not a XENIX named type file"}, {119, "ENAVAIL", "no XENIX semaphores available"}, {120, "EISNAM", "is a named type file"}, diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_loong64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_loong64.go index 6384c9831fc23..46ac1fcb22027 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_loong64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_loong64.go @@ -160,6 +160,7 @@ const ( NFDBITS = 0x40 NLDLY = 0x100 NOFLSH = 0x80 + NS_GET_ID = 0x8008b70d NS_GET_MNTNS_ID = 0x8008b705 NS_GET_NSTYPE = 0xb703 NS_GET_OWNER_UID = 0xb704 @@ -298,6 +299,7 @@ const ( RTC_WKALM_SET = 0x4028700f SCM_DEVMEM_DMABUF = 0x4f SCM_DEVMEM_LINEAR = 0x4e + SCM_INQ = 0x54 SCM_TIMESTAMPING = 0x25 SCM_TIMESTAMPING_OPT_STATS = 0x36 SCM_TIMESTAMPING_PKTINFO = 0x3a @@ -345,6 +347,7 @@ const ( SO_ERROR = 0x4 SO_INCOMING_CPU = 0x31 SO_INCOMING_NAPI_ID = 0x38 + SO_INQ = 0x54 SO_KEEPALIVE = 0x9 SO_LINGER = 0xd SO_LOCK_FILTER = 0x2c @@ -588,6 +591,8 @@ const ( EDESTADDRREQ = syscall.Errno(0x59) EDOTDOT = syscall.Errno(0x49) EDQUOT = syscall.Errno(0x7a) + EFSBADCRC = syscall.Errno(0x4a) + EFSCORRUPTED = syscall.Errno(0x75) EHOSTDOWN = syscall.Errno(0x70) EHOSTUNREACH = syscall.Errno(0x71) EHWPOISON = syscall.Errno(0x85) @@ -811,7 +816,7 @@ var errorList = [...]struct { {114, "EALREADY", "operation already in progress"}, {115, "EINPROGRESS", "operation now in progress"}, {116, "ESTALE", "stale file handle"}, - {117, "EUCLEAN", "structure needs cleaning"}, + {117, "EFSCORRUPTED", "structure needs cleaning"}, {118, "ENOTNAM", "not a XENIX named type file"}, {119, "ENAVAIL", "no XENIX semaphores available"}, {120, "EISNAM", "is a named type file"}, diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go index 553c1c6f15e5d..b55483e8a9806 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go @@ -156,6 +156,7 @@ const ( NFDBITS = 0x20 NLDLY = 0x100 NOFLSH = 0x80 + NS_GET_ID = 0x4008b70d NS_GET_MNTNS_ID = 0x4008b705 NS_GET_NSTYPE = 0x2000b703 NS_GET_OWNER_UID = 0x2000b704 @@ -304,6 +305,7 @@ const ( RTC_WKALM_SET = 0x8028700f SCM_DEVMEM_DMABUF = 0x4f SCM_DEVMEM_LINEAR = 0x4e + SCM_INQ = 0x54 SCM_TIMESTAMPING = 0x25 SCM_TIMESTAMPING_OPT_STATS = 0x36 SCM_TIMESTAMPING_PKTINFO = 0x3a @@ -351,6 +353,7 @@ const ( SO_ERROR = 0x1007 SO_INCOMING_CPU = 0x31 SO_INCOMING_NAPI_ID = 0x38 + SO_INQ = 0x54 SO_KEEPALIVE = 0x8 SO_LINGER = 0x80 SO_LOCK_FILTER = 0x2c @@ -597,6 +600,8 @@ const ( EDESTADDRREQ = syscall.Errno(0x60) EDOTDOT = syscall.Errno(0x49) EDQUOT = syscall.Errno(0x46d) + EFSBADCRC = syscall.Errno(0x4d) + EFSCORRUPTED = syscall.Errno(0x87) EHOSTDOWN = syscall.Errno(0x93) EHOSTUNREACH = syscall.Errno(0x94) EHWPOISON = syscall.Errno(0xa8) @@ -814,7 +819,7 @@ var errorList = [...]struct { {132, "ENOBUFS", "no buffer space available"}, {133, "EISCONN", "transport endpoint is already connected"}, {134, "ENOTCONN", "transport endpoint is not connected"}, - {135, "EUCLEAN", "structure needs cleaning"}, + {135, "EFSCORRUPTED", "structure needs cleaning"}, {137, "ENOTNAM", "not a XENIX named type file"}, {138, "ENAVAIL", "no XENIX semaphores available"}, {139, "EISNAM", "is a named type file"}, diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go index b3339f2099afb..71890c98a40df 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go @@ -156,6 +156,7 @@ const ( NFDBITS = 0x40 NLDLY = 0x100 NOFLSH = 0x80 + NS_GET_ID = 0x4008b70d NS_GET_MNTNS_ID = 0x4008b705 NS_GET_NSTYPE = 0x2000b703 NS_GET_OWNER_UID = 0x2000b704 @@ -304,6 +305,7 @@ const ( RTC_WKALM_SET = 0x8028700f SCM_DEVMEM_DMABUF = 0x4f SCM_DEVMEM_LINEAR = 0x4e + SCM_INQ = 0x54 SCM_TIMESTAMPING = 0x25 SCM_TIMESTAMPING_OPT_STATS = 0x36 SCM_TIMESTAMPING_PKTINFO = 0x3a @@ -351,6 +353,7 @@ const ( SO_ERROR = 0x1007 SO_INCOMING_CPU = 0x31 SO_INCOMING_NAPI_ID = 0x38 + SO_INQ = 0x54 SO_KEEPALIVE = 0x8 SO_LINGER = 0x80 SO_LOCK_FILTER = 0x2c @@ -597,6 +600,8 @@ const ( EDESTADDRREQ = syscall.Errno(0x60) EDOTDOT = syscall.Errno(0x49) EDQUOT = syscall.Errno(0x46d) + EFSBADCRC = syscall.Errno(0x4d) + EFSCORRUPTED = syscall.Errno(0x87) EHOSTDOWN = syscall.Errno(0x93) EHOSTUNREACH = syscall.Errno(0x94) EHWPOISON = syscall.Errno(0xa8) @@ -814,7 +819,7 @@ var errorList = [...]struct { {132, "ENOBUFS", "no buffer space available"}, {133, "EISCONN", "transport endpoint is already connected"}, {134, "ENOTCONN", "transport endpoint is not connected"}, - {135, "EUCLEAN", "structure needs cleaning"}, + {135, "EFSCORRUPTED", "structure needs cleaning"}, {137, "ENOTNAM", "not a XENIX named type file"}, {138, "ENAVAIL", "no XENIX semaphores available"}, {139, "EISNAM", "is a named type file"}, diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go index 177091d2bc372..a78b6cc14bf85 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go @@ -156,6 +156,7 @@ const ( NFDBITS = 0x40 NLDLY = 0x100 NOFLSH = 0x80 + NS_GET_ID = 0x4008b70d NS_GET_MNTNS_ID = 0x4008b705 NS_GET_NSTYPE = 0x2000b703 NS_GET_OWNER_UID = 0x2000b704 @@ -304,6 +305,7 @@ const ( RTC_WKALM_SET = 0x8028700f SCM_DEVMEM_DMABUF = 0x4f SCM_DEVMEM_LINEAR = 0x4e + SCM_INQ = 0x54 SCM_TIMESTAMPING = 0x25 SCM_TIMESTAMPING_OPT_STATS = 0x36 SCM_TIMESTAMPING_PKTINFO = 0x3a @@ -351,6 +353,7 @@ const ( SO_ERROR = 0x1007 SO_INCOMING_CPU = 0x31 SO_INCOMING_NAPI_ID = 0x38 + SO_INQ = 0x54 SO_KEEPALIVE = 0x8 SO_LINGER = 0x80 SO_LOCK_FILTER = 0x2c @@ -597,6 +600,8 @@ const ( EDESTADDRREQ = syscall.Errno(0x60) EDOTDOT = syscall.Errno(0x49) EDQUOT = syscall.Errno(0x46d) + EFSBADCRC = syscall.Errno(0x4d) + EFSCORRUPTED = syscall.Errno(0x87) EHOSTDOWN = syscall.Errno(0x93) EHOSTUNREACH = syscall.Errno(0x94) EHWPOISON = syscall.Errno(0xa8) @@ -814,7 +819,7 @@ var errorList = [...]struct { {132, "ENOBUFS", "no buffer space available"}, {133, "EISCONN", "transport endpoint is already connected"}, {134, "ENOTCONN", "transport endpoint is not connected"}, - {135, "EUCLEAN", "structure needs cleaning"}, + {135, "EFSCORRUPTED", "structure needs cleaning"}, {137, "ENOTNAM", "not a XENIX named type file"}, {138, "ENAVAIL", "no XENIX semaphores available"}, {139, "EISNAM", "is a named type file"}, diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go index c5abf156d09ff..d0e38ca7390b7 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go @@ -156,6 +156,7 @@ const ( NFDBITS = 0x20 NLDLY = 0x100 NOFLSH = 0x80 + NS_GET_ID = 0x4008b70d NS_GET_MNTNS_ID = 0x4008b705 NS_GET_NSTYPE = 0x2000b703 NS_GET_OWNER_UID = 0x2000b704 @@ -304,6 +305,7 @@ const ( RTC_WKALM_SET = 0x8028700f SCM_DEVMEM_DMABUF = 0x4f SCM_DEVMEM_LINEAR = 0x4e + SCM_INQ = 0x54 SCM_TIMESTAMPING = 0x25 SCM_TIMESTAMPING_OPT_STATS = 0x36 SCM_TIMESTAMPING_PKTINFO = 0x3a @@ -351,6 +353,7 @@ const ( SO_ERROR = 0x1007 SO_INCOMING_CPU = 0x31 SO_INCOMING_NAPI_ID = 0x38 + SO_INQ = 0x54 SO_KEEPALIVE = 0x8 SO_LINGER = 0x80 SO_LOCK_FILTER = 0x2c @@ -597,6 +600,8 @@ const ( EDESTADDRREQ = syscall.Errno(0x60) EDOTDOT = syscall.Errno(0x49) EDQUOT = syscall.Errno(0x46d) + EFSBADCRC = syscall.Errno(0x4d) + EFSCORRUPTED = syscall.Errno(0x87) EHOSTDOWN = syscall.Errno(0x93) EHOSTUNREACH = syscall.Errno(0x94) EHWPOISON = syscall.Errno(0xa8) @@ -814,7 +819,7 @@ var errorList = [...]struct { {132, "ENOBUFS", "no buffer space available"}, {133, "EISCONN", "transport endpoint is already connected"}, {134, "ENOTCONN", "transport endpoint is not connected"}, - {135, "EUCLEAN", "structure needs cleaning"}, + {135, "EFSCORRUPTED", "structure needs cleaning"}, {137, "ENOTNAM", "not a XENIX named type file"}, {138, "ENAVAIL", "no XENIX semaphores available"}, {139, "EISNAM", "is a named type file"}, diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc.go b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc.go index f1f3fadf576af..c883e14c772a3 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc.go @@ -158,6 +158,7 @@ const ( NL3 = 0x300 NLDLY = 0x300 NOFLSH = 0x80000000 + NS_GET_ID = 0x4008b70d NS_GET_MNTNS_ID = 0x4008b705 NS_GET_NSTYPE = 0x2000b703 NS_GET_OWNER_UID = 0x2000b704 @@ -359,6 +360,7 @@ const ( RTC_WKALM_SET = 0x8028700f SCM_DEVMEM_DMABUF = 0x4f SCM_DEVMEM_LINEAR = 0x4e + SCM_INQ = 0x54 SCM_TIMESTAMPING = 0x25 SCM_TIMESTAMPING_OPT_STATS = 0x36 SCM_TIMESTAMPING_PKTINFO = 0x3a @@ -406,6 +408,7 @@ const ( SO_ERROR = 0x4 SO_INCOMING_CPU = 0x31 SO_INCOMING_NAPI_ID = 0x38 + SO_INQ = 0x54 SO_KEEPALIVE = 0x9 SO_LINGER = 0xd SO_LOCK_FILTER = 0x2c @@ -653,6 +656,8 @@ const ( EDESTADDRREQ = syscall.Errno(0x59) EDOTDOT = syscall.Errno(0x49) EDQUOT = syscall.Errno(0x7a) + EFSBADCRC = syscall.Errno(0x4a) + EFSCORRUPTED = syscall.Errno(0x75) EHOSTDOWN = syscall.Errno(0x70) EHOSTUNREACH = syscall.Errno(0x71) EHWPOISON = syscall.Errno(0x85) @@ -877,7 +882,7 @@ var errorList = [...]struct { {114, "EALREADY", "operation already in progress"}, {115, "EINPROGRESS", "operation now in progress"}, {116, "ESTALE", "stale file handle"}, - {117, "EUCLEAN", "structure needs cleaning"}, + {117, "EFSCORRUPTED", "structure needs cleaning"}, {118, "ENOTNAM", "not a XENIX named type file"}, {119, "ENAVAIL", "no XENIX semaphores available"}, {120, "EISNAM", "is a named type file"}, diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go index 203ad9c54af2e..1834273d44a19 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go @@ -158,6 +158,7 @@ const ( NL3 = 0x300 NLDLY = 0x300 NOFLSH = 0x80000000 + NS_GET_ID = 0x4008b70d NS_GET_MNTNS_ID = 0x4008b705 NS_GET_NSTYPE = 0x2000b703 NS_GET_OWNER_UID = 0x2000b704 @@ -363,6 +364,7 @@ const ( RTC_WKALM_SET = 0x8028700f SCM_DEVMEM_DMABUF = 0x4f SCM_DEVMEM_LINEAR = 0x4e + SCM_INQ = 0x54 SCM_TIMESTAMPING = 0x25 SCM_TIMESTAMPING_OPT_STATS = 0x36 SCM_TIMESTAMPING_PKTINFO = 0x3a @@ -410,6 +412,7 @@ const ( SO_ERROR = 0x4 SO_INCOMING_CPU = 0x31 SO_INCOMING_NAPI_ID = 0x38 + SO_INQ = 0x54 SO_KEEPALIVE = 0x9 SO_LINGER = 0xd SO_LOCK_FILTER = 0x2c @@ -657,6 +660,8 @@ const ( EDESTADDRREQ = syscall.Errno(0x59) EDOTDOT = syscall.Errno(0x49) EDQUOT = syscall.Errno(0x7a) + EFSBADCRC = syscall.Errno(0x4a) + EFSCORRUPTED = syscall.Errno(0x75) EHOSTDOWN = syscall.Errno(0x70) EHOSTUNREACH = syscall.Errno(0x71) EHWPOISON = syscall.Errno(0x85) @@ -881,7 +886,7 @@ var errorList = [...]struct { {114, "EALREADY", "operation already in progress"}, {115, "EINPROGRESS", "operation now in progress"}, {116, "ESTALE", "stale file handle"}, - {117, "EUCLEAN", "structure needs cleaning"}, + {117, "EFSCORRUPTED", "structure needs cleaning"}, {118, "ENOTNAM", "not a XENIX named type file"}, {119, "ENAVAIL", "no XENIX semaphores available"}, {120, "EISNAM", "is a named type file"}, diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go index 4b9abcb21a230..39945dd9aec8a 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go @@ -158,6 +158,7 @@ const ( NL3 = 0x300 NLDLY = 0x300 NOFLSH = 0x80000000 + NS_GET_ID = 0x4008b70d NS_GET_MNTNS_ID = 0x4008b705 NS_GET_NSTYPE = 0x2000b703 NS_GET_OWNER_UID = 0x2000b704 @@ -363,6 +364,7 @@ const ( RTC_WKALM_SET = 0x8028700f SCM_DEVMEM_DMABUF = 0x4f SCM_DEVMEM_LINEAR = 0x4e + SCM_INQ = 0x54 SCM_TIMESTAMPING = 0x25 SCM_TIMESTAMPING_OPT_STATS = 0x36 SCM_TIMESTAMPING_PKTINFO = 0x3a @@ -410,6 +412,7 @@ const ( SO_ERROR = 0x4 SO_INCOMING_CPU = 0x31 SO_INCOMING_NAPI_ID = 0x38 + SO_INQ = 0x54 SO_KEEPALIVE = 0x9 SO_LINGER = 0xd SO_LOCK_FILTER = 0x2c @@ -657,6 +660,8 @@ const ( EDESTADDRREQ = syscall.Errno(0x59) EDOTDOT = syscall.Errno(0x49) EDQUOT = syscall.Errno(0x7a) + EFSBADCRC = syscall.Errno(0x4a) + EFSCORRUPTED = syscall.Errno(0x75) EHOSTDOWN = syscall.Errno(0x70) EHOSTUNREACH = syscall.Errno(0x71) EHWPOISON = syscall.Errno(0x85) @@ -881,7 +886,7 @@ var errorList = [...]struct { {114, "EALREADY", "operation already in progress"}, {115, "EINPROGRESS", "operation now in progress"}, {116, "ESTALE", "stale file handle"}, - {117, "EUCLEAN", "structure needs cleaning"}, + {117, "EFSCORRUPTED", "structure needs cleaning"}, {118, "ENOTNAM", "not a XENIX named type file"}, {119, "ENAVAIL", "no XENIX semaphores available"}, {120, "EISNAM", "is a named type file"}, diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go index f87983037d97c..bc0f372416fe1 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go @@ -11,553 +11,569 @@ package unix import "syscall" const ( - B1000000 = 0x1008 - B115200 = 0x1002 - B1152000 = 0x1009 - B1500000 = 0x100a - B2000000 = 0x100b - B230400 = 0x1003 - B2500000 = 0x100c - B3000000 = 0x100d - B3500000 = 0x100e - B4000000 = 0x100f - B460800 = 0x1004 - B500000 = 0x1005 - B57600 = 0x1001 - B576000 = 0x1006 - B921600 = 0x1007 - BLKALIGNOFF = 0x127a - BLKBSZGET = 0x80081270 - BLKBSZSET = 0x40081271 - BLKDISCARD = 0x1277 - BLKDISCARDZEROES = 0x127c - BLKFLSBUF = 0x1261 - BLKFRAGET = 0x1265 - BLKFRASET = 0x1264 - BLKGETDISKSEQ = 0x80081280 - BLKGETSIZE = 0x1260 - BLKGETSIZE64 = 0x80081272 - BLKIOMIN = 0x1278 - BLKIOOPT = 0x1279 - BLKPBSZGET = 0x127b - BLKRAGET = 0x1263 - BLKRASET = 0x1262 - BLKROGET = 0x125e - BLKROSET = 0x125d - BLKROTATIONAL = 0x127e - BLKRRPART = 0x125f - BLKSECDISCARD = 0x127d - BLKSECTGET = 0x1267 - BLKSECTSET = 0x1266 - BLKSSZGET = 0x1268 - BLKZEROOUT = 0x127f - BOTHER = 0x1000 - BS1 = 0x2000 - BSDLY = 0x2000 - CBAUD = 0x100f - CBAUDEX = 0x1000 - CIBAUD = 0x100f0000 - CLOCAL = 0x800 - CR1 = 0x200 - CR2 = 0x400 - CR3 = 0x600 - CRDLY = 0x600 - CREAD = 0x80 - CS6 = 0x10 - CS7 = 0x20 - CS8 = 0x30 - CSIZE = 0x30 - CSTOPB = 0x40 - DM_MPATH_PROBE_PATHS = 0xfd12 - ECCGETLAYOUT = 0x81484d11 - ECCGETSTATS = 0x80104d12 - ECHOCTL = 0x200 - ECHOE = 0x10 - ECHOK = 0x20 - ECHOKE = 0x800 - ECHONL = 0x40 - ECHOPRT = 0x400 - EFD_CLOEXEC = 0x80000 - EFD_NONBLOCK = 0x800 - EPIOCGPARAMS = 0x80088a02 - EPIOCSPARAMS = 0x40088a01 - EPOLL_CLOEXEC = 0x80000 - EXTPROC = 0x10000 - FF1 = 0x8000 - FFDLY = 0x8000 - FICLONE = 0x40049409 - FICLONERANGE = 0x4020940d - FLUSHO = 0x1000 - FS_IOC_ENABLE_VERITY = 0x40806685 - FS_IOC_GETFLAGS = 0x80086601 - FS_IOC_GET_ENCRYPTION_NONCE = 0x8010661b - FS_IOC_GET_ENCRYPTION_POLICY = 0x400c6615 - FS_IOC_GET_ENCRYPTION_PWSALT = 0x40106614 - FS_IOC_SETFLAGS = 0x40086602 - FS_IOC_SET_ENCRYPTION_POLICY = 0x800c6613 - F_GETLK = 0x5 - F_GETLK64 = 0x5 - F_GETOWN = 0x9 - F_RDLCK = 0x0 - F_SETLK = 0x6 - F_SETLK64 = 0x6 - F_SETLKW = 0x7 - F_SETLKW64 = 0x7 - F_SETOWN = 0x8 - F_UNLCK = 0x2 - F_WRLCK = 0x1 - HIDIOCGRAWINFO = 0x80084803 - HIDIOCGRDESC = 0x90044802 - HIDIOCGRDESCSIZE = 0x80044801 - HIDIOCREVOKE = 0x4004480d - HUPCL = 0x400 - ICANON = 0x2 - IEXTEN = 0x8000 - IN_CLOEXEC = 0x80000 - IN_NONBLOCK = 0x800 - IOCTL_MEI_NOTIFY_GET = 0x80044803 - IOCTL_MEI_NOTIFY_SET = 0x40044802 - IOCTL_VM_SOCKETS_GET_LOCAL_CID = 0x7b9 - IPV6_FLOWINFO_MASK = 0xffffff0f - IPV6_FLOWLABEL_MASK = 0xffff0f00 - ISIG = 0x1 - IUCLC = 0x200 - IXOFF = 0x1000 - IXON = 0x400 - MAP_ANON = 0x20 - MAP_ANONYMOUS = 0x20 - MAP_DENYWRITE = 0x800 - MAP_EXECUTABLE = 0x1000 - MAP_GROWSDOWN = 0x100 - MAP_HUGETLB = 0x40000 - MAP_LOCKED = 0x2000 - MAP_NONBLOCK = 0x10000 - MAP_NORESERVE = 0x4000 - MAP_POPULATE = 0x8000 - MAP_STACK = 0x20000 - MAP_SYNC = 0x80000 - MCL_CURRENT = 0x1 - MCL_FUTURE = 0x2 - MCL_ONFAULT = 0x4 - MEMERASE = 0x40084d02 - MEMERASE64 = 0x40104d14 - MEMGETBADBLOCK = 0x40084d0b - MEMGETINFO = 0x80204d01 - MEMGETOOBSEL = 0x80c84d0a - MEMGETREGIONCOUNT = 0x80044d07 - MEMISLOCKED = 0x80084d17 - MEMLOCK = 0x40084d05 - MEMREAD = 0xc0404d1a - MEMREADOOB = 0xc0104d04 - MEMSETBADBLOCK = 0x40084d0c - MEMUNLOCK = 0x40084d06 - MEMWRITEOOB = 0xc0104d03 - MTDFILEMODE = 0x4d13 - NFDBITS = 0x40 - NLDLY = 0x100 - NOFLSH = 0x80 - NS_GET_MNTNS_ID = 0x8008b705 - NS_GET_NSTYPE = 0xb703 - NS_GET_OWNER_UID = 0xb704 - NS_GET_PARENT = 0xb702 - NS_GET_PID_FROM_PIDNS = 0x8004b706 - NS_GET_PID_IN_PIDNS = 0x8004b708 - NS_GET_TGID_FROM_PIDNS = 0x8004b707 - NS_GET_TGID_IN_PIDNS = 0x8004b709 - NS_GET_USERNS = 0xb701 - OLCUC = 0x2 - ONLCR = 0x4 - OTPERASE = 0x400c4d19 - OTPGETREGIONCOUNT = 0x40044d0e - OTPGETREGIONINFO = 0x400c4d0f - OTPLOCK = 0x800c4d10 - OTPSELECT = 0x80044d0d - O_APPEND = 0x400 - O_ASYNC = 0x2000 - O_CLOEXEC = 0x80000 - O_CREAT = 0x40 - O_DIRECT = 0x4000 - O_DIRECTORY = 0x10000 - O_DSYNC = 0x1000 - O_EXCL = 0x80 - O_FSYNC = 0x101000 - O_LARGEFILE = 0x0 - O_NDELAY = 0x800 - O_NOATIME = 0x40000 - O_NOCTTY = 0x100 - O_NOFOLLOW = 0x20000 - O_NONBLOCK = 0x800 - O_PATH = 0x200000 - O_RSYNC = 0x101000 - O_SYNC = 0x101000 - O_TMPFILE = 0x410000 - O_TRUNC = 0x200 - PARENB = 0x100 - PARODD = 0x200 - PENDIN = 0x4000 - PERF_EVENT_IOC_DISABLE = 0x2401 - PERF_EVENT_IOC_ENABLE = 0x2400 - PERF_EVENT_IOC_ID = 0x80082407 - PERF_EVENT_IOC_MODIFY_ATTRIBUTES = 0x4008240b - PERF_EVENT_IOC_PAUSE_OUTPUT = 0x40042409 - PERF_EVENT_IOC_PERIOD = 0x40082404 - PERF_EVENT_IOC_QUERY_BPF = 0xc008240a - PERF_EVENT_IOC_REFRESH = 0x2402 - PERF_EVENT_IOC_RESET = 0x2403 - PERF_EVENT_IOC_SET_BPF = 0x40042408 - PERF_EVENT_IOC_SET_FILTER = 0x40082406 - PERF_EVENT_IOC_SET_OUTPUT = 0x2405 - PPPIOCATTACH = 0x4004743d - PPPIOCATTCHAN = 0x40047438 - PPPIOCBRIDGECHAN = 0x40047435 - PPPIOCCONNECT = 0x4004743a - PPPIOCDETACH = 0x4004743c - PPPIOCDISCONN = 0x7439 - PPPIOCGASYNCMAP = 0x80047458 - PPPIOCGCHAN = 0x80047437 - PPPIOCGDEBUG = 0x80047441 - PPPIOCGFLAGS = 0x8004745a - PPPIOCGIDLE = 0x8010743f - PPPIOCGIDLE32 = 0x8008743f - PPPIOCGIDLE64 = 0x8010743f - PPPIOCGL2TPSTATS = 0x80487436 - PPPIOCGMRU = 0x80047453 - PPPIOCGRASYNCMAP = 0x80047455 - PPPIOCGUNIT = 0x80047456 - PPPIOCGXASYNCMAP = 0x80207450 - PPPIOCSACTIVE = 0x40107446 - PPPIOCSASYNCMAP = 0x40047457 - PPPIOCSCOMPRESS = 0x4010744d - PPPIOCSDEBUG = 0x40047440 - PPPIOCSFLAGS = 0x40047459 - PPPIOCSMAXCID = 0x40047451 - PPPIOCSMRRU = 0x4004743b - PPPIOCSMRU = 0x40047452 - PPPIOCSNPMODE = 0x4008744b - PPPIOCSPASS = 0x40107447 - PPPIOCSRASYNCMAP = 0x40047454 - PPPIOCSXASYNCMAP = 0x4020744f - PPPIOCUNBRIDGECHAN = 0x7434 - PPPIOCXFERUNIT = 0x744e - PR_SET_PTRACER_ANY = 0xffffffffffffffff - PTP_CLOCK_GETCAPS = 0x80503d01 - PTP_CLOCK_GETCAPS2 = 0x80503d0a - PTP_ENABLE_PPS = 0x40043d04 - PTP_ENABLE_PPS2 = 0x40043d0d - PTP_EXTTS_REQUEST = 0x40103d02 - PTP_EXTTS_REQUEST2 = 0x40103d0b - PTP_MASK_CLEAR_ALL = 0x3d13 - PTP_MASK_EN_SINGLE = 0x40043d14 - PTP_PEROUT_REQUEST = 0x40383d03 - PTP_PEROUT_REQUEST2 = 0x40383d0c - PTP_PIN_SETFUNC = 0x40603d07 - PTP_PIN_SETFUNC2 = 0x40603d10 - PTP_SYS_OFFSET = 0x43403d05 - PTP_SYS_OFFSET2 = 0x43403d0e - PTRACE_GETFDPIC = 0x21 - PTRACE_GETFDPIC_EXEC = 0x0 - PTRACE_GETFDPIC_INTERP = 0x1 - RLIMIT_AS = 0x9 - RLIMIT_MEMLOCK = 0x8 - RLIMIT_NOFILE = 0x7 - RLIMIT_NPROC = 0x6 - RLIMIT_RSS = 0x5 - RNDADDENTROPY = 0x40085203 - RNDADDTOENTCNT = 0x40045201 - RNDCLEARPOOL = 0x5206 - RNDGETENTCNT = 0x80045200 - RNDGETPOOL = 0x80085202 - RNDRESEEDCRNG = 0x5207 - RNDZAPENTCNT = 0x5204 - RTC_AIE_OFF = 0x7002 - RTC_AIE_ON = 0x7001 - RTC_ALM_READ = 0x80247008 - RTC_ALM_SET = 0x40247007 - RTC_EPOCH_READ = 0x8008700d - RTC_EPOCH_SET = 0x4008700e - RTC_IRQP_READ = 0x8008700b - RTC_IRQP_SET = 0x4008700c - RTC_PARAM_GET = 0x40187013 - RTC_PARAM_SET = 0x40187014 - RTC_PIE_OFF = 0x7006 - RTC_PIE_ON = 0x7005 - RTC_PLL_GET = 0x80207011 - RTC_PLL_SET = 0x40207012 - RTC_RD_TIME = 0x80247009 - RTC_SET_TIME = 0x4024700a - RTC_UIE_OFF = 0x7004 - RTC_UIE_ON = 0x7003 - RTC_VL_CLR = 0x7014 - RTC_VL_READ = 0x80047013 - RTC_WIE_OFF = 0x7010 - RTC_WIE_ON = 0x700f - RTC_WKALM_RD = 0x80287010 - RTC_WKALM_SET = 0x4028700f - SCM_DEVMEM_DMABUF = 0x4f - SCM_DEVMEM_LINEAR = 0x4e - SCM_TIMESTAMPING = 0x25 - SCM_TIMESTAMPING_OPT_STATS = 0x36 - SCM_TIMESTAMPING_PKTINFO = 0x3a - SCM_TIMESTAMPNS = 0x23 - SCM_TS_OPT_ID = 0x51 - SCM_TXTIME = 0x3d - SCM_WIFI_STATUS = 0x29 - SECCOMP_IOCTL_NOTIF_ADDFD = 0x40182103 - SECCOMP_IOCTL_NOTIF_ID_VALID = 0x40082102 - SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x40082104 - SFD_CLOEXEC = 0x80000 - SFD_NONBLOCK = 0x800 - SIOCATMARK = 0x8905 - SIOCGPGRP = 0x8904 - SIOCGSTAMPNS_NEW = 0x80108907 - SIOCGSTAMP_NEW = 0x80108906 - SIOCINQ = 0x541b - SIOCOUTQ = 0x5411 - SIOCSPGRP = 0x8902 - SOCK_CLOEXEC = 0x80000 - SOCK_DGRAM = 0x2 - SOCK_NONBLOCK = 0x800 - SOCK_STREAM = 0x1 - SOL_SOCKET = 0x1 - SO_ACCEPTCONN = 0x1e - SO_ATTACH_BPF = 0x32 - SO_ATTACH_REUSEPORT_CBPF = 0x33 - SO_ATTACH_REUSEPORT_EBPF = 0x34 - SO_BINDTODEVICE = 0x19 - SO_BINDTOIFINDEX = 0x3e - SO_BPF_EXTENSIONS = 0x30 - SO_BROADCAST = 0x6 - SO_BSDCOMPAT = 0xe - SO_BUF_LOCK = 0x48 - SO_BUSY_POLL = 0x2e - SO_BUSY_POLL_BUDGET = 0x46 - SO_CNX_ADVICE = 0x35 - SO_COOKIE = 0x39 - SO_DETACH_REUSEPORT_BPF = 0x44 - SO_DEVMEM_DMABUF = 0x4f - SO_DEVMEM_DONTNEED = 0x50 - SO_DEVMEM_LINEAR = 0x4e - SO_DOMAIN = 0x27 - SO_DONTROUTE = 0x5 - SO_ERROR = 0x4 - SO_INCOMING_CPU = 0x31 - SO_INCOMING_NAPI_ID = 0x38 - SO_KEEPALIVE = 0x9 - SO_LINGER = 0xd - SO_LOCK_FILTER = 0x2c - SO_MARK = 0x24 - SO_MAX_PACING_RATE = 0x2f - SO_MEMINFO = 0x37 - SO_NETNS_COOKIE = 0x47 - SO_NOFCS = 0x2b - SO_OOBINLINE = 0xa - SO_PASSCRED = 0x10 - SO_PASSPIDFD = 0x4c - SO_PASSRIGHTS = 0x53 - SO_PASSSEC = 0x22 - SO_PEEK_OFF = 0x2a - SO_PEERCRED = 0x11 - SO_PEERGROUPS = 0x3b - SO_PEERPIDFD = 0x4d - SO_PEERSEC = 0x1f - SO_PREFER_BUSY_POLL = 0x45 - SO_PROTOCOL = 0x26 - SO_RCVBUF = 0x8 - SO_RCVBUFFORCE = 0x21 - SO_RCVLOWAT = 0x12 - SO_RCVMARK = 0x4b - SO_RCVPRIORITY = 0x52 - SO_RCVTIMEO = 0x14 - SO_RCVTIMEO_NEW = 0x42 - SO_RCVTIMEO_OLD = 0x14 - SO_RESERVE_MEM = 0x49 - SO_REUSEADDR = 0x2 - SO_REUSEPORT = 0xf - SO_RXQ_OVFL = 0x28 - SO_SECURITY_AUTHENTICATION = 0x16 - SO_SECURITY_ENCRYPTION_NETWORK = 0x18 - SO_SECURITY_ENCRYPTION_TRANSPORT = 0x17 - SO_SELECT_ERR_QUEUE = 0x2d - SO_SNDBUF = 0x7 - SO_SNDBUFFORCE = 0x20 - SO_SNDLOWAT = 0x13 - SO_SNDTIMEO = 0x15 - SO_SNDTIMEO_NEW = 0x43 - SO_SNDTIMEO_OLD = 0x15 - SO_TIMESTAMPING = 0x25 - SO_TIMESTAMPING_NEW = 0x41 - SO_TIMESTAMPING_OLD = 0x25 - SO_TIMESTAMPNS = 0x23 - SO_TIMESTAMPNS_NEW = 0x40 - SO_TIMESTAMPNS_OLD = 0x23 - SO_TIMESTAMP_NEW = 0x3f - SO_TXREHASH = 0x4a - SO_TXTIME = 0x3d - SO_TYPE = 0x3 - SO_WIFI_STATUS = 0x29 - SO_ZEROCOPY = 0x3c - TAB1 = 0x800 - TAB2 = 0x1000 - TAB3 = 0x1800 - TABDLY = 0x1800 - TCFLSH = 0x540b - TCGETA = 0x5405 - TCGETS = 0x5401 - TCGETS2 = 0x802c542a - TCGETX = 0x5432 - TCSAFLUSH = 0x2 - TCSBRK = 0x5409 - TCSBRKP = 0x5425 - TCSETA = 0x5406 - TCSETAF = 0x5408 - TCSETAW = 0x5407 - TCSETS = 0x5402 - TCSETS2 = 0x402c542b - TCSETSF = 0x5404 - TCSETSF2 = 0x402c542d - TCSETSW = 0x5403 - TCSETSW2 = 0x402c542c - TCSETX = 0x5433 - TCSETXF = 0x5434 - TCSETXW = 0x5435 - TCXONC = 0x540a - TFD_CLOEXEC = 0x80000 - TFD_NONBLOCK = 0x800 - TIOCCBRK = 0x5428 - TIOCCONS = 0x541d - TIOCEXCL = 0x540c - TIOCGDEV = 0x80045432 - TIOCGETD = 0x5424 - TIOCGEXCL = 0x80045440 - TIOCGICOUNT = 0x545d - TIOCGISO7816 = 0x80285442 - TIOCGLCKTRMIOS = 0x5456 - TIOCGPGRP = 0x540f - TIOCGPKT = 0x80045438 - TIOCGPTLCK = 0x80045439 - TIOCGPTN = 0x80045430 - TIOCGPTPEER = 0x5441 - TIOCGRS485 = 0x542e - TIOCGSERIAL = 0x541e - TIOCGSID = 0x5429 - TIOCGSOFTCAR = 0x5419 - TIOCGWINSZ = 0x5413 - TIOCINQ = 0x541b - TIOCLINUX = 0x541c - TIOCMBIC = 0x5417 - TIOCMBIS = 0x5416 - TIOCMGET = 0x5415 - TIOCMIWAIT = 0x545c - TIOCMSET = 0x5418 - TIOCM_CAR = 0x40 - TIOCM_CD = 0x40 - TIOCM_CTS = 0x20 - TIOCM_DSR = 0x100 - TIOCM_RI = 0x80 - TIOCM_RNG = 0x80 - TIOCM_SR = 0x10 - TIOCM_ST = 0x8 - TIOCNOTTY = 0x5422 - TIOCNXCL = 0x540d - TIOCOUTQ = 0x5411 - TIOCPKT = 0x5420 - TIOCSBRK = 0x5427 - TIOCSCTTY = 0x540e - TIOCSERCONFIG = 0x5453 - TIOCSERGETLSR = 0x5459 - TIOCSERGETMULTI = 0x545a - TIOCSERGSTRUCT = 0x5458 - TIOCSERGWILD = 0x5454 - TIOCSERSETMULTI = 0x545b - TIOCSERSWILD = 0x5455 - TIOCSER_TEMT = 0x1 - TIOCSETD = 0x5423 - TIOCSIG = 0x40045436 - TIOCSISO7816 = 0xc0285443 - TIOCSLCKTRMIOS = 0x5457 - TIOCSPGRP = 0x5410 - TIOCSPTLCK = 0x40045431 - TIOCSRS485 = 0x542f - TIOCSSERIAL = 0x541f - TIOCSSOFTCAR = 0x541a - TIOCSTI = 0x5412 - TIOCSWINSZ = 0x5414 - TIOCVHANGUP = 0x5437 - TOSTOP = 0x100 - TUNATTACHFILTER = 0x401054d5 - TUNDETACHFILTER = 0x401054d6 - TUNGETDEVNETNS = 0x54e3 - TUNGETFEATURES = 0x800454cf - TUNGETFILTER = 0x801054db - TUNGETIFF = 0x800454d2 - TUNGETSNDBUF = 0x800454d3 - TUNGETVNETBE = 0x800454df - TUNGETVNETHDRSZ = 0x800454d7 - TUNGETVNETLE = 0x800454dd - TUNSETCARRIER = 0x400454e2 - TUNSETDEBUG = 0x400454c9 - TUNSETFILTEREBPF = 0x800454e1 - TUNSETGROUP = 0x400454ce - TUNSETIFF = 0x400454ca - TUNSETIFINDEX = 0x400454da - TUNSETLINK = 0x400454cd - TUNSETNOCSUM = 0x400454c8 - TUNSETOFFLOAD = 0x400454d0 - TUNSETOWNER = 0x400454cc - TUNSETPERSIST = 0x400454cb - TUNSETQUEUE = 0x400454d9 - TUNSETSNDBUF = 0x400454d4 - TUNSETSTEERINGEBPF = 0x800454e0 - TUNSETTXFILTER = 0x400454d1 - TUNSETVNETBE = 0x400454de - TUNSETVNETHDRSZ = 0x400454d8 - TUNSETVNETLE = 0x400454dc - UBI_IOCATT = 0x40186f40 - UBI_IOCDET = 0x40046f41 - UBI_IOCEBCH = 0x40044f02 - UBI_IOCEBER = 0x40044f01 - UBI_IOCEBISMAP = 0x80044f05 - UBI_IOCEBMAP = 0x40084f03 - UBI_IOCEBUNMAP = 0x40044f04 - UBI_IOCMKVOL = 0x40986f00 - UBI_IOCRMVOL = 0x40046f01 - UBI_IOCRNVOL = 0x51106f03 - UBI_IOCRPEB = 0x40046f04 - UBI_IOCRSVOL = 0x400c6f02 - UBI_IOCSETVOLPROP = 0x40104f06 - UBI_IOCSPEB = 0x40046f05 - UBI_IOCVOLCRBLK = 0x40804f07 - UBI_IOCVOLRMBLK = 0x4f08 - UBI_IOCVOLUP = 0x40084f00 - VDISCARD = 0xd - VEOF = 0x4 - VEOL = 0xb - VEOL2 = 0x10 - VMIN = 0x6 - VREPRINT = 0xc - VSTART = 0x8 - VSTOP = 0x9 - VSUSP = 0xa - VSWTC = 0x7 - VT1 = 0x4000 - VTDLY = 0x4000 - VTIME = 0x5 - VWERASE = 0xe - WDIOC_GETBOOTSTATUS = 0x80045702 - WDIOC_GETPRETIMEOUT = 0x80045709 - WDIOC_GETSTATUS = 0x80045701 - WDIOC_GETSUPPORT = 0x80285700 - WDIOC_GETTEMP = 0x80045703 - WDIOC_GETTIMELEFT = 0x8004570a - WDIOC_GETTIMEOUT = 0x80045707 - WDIOC_KEEPALIVE = 0x80045705 - WDIOC_SETOPTIONS = 0x80045704 - WORDSIZE = 0x40 - XCASE = 0x4 - XTABS = 0x1800 - _HIDIOCGRAWNAME = 0x80804804 - _HIDIOCGRAWPHYS = 0x80404805 - _HIDIOCGRAWUNIQ = 0x80404808 + B1000000 = 0x1008 + B115200 = 0x1002 + B1152000 = 0x1009 + B1500000 = 0x100a + B2000000 = 0x100b + B230400 = 0x1003 + B2500000 = 0x100c + B3000000 = 0x100d + B3500000 = 0x100e + B4000000 = 0x100f + B460800 = 0x1004 + B500000 = 0x1005 + B57600 = 0x1001 + B576000 = 0x1006 + B921600 = 0x1007 + BLKALIGNOFF = 0x127a + BLKBSZGET = 0x80081270 + BLKBSZSET = 0x40081271 + BLKDISCARD = 0x1277 + BLKDISCARDZEROES = 0x127c + BLKFLSBUF = 0x1261 + BLKFRAGET = 0x1265 + BLKFRASET = 0x1264 + BLKGETDISKSEQ = 0x80081280 + BLKGETSIZE = 0x1260 + BLKGETSIZE64 = 0x80081272 + BLKIOMIN = 0x1278 + BLKIOOPT = 0x1279 + BLKPBSZGET = 0x127b + BLKRAGET = 0x1263 + BLKRASET = 0x1262 + BLKROGET = 0x125e + BLKROSET = 0x125d + BLKROTATIONAL = 0x127e + BLKRRPART = 0x125f + BLKSECDISCARD = 0x127d + BLKSECTGET = 0x1267 + BLKSECTSET = 0x1266 + BLKSSZGET = 0x1268 + BLKZEROOUT = 0x127f + BOTHER = 0x1000 + BS1 = 0x2000 + BSDLY = 0x2000 + CBAUD = 0x100f + CBAUDEX = 0x1000 + CIBAUD = 0x100f0000 + CLOCAL = 0x800 + CR1 = 0x200 + CR2 = 0x400 + CR3 = 0x600 + CRDLY = 0x600 + CREAD = 0x80 + CS6 = 0x10 + CS7 = 0x20 + CS8 = 0x30 + CSIZE = 0x30 + CSTOPB = 0x40 + DM_MPATH_PROBE_PATHS = 0xfd12 + ECCGETLAYOUT = 0x81484d11 + ECCGETSTATS = 0x80104d12 + ECHOCTL = 0x200 + ECHOE = 0x10 + ECHOK = 0x20 + ECHOKE = 0x800 + ECHONL = 0x40 + ECHOPRT = 0x400 + EFD_CLOEXEC = 0x80000 + EFD_NONBLOCK = 0x800 + EPIOCGPARAMS = 0x80088a02 + EPIOCSPARAMS = 0x40088a01 + EPOLL_CLOEXEC = 0x80000 + EXTPROC = 0x10000 + FF1 = 0x8000 + FFDLY = 0x8000 + FICLONE = 0x40049409 + FICLONERANGE = 0x4020940d + FLUSHO = 0x1000 + FS_IOC_ENABLE_VERITY = 0x40806685 + FS_IOC_GETFLAGS = 0x80086601 + FS_IOC_GET_ENCRYPTION_NONCE = 0x8010661b + FS_IOC_GET_ENCRYPTION_POLICY = 0x400c6615 + FS_IOC_GET_ENCRYPTION_PWSALT = 0x40106614 + FS_IOC_SETFLAGS = 0x40086602 + FS_IOC_SET_ENCRYPTION_POLICY = 0x800c6613 + F_GETLK = 0x5 + F_GETLK64 = 0x5 + F_GETOWN = 0x9 + F_RDLCK = 0x0 + F_SETLK = 0x6 + F_SETLK64 = 0x6 + F_SETLKW = 0x7 + F_SETLKW64 = 0x7 + F_SETOWN = 0x8 + F_UNLCK = 0x2 + F_WRLCK = 0x1 + HIDIOCGRAWINFO = 0x80084803 + HIDIOCGRDESC = 0x90044802 + HIDIOCGRDESCSIZE = 0x80044801 + HIDIOCREVOKE = 0x4004480d + HUPCL = 0x400 + ICANON = 0x2 + IEXTEN = 0x8000 + IN_CLOEXEC = 0x80000 + IN_NONBLOCK = 0x800 + IOCTL_MEI_NOTIFY_GET = 0x80044803 + IOCTL_MEI_NOTIFY_SET = 0x40044802 + IOCTL_VM_SOCKETS_GET_LOCAL_CID = 0x7b9 + IPV6_FLOWINFO_MASK = 0xffffff0f + IPV6_FLOWLABEL_MASK = 0xffff0f00 + ISIG = 0x1 + IUCLC = 0x200 + IXOFF = 0x1000 + IXON = 0x400 + MAP_ANON = 0x20 + MAP_ANONYMOUS = 0x20 + MAP_DENYWRITE = 0x800 + MAP_EXECUTABLE = 0x1000 + MAP_GROWSDOWN = 0x100 + MAP_HUGETLB = 0x40000 + MAP_LOCKED = 0x2000 + MAP_NONBLOCK = 0x10000 + MAP_NORESERVE = 0x4000 + MAP_POPULATE = 0x8000 + MAP_STACK = 0x20000 + MAP_SYNC = 0x80000 + MCL_CURRENT = 0x1 + MCL_FUTURE = 0x2 + MCL_ONFAULT = 0x4 + MEMERASE = 0x40084d02 + MEMERASE64 = 0x40104d14 + MEMGETBADBLOCK = 0x40084d0b + MEMGETINFO = 0x80204d01 + MEMGETOOBSEL = 0x80c84d0a + MEMGETREGIONCOUNT = 0x80044d07 + MEMISLOCKED = 0x80084d17 + MEMLOCK = 0x40084d05 + MEMREAD = 0xc0404d1a + MEMREADOOB = 0xc0104d04 + MEMSETBADBLOCK = 0x40084d0c + MEMUNLOCK = 0x40084d06 + MEMWRITEOOB = 0xc0104d03 + MTDFILEMODE = 0x4d13 + NFDBITS = 0x40 + NLDLY = 0x100 + NOFLSH = 0x80 + NS_GET_ID = 0x8008b70d + NS_GET_MNTNS_ID = 0x8008b705 + NS_GET_NSTYPE = 0xb703 + NS_GET_OWNER_UID = 0xb704 + NS_GET_PARENT = 0xb702 + NS_GET_PID_FROM_PIDNS = 0x8004b706 + NS_GET_PID_IN_PIDNS = 0x8004b708 + NS_GET_TGID_FROM_PIDNS = 0x8004b707 + NS_GET_TGID_IN_PIDNS = 0x8004b709 + NS_GET_USERNS = 0xb701 + OLCUC = 0x2 + ONLCR = 0x4 + OTPERASE = 0x400c4d19 + OTPGETREGIONCOUNT = 0x40044d0e + OTPGETREGIONINFO = 0x400c4d0f + OTPLOCK = 0x800c4d10 + OTPSELECT = 0x80044d0d + O_APPEND = 0x400 + O_ASYNC = 0x2000 + O_CLOEXEC = 0x80000 + O_CREAT = 0x40 + O_DIRECT = 0x4000 + O_DIRECTORY = 0x10000 + O_DSYNC = 0x1000 + O_EXCL = 0x80 + O_FSYNC = 0x101000 + O_LARGEFILE = 0x0 + O_NDELAY = 0x800 + O_NOATIME = 0x40000 + O_NOCTTY = 0x100 + O_NOFOLLOW = 0x20000 + O_NONBLOCK = 0x800 + O_PATH = 0x200000 + O_RSYNC = 0x101000 + O_SYNC = 0x101000 + O_TMPFILE = 0x410000 + O_TRUNC = 0x200 + PARENB = 0x100 + PARODD = 0x200 + PENDIN = 0x4000 + PERF_EVENT_IOC_DISABLE = 0x2401 + PERF_EVENT_IOC_ENABLE = 0x2400 + PERF_EVENT_IOC_ID = 0x80082407 + PERF_EVENT_IOC_MODIFY_ATTRIBUTES = 0x4008240b + PERF_EVENT_IOC_PAUSE_OUTPUT = 0x40042409 + PERF_EVENT_IOC_PERIOD = 0x40082404 + PERF_EVENT_IOC_QUERY_BPF = 0xc008240a + PERF_EVENT_IOC_REFRESH = 0x2402 + PERF_EVENT_IOC_RESET = 0x2403 + PERF_EVENT_IOC_SET_BPF = 0x40042408 + PERF_EVENT_IOC_SET_FILTER = 0x40082406 + PERF_EVENT_IOC_SET_OUTPUT = 0x2405 + PPPIOCATTACH = 0x4004743d + PPPIOCATTCHAN = 0x40047438 + PPPIOCBRIDGECHAN = 0x40047435 + PPPIOCCONNECT = 0x4004743a + PPPIOCDETACH = 0x4004743c + PPPIOCDISCONN = 0x7439 + PPPIOCGASYNCMAP = 0x80047458 + PPPIOCGCHAN = 0x80047437 + PPPIOCGDEBUG = 0x80047441 + PPPIOCGFLAGS = 0x8004745a + PPPIOCGIDLE = 0x8010743f + PPPIOCGIDLE32 = 0x8008743f + PPPIOCGIDLE64 = 0x8010743f + PPPIOCGL2TPSTATS = 0x80487436 + PPPIOCGMRU = 0x80047453 + PPPIOCGRASYNCMAP = 0x80047455 + PPPIOCGUNIT = 0x80047456 + PPPIOCGXASYNCMAP = 0x80207450 + PPPIOCSACTIVE = 0x40107446 + PPPIOCSASYNCMAP = 0x40047457 + PPPIOCSCOMPRESS = 0x4010744d + PPPIOCSDEBUG = 0x40047440 + PPPIOCSFLAGS = 0x40047459 + PPPIOCSMAXCID = 0x40047451 + PPPIOCSMRRU = 0x4004743b + PPPIOCSMRU = 0x40047452 + PPPIOCSNPMODE = 0x4008744b + PPPIOCSPASS = 0x40107447 + PPPIOCSRASYNCMAP = 0x40047454 + PPPIOCSXASYNCMAP = 0x4020744f + PPPIOCUNBRIDGECHAN = 0x7434 + PPPIOCXFERUNIT = 0x744e + PR_SET_PTRACER_ANY = 0xffffffffffffffff + PTP_CLOCK_GETCAPS = 0x80503d01 + PTP_CLOCK_GETCAPS2 = 0x80503d0a + PTP_ENABLE_PPS = 0x40043d04 + PTP_ENABLE_PPS2 = 0x40043d0d + PTP_EXTTS_REQUEST = 0x40103d02 + PTP_EXTTS_REQUEST2 = 0x40103d0b + PTP_MASK_CLEAR_ALL = 0x3d13 + PTP_MASK_EN_SINGLE = 0x40043d14 + PTP_PEROUT_REQUEST = 0x40383d03 + PTP_PEROUT_REQUEST2 = 0x40383d0c + PTP_PIN_SETFUNC = 0x40603d07 + PTP_PIN_SETFUNC2 = 0x40603d10 + PTP_SYS_OFFSET = 0x43403d05 + PTP_SYS_OFFSET2 = 0x43403d0e + PTRACE_CFI_BRANCH_EXPECTED_LANDING_PAD_BIT = 0x2 + PTRACE_CFI_BRANCH_EXPECTED_LANDING_PAD_STATE = 0x4 + PTRACE_CFI_BRANCH_LANDING_PAD_EN_BIT = 0x0 + PTRACE_CFI_BRANCH_LANDING_PAD_EN_STATE = 0x1 + PTRACE_CFI_BRANCH_LANDING_PAD_LOCK_BIT = 0x1 + PTRACE_CFI_BRANCH_LANDING_PAD_LOCK_STATE = 0x2 + PTRACE_CFI_SHADOW_STACK_EN_BIT = 0x3 + PTRACE_CFI_SHADOW_STACK_EN_STATE = 0x8 + PTRACE_CFI_SHADOW_STACK_LOCK_BIT = 0x4 + PTRACE_CFI_SHADOW_STACK_LOCK_STATE = 0x10 + PTRACE_CFI_SHADOW_STACK_PTR_BIT = 0x5 + PTRACE_CFI_SHADOW_STACK_PTR_STATE = 0x20 + PTRACE_CFI_STATE_INVALID_MASK = 0xffffffffffffffc0 + PTRACE_GETFDPIC = 0x21 + PTRACE_GETFDPIC_EXEC = 0x0 + PTRACE_GETFDPIC_INTERP = 0x1 + RLIMIT_AS = 0x9 + RLIMIT_MEMLOCK = 0x8 + RLIMIT_NOFILE = 0x7 + RLIMIT_NPROC = 0x6 + RLIMIT_RSS = 0x5 + RNDADDENTROPY = 0x40085203 + RNDADDTOENTCNT = 0x40045201 + RNDCLEARPOOL = 0x5206 + RNDGETENTCNT = 0x80045200 + RNDGETPOOL = 0x80085202 + RNDRESEEDCRNG = 0x5207 + RNDZAPENTCNT = 0x5204 + RTC_AIE_OFF = 0x7002 + RTC_AIE_ON = 0x7001 + RTC_ALM_READ = 0x80247008 + RTC_ALM_SET = 0x40247007 + RTC_EPOCH_READ = 0x8008700d + RTC_EPOCH_SET = 0x4008700e + RTC_IRQP_READ = 0x8008700b + RTC_IRQP_SET = 0x4008700c + RTC_PARAM_GET = 0x40187013 + RTC_PARAM_SET = 0x40187014 + RTC_PIE_OFF = 0x7006 + RTC_PIE_ON = 0x7005 + RTC_PLL_GET = 0x80207011 + RTC_PLL_SET = 0x40207012 + RTC_RD_TIME = 0x80247009 + RTC_SET_TIME = 0x4024700a + RTC_UIE_OFF = 0x7004 + RTC_UIE_ON = 0x7003 + RTC_VL_CLR = 0x7014 + RTC_VL_READ = 0x80047013 + RTC_WIE_OFF = 0x7010 + RTC_WIE_ON = 0x700f + RTC_WKALM_RD = 0x80287010 + RTC_WKALM_SET = 0x4028700f + SCM_DEVMEM_DMABUF = 0x4f + SCM_DEVMEM_LINEAR = 0x4e + SCM_INQ = 0x54 + SCM_TIMESTAMPING = 0x25 + SCM_TIMESTAMPING_OPT_STATS = 0x36 + SCM_TIMESTAMPING_PKTINFO = 0x3a + SCM_TIMESTAMPNS = 0x23 + SCM_TS_OPT_ID = 0x51 + SCM_TXTIME = 0x3d + SCM_WIFI_STATUS = 0x29 + SECCOMP_IOCTL_NOTIF_ADDFD = 0x40182103 + SECCOMP_IOCTL_NOTIF_ID_VALID = 0x40082102 + SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x40082104 + SFD_CLOEXEC = 0x80000 + SFD_NONBLOCK = 0x800 + SIOCATMARK = 0x8905 + SIOCGPGRP = 0x8904 + SIOCGSTAMPNS_NEW = 0x80108907 + SIOCGSTAMP_NEW = 0x80108906 + SIOCINQ = 0x541b + SIOCOUTQ = 0x5411 + SIOCSPGRP = 0x8902 + SOCK_CLOEXEC = 0x80000 + SOCK_DGRAM = 0x2 + SOCK_NONBLOCK = 0x800 + SOCK_STREAM = 0x1 + SOL_SOCKET = 0x1 + SO_ACCEPTCONN = 0x1e + SO_ATTACH_BPF = 0x32 + SO_ATTACH_REUSEPORT_CBPF = 0x33 + SO_ATTACH_REUSEPORT_EBPF = 0x34 + SO_BINDTODEVICE = 0x19 + SO_BINDTOIFINDEX = 0x3e + SO_BPF_EXTENSIONS = 0x30 + SO_BROADCAST = 0x6 + SO_BSDCOMPAT = 0xe + SO_BUF_LOCK = 0x48 + SO_BUSY_POLL = 0x2e + SO_BUSY_POLL_BUDGET = 0x46 + SO_CNX_ADVICE = 0x35 + SO_COOKIE = 0x39 + SO_DETACH_REUSEPORT_BPF = 0x44 + SO_DEVMEM_DMABUF = 0x4f + SO_DEVMEM_DONTNEED = 0x50 + SO_DEVMEM_LINEAR = 0x4e + SO_DOMAIN = 0x27 + SO_DONTROUTE = 0x5 + SO_ERROR = 0x4 + SO_INCOMING_CPU = 0x31 + SO_INCOMING_NAPI_ID = 0x38 + SO_INQ = 0x54 + SO_KEEPALIVE = 0x9 + SO_LINGER = 0xd + SO_LOCK_FILTER = 0x2c + SO_MARK = 0x24 + SO_MAX_PACING_RATE = 0x2f + SO_MEMINFO = 0x37 + SO_NETNS_COOKIE = 0x47 + SO_NOFCS = 0x2b + SO_OOBINLINE = 0xa + SO_PASSCRED = 0x10 + SO_PASSPIDFD = 0x4c + SO_PASSRIGHTS = 0x53 + SO_PASSSEC = 0x22 + SO_PEEK_OFF = 0x2a + SO_PEERCRED = 0x11 + SO_PEERGROUPS = 0x3b + SO_PEERPIDFD = 0x4d + SO_PEERSEC = 0x1f + SO_PREFER_BUSY_POLL = 0x45 + SO_PROTOCOL = 0x26 + SO_RCVBUF = 0x8 + SO_RCVBUFFORCE = 0x21 + SO_RCVLOWAT = 0x12 + SO_RCVMARK = 0x4b + SO_RCVPRIORITY = 0x52 + SO_RCVTIMEO = 0x14 + SO_RCVTIMEO_NEW = 0x42 + SO_RCVTIMEO_OLD = 0x14 + SO_RESERVE_MEM = 0x49 + SO_REUSEADDR = 0x2 + SO_REUSEPORT = 0xf + SO_RXQ_OVFL = 0x28 + SO_SECURITY_AUTHENTICATION = 0x16 + SO_SECURITY_ENCRYPTION_NETWORK = 0x18 + SO_SECURITY_ENCRYPTION_TRANSPORT = 0x17 + SO_SELECT_ERR_QUEUE = 0x2d + SO_SNDBUF = 0x7 + SO_SNDBUFFORCE = 0x20 + SO_SNDLOWAT = 0x13 + SO_SNDTIMEO = 0x15 + SO_SNDTIMEO_NEW = 0x43 + SO_SNDTIMEO_OLD = 0x15 + SO_TIMESTAMPING = 0x25 + SO_TIMESTAMPING_NEW = 0x41 + SO_TIMESTAMPING_OLD = 0x25 + SO_TIMESTAMPNS = 0x23 + SO_TIMESTAMPNS_NEW = 0x40 + SO_TIMESTAMPNS_OLD = 0x23 + SO_TIMESTAMP_NEW = 0x3f + SO_TXREHASH = 0x4a + SO_TXTIME = 0x3d + SO_TYPE = 0x3 + SO_WIFI_STATUS = 0x29 + SO_ZEROCOPY = 0x3c + TAB1 = 0x800 + TAB2 = 0x1000 + TAB3 = 0x1800 + TABDLY = 0x1800 + TCFLSH = 0x540b + TCGETA = 0x5405 + TCGETS = 0x5401 + TCGETS2 = 0x802c542a + TCGETX = 0x5432 + TCSAFLUSH = 0x2 + TCSBRK = 0x5409 + TCSBRKP = 0x5425 + TCSETA = 0x5406 + TCSETAF = 0x5408 + TCSETAW = 0x5407 + TCSETS = 0x5402 + TCSETS2 = 0x402c542b + TCSETSF = 0x5404 + TCSETSF2 = 0x402c542d + TCSETSW = 0x5403 + TCSETSW2 = 0x402c542c + TCSETX = 0x5433 + TCSETXF = 0x5434 + TCSETXW = 0x5435 + TCXONC = 0x540a + TFD_CLOEXEC = 0x80000 + TFD_NONBLOCK = 0x800 + TIOCCBRK = 0x5428 + TIOCCONS = 0x541d + TIOCEXCL = 0x540c + TIOCGDEV = 0x80045432 + TIOCGETD = 0x5424 + TIOCGEXCL = 0x80045440 + TIOCGICOUNT = 0x545d + TIOCGISO7816 = 0x80285442 + TIOCGLCKTRMIOS = 0x5456 + TIOCGPGRP = 0x540f + TIOCGPKT = 0x80045438 + TIOCGPTLCK = 0x80045439 + TIOCGPTN = 0x80045430 + TIOCGPTPEER = 0x5441 + TIOCGRS485 = 0x542e + TIOCGSERIAL = 0x541e + TIOCGSID = 0x5429 + TIOCGSOFTCAR = 0x5419 + TIOCGWINSZ = 0x5413 + TIOCINQ = 0x541b + TIOCLINUX = 0x541c + TIOCMBIC = 0x5417 + TIOCMBIS = 0x5416 + TIOCMGET = 0x5415 + TIOCMIWAIT = 0x545c + TIOCMSET = 0x5418 + TIOCM_CAR = 0x40 + TIOCM_CD = 0x40 + TIOCM_CTS = 0x20 + TIOCM_DSR = 0x100 + TIOCM_RI = 0x80 + TIOCM_RNG = 0x80 + TIOCM_SR = 0x10 + TIOCM_ST = 0x8 + TIOCNOTTY = 0x5422 + TIOCNXCL = 0x540d + TIOCOUTQ = 0x5411 + TIOCPKT = 0x5420 + TIOCSBRK = 0x5427 + TIOCSCTTY = 0x540e + TIOCSERCONFIG = 0x5453 + TIOCSERGETLSR = 0x5459 + TIOCSERGETMULTI = 0x545a + TIOCSERGSTRUCT = 0x5458 + TIOCSERGWILD = 0x5454 + TIOCSERSETMULTI = 0x545b + TIOCSERSWILD = 0x5455 + TIOCSER_TEMT = 0x1 + TIOCSETD = 0x5423 + TIOCSIG = 0x40045436 + TIOCSISO7816 = 0xc0285443 + TIOCSLCKTRMIOS = 0x5457 + TIOCSPGRP = 0x5410 + TIOCSPTLCK = 0x40045431 + TIOCSRS485 = 0x542f + TIOCSSERIAL = 0x541f + TIOCSSOFTCAR = 0x541a + TIOCSTI = 0x5412 + TIOCSWINSZ = 0x5414 + TIOCVHANGUP = 0x5437 + TOSTOP = 0x100 + TUNATTACHFILTER = 0x401054d5 + TUNDETACHFILTER = 0x401054d6 + TUNGETDEVNETNS = 0x54e3 + TUNGETFEATURES = 0x800454cf + TUNGETFILTER = 0x801054db + TUNGETIFF = 0x800454d2 + TUNGETSNDBUF = 0x800454d3 + TUNGETVNETBE = 0x800454df + TUNGETVNETHDRSZ = 0x800454d7 + TUNGETVNETLE = 0x800454dd + TUNSETCARRIER = 0x400454e2 + TUNSETDEBUG = 0x400454c9 + TUNSETFILTEREBPF = 0x800454e1 + TUNSETGROUP = 0x400454ce + TUNSETIFF = 0x400454ca + TUNSETIFINDEX = 0x400454da + TUNSETLINK = 0x400454cd + TUNSETNOCSUM = 0x400454c8 + TUNSETOFFLOAD = 0x400454d0 + TUNSETOWNER = 0x400454cc + TUNSETPERSIST = 0x400454cb + TUNSETQUEUE = 0x400454d9 + TUNSETSNDBUF = 0x400454d4 + TUNSETSTEERINGEBPF = 0x800454e0 + TUNSETTXFILTER = 0x400454d1 + TUNSETVNETBE = 0x400454de + TUNSETVNETHDRSZ = 0x400454d8 + TUNSETVNETLE = 0x400454dc + UBI_IOCATT = 0x40186f40 + UBI_IOCDET = 0x40046f41 + UBI_IOCEBCH = 0x40044f02 + UBI_IOCEBER = 0x40044f01 + UBI_IOCEBISMAP = 0x80044f05 + UBI_IOCEBMAP = 0x40084f03 + UBI_IOCEBUNMAP = 0x40044f04 + UBI_IOCMKVOL = 0x40986f00 + UBI_IOCRMVOL = 0x40046f01 + UBI_IOCRNVOL = 0x51106f03 + UBI_IOCRPEB = 0x40046f04 + UBI_IOCRSVOL = 0x400c6f02 + UBI_IOCSETVOLPROP = 0x40104f06 + UBI_IOCSPEB = 0x40046f05 + UBI_IOCVOLCRBLK = 0x40804f07 + UBI_IOCVOLRMBLK = 0x4f08 + UBI_IOCVOLUP = 0x40084f00 + VDISCARD = 0xd + VEOF = 0x4 + VEOL = 0xb + VEOL2 = 0x10 + VMIN = 0x6 + VREPRINT = 0xc + VSTART = 0x8 + VSTOP = 0x9 + VSUSP = 0xa + VSWTC = 0x7 + VT1 = 0x4000 + VTDLY = 0x4000 + VTIME = 0x5 + VWERASE = 0xe + WDIOC_GETBOOTSTATUS = 0x80045702 + WDIOC_GETPRETIMEOUT = 0x80045709 + WDIOC_GETSTATUS = 0x80045701 + WDIOC_GETSUPPORT = 0x80285700 + WDIOC_GETTEMP = 0x80045703 + WDIOC_GETTIMELEFT = 0x8004570a + WDIOC_GETTIMEOUT = 0x80045707 + WDIOC_KEEPALIVE = 0x80045705 + WDIOC_SETOPTIONS = 0x80045704 + WORDSIZE = 0x40 + XCASE = 0x4 + XTABS = 0x1800 + _HIDIOCGRAWNAME = 0x80804804 + _HIDIOCGRAWPHYS = 0x80404805 + _HIDIOCGRAWUNIQ = 0x80404808 ) // Errors @@ -585,6 +601,8 @@ const ( EDESTADDRREQ = syscall.Errno(0x59) EDOTDOT = syscall.Errno(0x49) EDQUOT = syscall.Errno(0x7a) + EFSBADCRC = syscall.Errno(0x4a) + EFSCORRUPTED = syscall.Errno(0x75) EHOSTDOWN = syscall.Errno(0x70) EHOSTUNREACH = syscall.Errno(0x71) EHWPOISON = syscall.Errno(0x85) @@ -808,7 +826,7 @@ var errorList = [...]struct { {114, "EALREADY", "operation already in progress"}, {115, "EINPROGRESS", "operation now in progress"}, {116, "ESTALE", "stale file handle"}, - {117, "EUCLEAN", "structure needs cleaning"}, + {117, "EFSCORRUPTED", "structure needs cleaning"}, {118, "ENOTNAM", "not a XENIX named type file"}, {119, "ENAVAIL", "no XENIX semaphores available"}, {120, "EISNAM", "is a named type file"}, diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go b/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go index 64347eb354c97..6e87bd6598b0d 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go @@ -156,6 +156,7 @@ const ( NFDBITS = 0x40 NLDLY = 0x100 NOFLSH = 0x80 + NS_GET_ID = 0x8008b70d NS_GET_MNTNS_ID = 0x8008b705 NS_GET_NSTYPE = 0xb703 NS_GET_OWNER_UID = 0xb704 @@ -367,6 +368,7 @@ const ( RTC_WKALM_SET = 0x4028700f SCM_DEVMEM_DMABUF = 0x4f SCM_DEVMEM_LINEAR = 0x4e + SCM_INQ = 0x54 SCM_TIMESTAMPING = 0x25 SCM_TIMESTAMPING_OPT_STATS = 0x36 SCM_TIMESTAMPING_PKTINFO = 0x3a @@ -414,6 +416,7 @@ const ( SO_ERROR = 0x4 SO_INCOMING_CPU = 0x31 SO_INCOMING_NAPI_ID = 0x38 + SO_INQ = 0x54 SO_KEEPALIVE = 0x9 SO_LINGER = 0xd SO_LOCK_FILTER = 0x2c @@ -657,6 +660,8 @@ const ( EDESTADDRREQ = syscall.Errno(0x59) EDOTDOT = syscall.Errno(0x49) EDQUOT = syscall.Errno(0x7a) + EFSBADCRC = syscall.Errno(0x4a) + EFSCORRUPTED = syscall.Errno(0x75) EHOSTDOWN = syscall.Errno(0x70) EHOSTUNREACH = syscall.Errno(0x71) EHWPOISON = syscall.Errno(0x85) @@ -880,7 +885,7 @@ var errorList = [...]struct { {114, "EALREADY", "operation already in progress"}, {115, "EINPROGRESS", "operation now in progress"}, {116, "ESTALE", "stale file handle"}, - {117, "EUCLEAN", "structure needs cleaning"}, + {117, "EFSCORRUPTED", "structure needs cleaning"}, {118, "ENOTNAM", "not a XENIX named type file"}, {119, "ENAVAIL", "no XENIX semaphores available"}, {120, "EISNAM", "is a named type file"}, diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go index 7d71911718fb8..7e2b2e8a6677e 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go @@ -161,6 +161,7 @@ const ( NFDBITS = 0x40 NLDLY = 0x100 NOFLSH = 0x80 + NS_GET_ID = 0x4008b70d NS_GET_MNTNS_ID = 0x4008b705 NS_GET_NSTYPE = 0x2000b703 NS_GET_OWNER_UID = 0x2000b704 @@ -358,6 +359,7 @@ const ( RTC_WKALM_SET = 0x8028700f SCM_DEVMEM_DMABUF = 0x58 SCM_DEVMEM_LINEAR = 0x57 + SCM_INQ = 0x5d SCM_TIMESTAMPING = 0x23 SCM_TIMESTAMPING_OPT_STATS = 0x38 SCM_TIMESTAMPING_PKTINFO = 0x3c @@ -453,6 +455,7 @@ const ( SO_ERROR = 0x1007 SO_INCOMING_CPU = 0x33 SO_INCOMING_NAPI_ID = 0x3a + SO_INQ = 0x5d SO_KEEPALIVE = 0x8 SO_LINGER = 0x80 SO_LOCK_FILTER = 0x28 @@ -694,6 +697,8 @@ const ( EDESTADDRREQ = syscall.Errno(0x27) EDOTDOT = syscall.Errno(0x58) EDQUOT = syscall.Errno(0x45) + EFSBADCRC = syscall.Errno(0x4c) + EFSCORRUPTED = syscall.Errno(0x75) EHOSTDOWN = syscall.Errno(0x40) EHOSTUNREACH = syscall.Errno(0x41) EHWPOISON = syscall.Errno(0x87) @@ -921,7 +926,7 @@ var errorList = [...]struct { {114, "ELIBACC", "can not access a needed shared library"}, {115, "ENOTUNIQ", "name not unique on network"}, {116, "ERESTART", "interrupted system call should be restarted"}, - {117, "EUCLEAN", "structure needs cleaning"}, + {117, "EFSCORRUPTED", "structure needs cleaning"}, {118, "ENOTNAM", "not a XENIX named type file"}, {119, "ENAVAIL", "no XENIX semaphores available"}, {120, "EISNAM", "is a named type file"}, diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux.go b/vendor/golang.org/x/sys/unix/zsyscall_linux.go index 886f5de5bc133..80f40e4013578 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux.go @@ -1785,7 +1785,7 @@ func writev(fd int, iovs []Iovec) (n int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func preadv(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr) (n int, err error) { +func preadvSyscall(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr) (n int, err error) { var _p0 unsafe.Pointer if len(iovs) > 0 { _p0 = unsafe.Pointer(&iovs[0]) @@ -1802,7 +1802,7 @@ func preadv(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr) (n int, err er // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func pwritev(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr) (n int, err error) { +func pwritevSyscall(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr) (n int, err error) { var _p0 unsafe.Pointer if len(iovs) > 0 { _p0 = unsafe.Pointer(&iovs[0]) @@ -1819,7 +1819,7 @@ func pwritev(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr) (n int, err e // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func preadv2(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr, flags int) (n int, err error) { +func preadv2Syscall(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr, flags int) (n int, err error) { var _p0 unsafe.Pointer if len(iovs) > 0 { _p0 = unsafe.Pointer(&iovs[0]) @@ -1836,7 +1836,7 @@ func preadv2(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr, flags int) (n // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func pwritev2(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr, flags int) (n int, err error) { +func pwritev2Syscall(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr, flags int) (n int, err error) { var _p0 unsafe.Pointer if len(iovs) > 0 { _p0 = unsafe.Pointer(&iovs[0]) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go index 1851df14e878e..6487475f0d854 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go @@ -1633,6 +1633,90 @@ var libc_pwrite_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func readv(fd int, iovecs []Iovec) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall(libc_readv_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs))) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_readv_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_readv readv "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func writev(fd int, iovecs []Iovec) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall(libc_writev_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs))) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_writev_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_writev writev "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func preadv(fd int, iovecs []Iovec, offset int64) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall6(libc_preadv_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs)), uintptr(offset), uintptr(offset>>32), 0) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_preadv_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_preadv preadv "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func pwritev(fd int, iovecs []Iovec, offset int64) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall6(libc_pwritev_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs)), uintptr(offset), uintptr(offset>>32), 0) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_pwritev_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_pwritev pwritev "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func read(fd int, p []byte) (n int, err error) { var _p0 unsafe.Pointer if len(p) > 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.s index 0b43c6936563a..f10201dacbe2d 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.s @@ -498,6 +498,26 @@ TEXT libc_pwrite_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_pwrite_trampoline_addr(SB), RODATA, $4 DATA ·libc_pwrite_trampoline_addr(SB)/4, $libc_pwrite_trampoline<>(SB) +TEXT libc_readv_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_readv(SB) +GLOBL ·libc_readv_trampoline_addr(SB), RODATA, $4 +DATA ·libc_readv_trampoline_addr(SB)/4, $libc_readv_trampoline<>(SB) + +TEXT libc_writev_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_writev(SB) +GLOBL ·libc_writev_trampoline_addr(SB), RODATA, $4 +DATA ·libc_writev_trampoline_addr(SB)/4, $libc_writev_trampoline<>(SB) + +TEXT libc_preadv_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_preadv(SB) +GLOBL ·libc_preadv_trampoline_addr(SB), RODATA, $4 +DATA ·libc_preadv_trampoline_addr(SB)/4, $libc_preadv_trampoline<>(SB) + +TEXT libc_pwritev_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_pwritev(SB) +GLOBL ·libc_pwritev_trampoline_addr(SB), RODATA, $4 +DATA ·libc_pwritev_trampoline_addr(SB)/4, $libc_pwritev_trampoline<>(SB) + TEXT libc_read_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_read(SB) GLOBL ·libc_read_trampoline_addr(SB), RODATA, $4 diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go index e1ec0dbe4ecf6..50980475d5a27 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go @@ -1633,6 +1633,90 @@ var libc_pwrite_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func readv(fd int, iovecs []Iovec) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall(libc_readv_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs))) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_readv_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_readv readv "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func writev(fd int, iovecs []Iovec) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall(libc_writev_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs))) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_writev_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_writev writev "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func preadv(fd int, iovecs []Iovec, offset int64) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall6(libc_preadv_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs)), uintptr(offset), 0, 0) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_preadv_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_preadv preadv "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func pwritev(fd int, iovecs []Iovec, offset int64) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall6(libc_pwritev_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs)), uintptr(offset), 0, 0) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_pwritev_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_pwritev pwritev "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func read(fd int, p []byte) (n int, err error) { var _p0 unsafe.Pointer if len(p) > 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.s index 880c6d6e3167b..9de2cbaa453c8 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.s @@ -498,6 +498,26 @@ TEXT libc_pwrite_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_pwrite_trampoline_addr(SB), RODATA, $8 DATA ·libc_pwrite_trampoline_addr(SB)/8, $libc_pwrite_trampoline<>(SB) +TEXT libc_readv_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_readv(SB) +GLOBL ·libc_readv_trampoline_addr(SB), RODATA, $8 +DATA ·libc_readv_trampoline_addr(SB)/8, $libc_readv_trampoline<>(SB) + +TEXT libc_writev_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_writev(SB) +GLOBL ·libc_writev_trampoline_addr(SB), RODATA, $8 +DATA ·libc_writev_trampoline_addr(SB)/8, $libc_writev_trampoline<>(SB) + +TEXT libc_preadv_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_preadv(SB) +GLOBL ·libc_preadv_trampoline_addr(SB), RODATA, $8 +DATA ·libc_preadv_trampoline_addr(SB)/8, $libc_preadv_trampoline<>(SB) + +TEXT libc_pwritev_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_pwritev(SB) +GLOBL ·libc_pwritev_trampoline_addr(SB), RODATA, $8 +DATA ·libc_pwritev_trampoline_addr(SB)/8, $libc_pwritev_trampoline<>(SB) + TEXT libc_read_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_read(SB) GLOBL ·libc_read_trampoline_addr(SB), RODATA, $8 diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go index 7c8452a63e929..33c9c3a43e918 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go @@ -1633,6 +1633,90 @@ var libc_pwrite_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func readv(fd int, iovecs []Iovec) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall(libc_readv_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs))) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_readv_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_readv readv "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func writev(fd int, iovecs []Iovec) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall(libc_writev_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs))) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_writev_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_writev writev "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func preadv(fd int, iovecs []Iovec, offset int64) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall6(libc_preadv_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs)), 0, uintptr(offset), uintptr(offset>>32)) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_preadv_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_preadv preadv "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func pwritev(fd int, iovecs []Iovec, offset int64) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall6(libc_pwritev_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs)), 0, uintptr(offset), uintptr(offset>>32)) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_pwritev_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_pwritev pwritev "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func read(fd int, p []byte) (n int, err error) { var _p0 unsafe.Pointer if len(p) > 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.s index b8ef95b0fa1cb..c6b9175a6f48e 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.s @@ -498,6 +498,26 @@ TEXT libc_pwrite_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_pwrite_trampoline_addr(SB), RODATA, $4 DATA ·libc_pwrite_trampoline_addr(SB)/4, $libc_pwrite_trampoline<>(SB) +TEXT libc_readv_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_readv(SB) +GLOBL ·libc_readv_trampoline_addr(SB), RODATA, $4 +DATA ·libc_readv_trampoline_addr(SB)/4, $libc_readv_trampoline<>(SB) + +TEXT libc_writev_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_writev(SB) +GLOBL ·libc_writev_trampoline_addr(SB), RODATA, $4 +DATA ·libc_writev_trampoline_addr(SB)/4, $libc_writev_trampoline<>(SB) + +TEXT libc_preadv_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_preadv(SB) +GLOBL ·libc_preadv_trampoline_addr(SB), RODATA, $4 +DATA ·libc_preadv_trampoline_addr(SB)/4, $libc_preadv_trampoline<>(SB) + +TEXT libc_pwritev_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_pwritev(SB) +GLOBL ·libc_pwritev_trampoline_addr(SB), RODATA, $4 +DATA ·libc_pwritev_trampoline_addr(SB)/4, $libc_pwritev_trampoline<>(SB) + TEXT libc_read_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_read(SB) GLOBL ·libc_read_trampoline_addr(SB), RODATA, $4 diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go index 2ffdf861f757f..d3410262e96f5 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go @@ -1633,6 +1633,90 @@ var libc_pwrite_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func readv(fd int, iovecs []Iovec) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall(libc_readv_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs))) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_readv_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_readv readv "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func writev(fd int, iovecs []Iovec) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall(libc_writev_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs))) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_writev_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_writev writev "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func preadv(fd int, iovecs []Iovec, offset int64) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall6(libc_preadv_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs)), uintptr(offset), 0, 0) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_preadv_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_preadv preadv "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func pwritev(fd int, iovecs []Iovec, offset int64) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall6(libc_pwritev_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs)), uintptr(offset), 0, 0) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_pwritev_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_pwritev pwritev "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func read(fd int, p []byte) (n int, err error) { var _p0 unsafe.Pointer if len(p) > 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.s index 2af3b5c762fd1..1be10bb45cced 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.s @@ -498,6 +498,26 @@ TEXT libc_pwrite_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_pwrite_trampoline_addr(SB), RODATA, $8 DATA ·libc_pwrite_trampoline_addr(SB)/8, $libc_pwrite_trampoline<>(SB) +TEXT libc_readv_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_readv(SB) +GLOBL ·libc_readv_trampoline_addr(SB), RODATA, $8 +DATA ·libc_readv_trampoline_addr(SB)/8, $libc_readv_trampoline<>(SB) + +TEXT libc_writev_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_writev(SB) +GLOBL ·libc_writev_trampoline_addr(SB), RODATA, $8 +DATA ·libc_writev_trampoline_addr(SB)/8, $libc_writev_trampoline<>(SB) + +TEXT libc_preadv_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_preadv(SB) +GLOBL ·libc_preadv_trampoline_addr(SB), RODATA, $8 +DATA ·libc_preadv_trampoline_addr(SB)/8, $libc_preadv_trampoline<>(SB) + +TEXT libc_pwritev_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_pwritev(SB) +GLOBL ·libc_pwritev_trampoline_addr(SB), RODATA, $8 +DATA ·libc_pwritev_trampoline_addr(SB)/8, $libc_pwritev_trampoline<>(SB) + TEXT libc_read_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_read(SB) GLOBL ·libc_read_trampoline_addr(SB), RODATA, $8 diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.go index 1da08d52675f9..dea19d54e451e 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.go @@ -1633,6 +1633,90 @@ var libc_pwrite_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func readv(fd int, iovecs []Iovec) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall(libc_readv_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs))) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_readv_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_readv readv "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func writev(fd int, iovecs []Iovec) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall(libc_writev_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs))) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_writev_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_writev writev "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func preadv(fd int, iovecs []Iovec, offset int64) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall6(libc_preadv_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs)), uintptr(offset), 0, 0) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_preadv_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_preadv preadv "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func pwritev(fd int, iovecs []Iovec, offset int64) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall6(libc_pwritev_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs)), uintptr(offset), 0, 0) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_pwritev_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_pwritev pwritev "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func read(fd int, p []byte) (n int, err error) { var _p0 unsafe.Pointer if len(p) > 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.s index b7a251353b0fc..a9fec24d964ef 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.s @@ -498,6 +498,26 @@ TEXT libc_pwrite_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_pwrite_trampoline_addr(SB), RODATA, $8 DATA ·libc_pwrite_trampoline_addr(SB)/8, $libc_pwrite_trampoline<>(SB) +TEXT libc_readv_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_readv(SB) +GLOBL ·libc_readv_trampoline_addr(SB), RODATA, $8 +DATA ·libc_readv_trampoline_addr(SB)/8, $libc_readv_trampoline<>(SB) + +TEXT libc_writev_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_writev(SB) +GLOBL ·libc_writev_trampoline_addr(SB), RODATA, $8 +DATA ·libc_writev_trampoline_addr(SB)/8, $libc_writev_trampoline<>(SB) + +TEXT libc_preadv_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_preadv(SB) +GLOBL ·libc_preadv_trampoline_addr(SB), RODATA, $8 +DATA ·libc_preadv_trampoline_addr(SB)/8, $libc_preadv_trampoline<>(SB) + +TEXT libc_pwritev_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_pwritev(SB) +GLOBL ·libc_pwritev_trampoline_addr(SB), RODATA, $8 +DATA ·libc_pwritev_trampoline_addr(SB)/8, $libc_pwritev_trampoline<>(SB) + TEXT libc_read_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_read(SB) GLOBL ·libc_read_trampoline_addr(SB), RODATA, $8 diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.go index 6e85b0aac95c4..436efb586b72d 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.go @@ -1633,6 +1633,90 @@ var libc_pwrite_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func readv(fd int, iovecs []Iovec) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall(libc_readv_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs))) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_readv_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_readv readv "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func writev(fd int, iovecs []Iovec) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall(libc_writev_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs))) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_writev_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_writev writev "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func preadv(fd int, iovecs []Iovec, offset int64) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall6(libc_preadv_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs)), uintptr(offset), 0, 0) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_preadv_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_preadv preadv "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func pwritev(fd int, iovecs []Iovec, offset int64) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall6(libc_pwritev_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs)), uintptr(offset), 0, 0) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_pwritev_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_pwritev pwritev "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func read(fd int, p []byte) (n int, err error) { var _p0 unsafe.Pointer if len(p) > 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.s index f15dadf0552f4..441ed4e4099fa 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.s @@ -597,6 +597,30 @@ TEXT libc_pwrite_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_pwrite_trampoline_addr(SB), RODATA, $8 DATA ·libc_pwrite_trampoline_addr(SB)/8, $libc_pwrite_trampoline<>(SB) +TEXT libc_readv_trampoline<>(SB),NOSPLIT,$0-0 + CALL libc_readv(SB) + RET +GLOBL ·libc_readv_trampoline_addr(SB), RODATA, $8 +DATA ·libc_readv_trampoline_addr(SB)/8, $libc_readv_trampoline<>(SB) + +TEXT libc_writev_trampoline<>(SB),NOSPLIT,$0-0 + CALL libc_writev(SB) + RET +GLOBL ·libc_writev_trampoline_addr(SB), RODATA, $8 +DATA ·libc_writev_trampoline_addr(SB)/8, $libc_writev_trampoline<>(SB) + +TEXT libc_preadv_trampoline<>(SB),NOSPLIT,$0-0 + CALL libc_preadv(SB) + RET +GLOBL ·libc_preadv_trampoline_addr(SB), RODATA, $8 +DATA ·libc_preadv_trampoline_addr(SB)/8, $libc_preadv_trampoline<>(SB) + +TEXT libc_pwritev_trampoline<>(SB),NOSPLIT,$0-0 + CALL libc_pwritev(SB) + RET +GLOBL ·libc_pwritev_trampoline_addr(SB), RODATA, $8 +DATA ·libc_pwritev_trampoline_addr(SB)/8, $libc_pwritev_trampoline<>(SB) + TEXT libc_read_trampoline<>(SB),NOSPLIT,$0-0 CALL libc_read(SB) RET diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.go index 28b487df2514d..d801e4b4e8642 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.go @@ -1633,6 +1633,90 @@ var libc_pwrite_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func readv(fd int, iovecs []Iovec) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall(libc_readv_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs))) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_readv_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_readv readv "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func writev(fd int, iovecs []Iovec) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall(libc_writev_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs))) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_writev_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_writev writev "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func preadv(fd int, iovecs []Iovec, offset int64) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall6(libc_preadv_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs)), uintptr(offset), 0, 0) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_preadv_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_preadv preadv "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func pwritev(fd int, iovecs []Iovec, offset int64) (n int, err error) { + var _p0 unsafe.Pointer + if len(iovecs) > 0 { + _p0 = unsafe.Pointer(&iovecs[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall6(libc_pwritev_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(iovecs)), uintptr(offset), 0, 0) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_pwritev_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_pwritev pwritev "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func read(fd int, p []byte) (n int, err error) { var _p0 unsafe.Pointer if len(p) > 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.s index 1e7f321e436c1..b15cc01747e13 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.s @@ -498,6 +498,26 @@ TEXT libc_pwrite_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_pwrite_trampoline_addr(SB), RODATA, $8 DATA ·libc_pwrite_trampoline_addr(SB)/8, $libc_pwrite_trampoline<>(SB) +TEXT libc_readv_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_readv(SB) +GLOBL ·libc_readv_trampoline_addr(SB), RODATA, $8 +DATA ·libc_readv_trampoline_addr(SB)/8, $libc_readv_trampoline<>(SB) + +TEXT libc_writev_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_writev(SB) +GLOBL ·libc_writev_trampoline_addr(SB), RODATA, $8 +DATA ·libc_writev_trampoline_addr(SB)/8, $libc_writev_trampoline<>(SB) + +TEXT libc_preadv_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_preadv(SB) +GLOBL ·libc_preadv_trampoline_addr(SB), RODATA, $8 +DATA ·libc_preadv_trampoline_addr(SB)/8, $libc_preadv_trampoline<>(SB) + +TEXT libc_pwritev_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_pwritev(SB) +GLOBL ·libc_pwritev_trampoline_addr(SB), RODATA, $8 +DATA ·libc_pwritev_trampoline_addr(SB)/8, $libc_pwritev_trampoline<>(SB) + TEXT libc_read_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_read(SB) GLOBL ·libc_read_trampoline_addr(SB), RODATA, $8 diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go index aca56ee494747..49d1b880310cd 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go @@ -463,4 +463,8 @@ const ( SYS_LISTXATTRAT = 465 SYS_REMOVEXATTRAT = 466 SYS_OPEN_TREE_ATTR = 467 + SYS_FILE_GETATTR = 468 + SYS_FILE_SETATTR = 469 + SYS_LISTNS = 470 + SYS_RSEQ_SLICE_YIELD = 471 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go index 2ea1ef58c3ecc..f11f1de777dc8 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go @@ -342,6 +342,7 @@ const ( SYS_IO_PGETEVENTS = 333 SYS_RSEQ = 334 SYS_URETPROBE = 335 + SYS_UPROBE = 336 SYS_PIDFD_SEND_SIGNAL = 424 SYS_IO_URING_SETUP = 425 SYS_IO_URING_ENTER = 426 @@ -386,4 +387,8 @@ const ( SYS_LISTXATTRAT = 465 SYS_REMOVEXATTRAT = 466 SYS_OPEN_TREE_ATTR = 467 + SYS_FILE_GETATTR = 468 + SYS_FILE_SETATTR = 469 + SYS_LISTNS = 470 + SYS_RSEQ_SLICE_YIELD = 471 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go index d22c8af31968e..bad740b79a81e 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go @@ -427,4 +427,8 @@ const ( SYS_LISTXATTRAT = 465 SYS_REMOVEXATTRAT = 466 SYS_OPEN_TREE_ATTR = 467 + SYS_FILE_GETATTR = 468 + SYS_FILE_SETATTR = 469 + SYS_LISTNS = 470 + SYS_RSEQ_SLICE_YIELD = 471 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go index 5ee264ae97432..fe646d18efc1d 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go @@ -330,4 +330,8 @@ const ( SYS_LISTXATTRAT = 465 SYS_REMOVEXATTRAT = 466 SYS_OPEN_TREE_ATTR = 467 + SYS_FILE_GETATTR = 468 + SYS_FILE_SETATTR = 469 + SYS_LISTNS = 470 + SYS_RSEQ_SLICE_YIELD = 471 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_loong64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_loong64.go index f9f03ebf5fa1b..4362f6d559019 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_loong64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_loong64.go @@ -306,6 +306,7 @@ const ( SYS_LANDLOCK_CREATE_RULESET = 444 SYS_LANDLOCK_ADD_RULE = 445 SYS_LANDLOCK_RESTRICT_SELF = 446 + SYS_MEMFD_SECRET = 447 SYS_PROCESS_MRELEASE = 448 SYS_FUTEX_WAITV = 449 SYS_SET_MEMPOLICY_HOME_NODE = 450 @@ -326,4 +327,8 @@ const ( SYS_LISTXATTRAT = 465 SYS_REMOVEXATTRAT = 466 SYS_OPEN_TREE_ATTR = 467 + SYS_FILE_GETATTR = 468 + SYS_FILE_SETATTR = 469 + SYS_LISTNS = 470 + SYS_RSEQ_SLICE_YIELD = 471 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go index 87c2118e84967..b63d155ae9fe4 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go @@ -447,4 +447,8 @@ const ( SYS_LISTXATTRAT = 4465 SYS_REMOVEXATTRAT = 4466 SYS_OPEN_TREE_ATTR = 4467 + SYS_FILE_GETATTR = 4468 + SYS_FILE_SETATTR = 4469 + SYS_LISTNS = 4470 + SYS_RSEQ_SLICE_YIELD = 4471 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go index 391ad102fb68e..435d43319b82e 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go @@ -377,4 +377,8 @@ const ( SYS_LISTXATTRAT = 5465 SYS_REMOVEXATTRAT = 5466 SYS_OPEN_TREE_ATTR = 5467 + SYS_FILE_GETATTR = 5468 + SYS_FILE_SETATTR = 5469 + SYS_LISTNS = 5470 + SYS_RSEQ_SLICE_YIELD = 5471 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go index 5656157757a9d..dcc0468d67911 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go @@ -377,4 +377,8 @@ const ( SYS_LISTXATTRAT = 5465 SYS_REMOVEXATTRAT = 5466 SYS_OPEN_TREE_ATTR = 5467 + SYS_FILE_GETATTR = 5468 + SYS_FILE_SETATTR = 5469 + SYS_LISTNS = 5470 + SYS_RSEQ_SLICE_YIELD = 5471 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go index 0482b52e3c386..b96f85ebd6076 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go @@ -447,4 +447,8 @@ const ( SYS_LISTXATTRAT = 4465 SYS_REMOVEXATTRAT = 4466 SYS_OPEN_TREE_ATTR = 4467 + SYS_FILE_GETATTR = 4468 + SYS_FILE_SETATTR = 4469 + SYS_LISTNS = 4470 + SYS_RSEQ_SLICE_YIELD = 4471 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc.go index 71806f08f3870..bffa2bd1e7468 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc.go @@ -454,4 +454,8 @@ const ( SYS_LISTXATTRAT = 465 SYS_REMOVEXATTRAT = 466 SYS_OPEN_TREE_ATTR = 467 + SYS_FILE_GETATTR = 468 + SYS_FILE_SETATTR = 469 + SYS_LISTNS = 470 + SYS_RSEQ_SLICE_YIELD = 471 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go index e35a7105829d4..57bfc6b260642 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go @@ -426,4 +426,8 @@ const ( SYS_LISTXATTRAT = 465 SYS_REMOVEXATTRAT = 466 SYS_OPEN_TREE_ATTR = 467 + SYS_FILE_GETATTR = 468 + SYS_FILE_SETATTR = 469 + SYS_LISTNS = 470 + SYS_RSEQ_SLICE_YIELD = 471 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go index 2aea476705e1e..750f706d579a3 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go @@ -426,4 +426,8 @@ const ( SYS_LISTXATTRAT = 465 SYS_REMOVEXATTRAT = 466 SYS_OPEN_TREE_ATTR = 467 + SYS_FILE_GETATTR = 468 + SYS_FILE_SETATTR = 469 + SYS_LISTNS = 470 + SYS_RSEQ_SLICE_YIELD = 471 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go index 6c9bb4e560781..303ccbf46fbc3 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go @@ -331,4 +331,8 @@ const ( SYS_LISTXATTRAT = 465 SYS_REMOVEXATTRAT = 466 SYS_OPEN_TREE_ATTR = 467 + SYS_FILE_GETATTR = 468 + SYS_FILE_SETATTR = 469 + SYS_LISTNS = 470 + SYS_RSEQ_SLICE_YIELD = 471 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go index 680bc9915a314..5e5dd4ccb07ae 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go @@ -392,4 +392,8 @@ const ( SYS_LISTXATTRAT = 465 SYS_REMOVEXATTRAT = 466 SYS_OPEN_TREE_ATTR = 467 + SYS_FILE_GETATTR = 468 + SYS_FILE_SETATTR = 469 + SYS_LISTNS = 470 + SYS_RSEQ_SLICE_YIELD = 471 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_sparc64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_sparc64.go index 620f271052f98..f7c4fb3df987b 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_sparc64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_sparc64.go @@ -374,6 +374,7 @@ const ( SYS_FSMOUNT = 432 SYS_FSPICK = 433 SYS_PIDFD_OPEN = 434 + SYS_CLONE3 = 435 SYS_CLOSE_RANGE = 436 SYS_OPENAT2 = 437 SYS_PIDFD_GETFD = 438 @@ -405,4 +406,8 @@ const ( SYS_LISTXATTRAT = 465 SYS_REMOVEXATTRAT = 466 SYS_OPEN_TREE_ATTR = 467 + SYS_FILE_GETATTR = 468 + SYS_FILE_SETATTR = 469 + SYS_LISTNS = 470 + SYS_RSEQ_SLICE_YIELD = 471 ) diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux.go b/vendor/golang.org/x/sys/unix/ztypes_linux.go index 45476a73c61d8..526a0d5f434eb 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux.go @@ -18,6 +18,11 @@ type ( _C_long_long int64 ) +type KernelTimespec struct { + Sec int64 + Nsec int64 +} + type ItimerSpec struct { Interval Timespec Value Timespec @@ -521,6 +526,14 @@ type TCPInfo struct { Total_rto uint16 Total_rto_recoveries uint16 Total_rto_time uint32 + Received_ce uint32 + Delivered_e1_bytes uint32 + Delivered_e0_bytes uint32 + Delivered_ce_bytes uint32 + Received_e1_bytes uint32 + Received_e0_bytes uint32 + Received_ce_bytes uint32 + _ [4]byte } type TCPVegasInfo struct { @@ -586,7 +599,7 @@ const ( SizeofIPv6MTUInfo = 0x20 SizeofICMPv6Filter = 0x20 SizeofUcred = 0xc - SizeofTCPInfo = 0xf8 + SizeofTCPInfo = 0x118 SizeofTCPCCInfo = 0x14 SizeofCanFilter = 0x8 SizeofTCPRepairOpt = 0x8 @@ -1324,7 +1337,7 @@ const ( PERF_RECORD_CGROUP = 0x13 PERF_RECORD_TEXT_POKE = 0x14 PERF_RECORD_AUX_OUTPUT_HW_ID = 0x15 - PERF_RECORD_MAX = 0x16 + PERF_RECORD_MAX = 0x17 PERF_RECORD_KSYMBOL_TYPE_UNKNOWN = 0x0 PERF_RECORD_KSYMBOL_TYPE_BPF = 0x1 PERF_RECORD_KSYMBOL_TYPE_OOL = 0x2 @@ -3566,7 +3579,7 @@ const ( DEVLINK_ATTR_LINECARD_SUPPORTED_TYPES = 0xae DEVLINK_ATTR_NESTED_DEVLINK = 0xaf DEVLINK_ATTR_SELFTESTS = 0xb0 - DEVLINK_ATTR_MAX = 0xb3 + DEVLINK_ATTR_MAX = 0xb7 DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE = 0x0 DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX = 0x1 DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT = 0x0 @@ -3888,7 +3901,7 @@ const ( ETHTOOL_MSG_PHY_GET = 0x2d ETHTOOL_MSG_TSCONFIG_GET = 0x2e ETHTOOL_MSG_TSCONFIG_SET = 0x2f - ETHTOOL_MSG_USER_MAX = 0x2f + ETHTOOL_MSG_USER_MAX = 0x33 ETHTOOL_MSG_KERNEL_NONE = 0x0 ETHTOOL_MSG_STRSET_GET_REPLY = 0x1 ETHTOOL_MSG_LINKINFO_GET_REPLY = 0x2 @@ -3938,7 +3951,7 @@ const ( ETHTOOL_MSG_PHY_NTF = 0x2e ETHTOOL_MSG_TSCONFIG_GET_REPLY = 0x2f ETHTOOL_MSG_TSCONFIG_SET_REPLY = 0x30 - ETHTOOL_MSG_KERNEL_MAX = 0x30 + ETHTOOL_MSG_KERNEL_MAX = 0x36 ETHTOOL_FLAG_COMPACT_BITSETS = 0x1 ETHTOOL_FLAG_OMIT_REPLY = 0x2 ETHTOOL_FLAG_STATS = 0x4 @@ -4867,7 +4880,7 @@ const ( NL80211_ATTR_MAC_HINT = 0xc8 NL80211_ATTR_MAC_MASK = 0xd7 NL80211_ATTR_MAX_AP_ASSOC_STA = 0xca - NL80211_ATTR_MAX = 0x151 + NL80211_ATTR_MAX = 0x15c NL80211_ATTR_MAX_CRIT_PROT_DURATION = 0xb4 NL80211_ATTR_MAX_CSA_COUNTERS = 0xce NL80211_ATTR_MAX_HW_TIMESTAMP_PEERS = 0x143 @@ -5082,12 +5095,12 @@ const ( NL80211_ATTR_WOWLAN_TRIGGERS = 0x75 NL80211_ATTR_WOWLAN_TRIGGERS_SUPPORTED = 0x76 NL80211_ATTR_WPA_VERSIONS = 0x4b - NL80211_AUTHTYPE_AUTOMATIC = 0x8 + NL80211_AUTHTYPE_AUTOMATIC = 0x9 NL80211_AUTHTYPE_FILS_PK = 0x7 NL80211_AUTHTYPE_FILS_SK = 0x5 NL80211_AUTHTYPE_FILS_SK_PFS = 0x6 NL80211_AUTHTYPE_FT = 0x2 - NL80211_AUTHTYPE_MAX = 0x7 + NL80211_AUTHTYPE_MAX = 0x8 NL80211_AUTHTYPE_NETWORK_EAP = 0x3 NL80211_AUTHTYPE_OPEN_SYSTEM = 0x0 NL80211_AUTHTYPE_SAE = 0x4 @@ -5120,7 +5133,7 @@ const ( NL80211_BAND_IFTYPE_ATTR_HE_CAP_PHY = 0x3 NL80211_BAND_IFTYPE_ATTR_HE_CAP_PPE = 0x5 NL80211_BAND_IFTYPE_ATTR_IFTYPES = 0x1 - NL80211_BAND_IFTYPE_ATTR_MAX = 0xb + NL80211_BAND_IFTYPE_ATTR_MAX = 0xd NL80211_BAND_IFTYPE_ATTR_VENDOR_ELEMS = 0x7 NL80211_BAND_LC = 0x5 NL80211_BAND_S1GHZ = 0x4 @@ -5255,7 +5268,7 @@ const ( NL80211_CMD_LEAVE_MESH = 0x45 NL80211_CMD_LEAVE_OCB = 0x6d NL80211_CMD_LINKS_REMOVED = 0x9a - NL80211_CMD_MAX = 0x9d + NL80211_CMD_MAX = 0x9f NL80211_CMD_MICHAEL_MIC_FAILURE = 0x29 NL80211_CMD_MODIFY_LINK_STA = 0x97 NL80211_CMD_NAN_MATCH = 0x78 @@ -5501,7 +5514,7 @@ const ( NL80211_FREQUENCY_ATTR_GO_CONCURRENT = 0xf NL80211_FREQUENCY_ATTR_INDOOR_ONLY = 0xe NL80211_FREQUENCY_ATTR_IR_CONCURRENT = 0xf - NL80211_FREQUENCY_ATTR_MAX = 0x22 + NL80211_FREQUENCY_ATTR_MAX = 0x27 NL80211_FREQUENCY_ATTR_MAX_TX_POWER = 0x6 NL80211_FREQUENCY_ATTR_NO_10MHZ = 0x11 NL80211_FREQUENCY_ATTR_NO_160MHZ = 0xc @@ -5766,7 +5779,7 @@ const ( NL80211_PMSR_FTM_CAPA_ATTR_ASAP = 0x1 NL80211_PMSR_FTM_CAPA_ATTR_BANDWIDTHS = 0x6 NL80211_PMSR_FTM_CAPA_ATTR_MAX_BURSTS_EXPONENT = 0x7 - NL80211_PMSR_FTM_CAPA_ATTR_MAX = 0xa + NL80211_PMSR_FTM_CAPA_ATTR_MAX = 0x12 NL80211_PMSR_FTM_CAPA_ATTR_MAX_FTMS_PER_BURST = 0x8 NL80211_PMSR_FTM_CAPA_ATTR_NON_ASAP = 0x2 NL80211_PMSR_FTM_CAPA_ATTR_NON_TRIGGER_BASED = 0xa @@ -5788,7 +5801,7 @@ const ( NL80211_PMSR_FTM_REQ_ATTR_BURST_PERIOD = 0x4 NL80211_PMSR_FTM_REQ_ATTR_FTMS_PER_BURST = 0x6 NL80211_PMSR_FTM_REQ_ATTR_LMR_FEEDBACK = 0xc - NL80211_PMSR_FTM_REQ_ATTR_MAX = 0xd + NL80211_PMSR_FTM_REQ_ATTR_MAX = 0xe NL80211_PMSR_FTM_REQ_ATTR_NON_TRIGGER_BASED = 0xb NL80211_PMSR_FTM_REQ_ATTR_NUM_BURSTS_EXP = 0x3 NL80211_PMSR_FTM_REQ_ATTR_NUM_FTMR_RETRIES = 0x7 @@ -5806,7 +5819,7 @@ const ( NL80211_PMSR_FTM_RESP_ATTR_FAIL_REASON = 0x1 NL80211_PMSR_FTM_RESP_ATTR_FTMS_PER_BURST = 0x8 NL80211_PMSR_FTM_RESP_ATTR_LCI = 0x13 - NL80211_PMSR_FTM_RESP_ATTR_MAX = 0x15 + NL80211_PMSR_FTM_RESP_ATTR_MAX = 0x16 NL80211_PMSR_FTM_RESP_ATTR_NUM_BURSTS_EXP = 0x6 NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_ATTEMPTS = 0x3 NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_SUCCESSES = 0x4 @@ -5913,7 +5926,7 @@ const ( NL80211_RATE_INFO_HE_RU_ALLOC_52 = 0x1 NL80211_RATE_INFO_HE_RU_ALLOC_996 = 0x5 NL80211_RATE_INFO_HE_RU_ALLOC = 0x11 - NL80211_RATE_INFO_MAX = 0x1d + NL80211_RATE_INFO_MAX = 0x20 NL80211_RATE_INFO_MCS = 0x2 NL80211_RATE_INFO_S1G_MCS = 0x17 NL80211_RATE_INFO_S1G_NSS = 0x18 @@ -6167,7 +6180,7 @@ const ( NL80211_TXRATE_HT = 0x2 NL80211_TXRATE_LEGACY = 0x1 NL80211_TX_RATE_LIMITED = 0x1 - NL80211_TXRATE_MAX = 0x7 + NL80211_TXRATE_MAX = 0xa NL80211_TXRATE_MCS = 0x2 NL80211_TXRATE_VHT = 0x3 NL80211_UNSOL_BCAST_PROBE_RESP_ATTR_INT = 0x1 @@ -6183,7 +6196,7 @@ const ( NL80211_WIPHY_RADIO_ATTR_FREQ_RANGE = 0x2 NL80211_WIPHY_RADIO_ATTR_INDEX = 0x1 NL80211_WIPHY_RADIO_ATTR_INTERFACE_COMBINATION = 0x3 - NL80211_WIPHY_RADIO_ATTR_MAX = 0x4 + NL80211_WIPHY_RADIO_ATTR_MAX = 0x5 NL80211_WIPHY_RADIO_FREQ_ATTR_END = 0x2 NL80211_WIPHY_RADIO_FREQ_ATTR_MAX = 0x2 NL80211_WIPHY_RADIO_FREQ_ATTR_START = 0x1 @@ -6384,3 +6397,79 @@ const ( MPOL_PREFERRED_MANY = 0x5 MPOL_WEIGHTED_INTERLEAVE = 0x6 ) + +const ( + GPIO_V2_GET_LINEINFO_IOCTL = 0xc100b405 + GPIO_V2_GET_LINE_IOCTL = 0xc250b407 + GPIO_V2_LINE_GET_VALUES_IOCTL = 0xc010b40e + GPIO_V2_LINE_SET_VALUES_IOCTL = 0xc010b40f + GPIO_V2_GET_LINEINFO_WATCH_IOCTL = 0xc100b406 + GPIO_GET_LINEINFO_UNWATCH_IOCTL = 0xc004b40c +) +const ( + GPIO_V2_LINE_ATTR_ID_FLAGS = 0x1 + GPIO_V2_LINE_ATTR_ID_OUTPUT_VALUES = 0x2 + GPIO_V2_LINE_ATTR_ID_DEBOUNCE = 0x3 + GPIO_V2_LINE_CHANGED_REQUESTED = 0x1 + GPIO_V2_LINE_CHANGED_RELEASED = 0x2 + GPIO_V2_LINE_CHANGED_CONFIG = 0x3 + GPIO_V2_LINE_EVENT_RISING_EDGE = 0x1 + GPIO_V2_LINE_EVENT_FALLING_EDGE = 0x2 +) + +type GPIOChipInfo struct { + Name [32]byte + Label [32]byte + Lines uint32 +} +type GPIOV2LineValues struct { + Bits uint64 + Mask uint64 +} +type GPIOV2LineAttribute struct { + Id uint32 + _ uint32 + Flags uint64 +} +type GPIOV2LineConfigAttribute struct { + Attr GPIOV2LineAttribute + Mask uint64 +} +type GPIOV2LineConfig struct { + Flags uint64 + Num_attrs uint32 + _ [5]uint32 + Attrs [10]GPIOV2LineConfigAttribute +} +type GPIOV2LineRequest struct { + Offsets [64]uint32 + Consumer [32]byte + Config GPIOV2LineConfig + Num_lines uint32 + Event_buffer_size uint32 + _ [5]uint32 + Fd int32 +} +type GPIOV2LineInfo struct { + Name [32]byte + Consumer [32]byte + Offset uint32 + Num_attrs uint32 + Flags uint64 + Attrs [10]GPIOV2LineAttribute + _ [4]uint32 +} +type GPIOV2LineInfoChanged struct { + Info GPIOV2LineInfo + Timestamp_ns uint64 + Event_type uint32 + _ [5]uint32 +} +type GPIOV2LineEvent struct { + Timestamp_ns uint64 + Id uint32 + Offset uint32 + Seqno uint32 + Line_seqno uint32 + _ [6]uint32 +} diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_386.go b/vendor/golang.org/x/sys/unix/ztypes_linux_386.go index 485f2d3a1bc8c..aede1de7f2e92 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_386.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_386.go @@ -354,6 +354,14 @@ type Taskstats struct { Wpcopy_delay_min uint64 Irq_delay_max uint64 Irq_delay_min uint64 + Cpu_delay_max_ts KernelTimespec + Blkio_delay_max_ts KernelTimespec + Swapin_delay_max_ts KernelTimespec + Freepages_delay_max_ts KernelTimespec + Thrashing_delay_max_ts KernelTimespec + Compact_delay_max_ts KernelTimespec + Wpcopy_delay_max_ts KernelTimespec + Irq_delay_max_ts KernelTimespec } type cpuMask uint32 @@ -703,3 +711,7 @@ type SysvShmDesc struct { _ uint32 _ uint32 } + +const ( + GPIO_GET_CHIPINFO_IOCTL = 0x8044b401 +) diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go index ecbd1ad8bc54b..bb3bc4dc2c4ab 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go @@ -367,6 +367,14 @@ type Taskstats struct { Wpcopy_delay_min uint64 Irq_delay_max uint64 Irq_delay_min uint64 + Cpu_delay_max_ts KernelTimespec + Blkio_delay_max_ts KernelTimespec + Swapin_delay_max_ts KernelTimespec + Freepages_delay_max_ts KernelTimespec + Thrashing_delay_max_ts KernelTimespec + Compact_delay_max_ts KernelTimespec + Wpcopy_delay_max_ts KernelTimespec + Irq_delay_max_ts KernelTimespec } type cpuMask uint64 @@ -717,3 +725,7 @@ type SysvShmDesc struct { _ uint64 _ uint64 } + +const ( + GPIO_GET_CHIPINFO_IOCTL = 0x8044b401 +) diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_arm.go b/vendor/golang.org/x/sys/unix/ztypes_linux_arm.go index 02f0463a44b24..1fdf4c5175c81 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_arm.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_arm.go @@ -345,6 +345,14 @@ type Taskstats struct { Wpcopy_delay_min uint64 Irq_delay_max uint64 Irq_delay_min uint64 + Cpu_delay_max_ts KernelTimespec + Blkio_delay_max_ts KernelTimespec + Swapin_delay_max_ts KernelTimespec + Freepages_delay_max_ts KernelTimespec + Thrashing_delay_max_ts KernelTimespec + Compact_delay_max_ts KernelTimespec + Wpcopy_delay_max_ts KernelTimespec + Irq_delay_max_ts KernelTimespec } type cpuMask uint32 @@ -697,3 +705,7 @@ type SysvShmDesc struct { _ uint32 _ uint32 } + +const ( + GPIO_GET_CHIPINFO_IOCTL = 0x8044b401 +) diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go index 6f4d400d2417c..063e6f0b41e21 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go @@ -346,6 +346,14 @@ type Taskstats struct { Wpcopy_delay_min uint64 Irq_delay_max uint64 Irq_delay_min uint64 + Cpu_delay_max_ts KernelTimespec + Blkio_delay_max_ts KernelTimespec + Swapin_delay_max_ts KernelTimespec + Freepages_delay_max_ts KernelTimespec + Thrashing_delay_max_ts KernelTimespec + Compact_delay_max_ts KernelTimespec + Wpcopy_delay_max_ts KernelTimespec + Irq_delay_max_ts KernelTimespec } type cpuMask uint64 @@ -696,3 +704,7 @@ type SysvShmDesc struct { _ uint64 _ uint64 } + +const ( + GPIO_GET_CHIPINFO_IOCTL = 0x8044b401 +) diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_loong64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_loong64.go index cd532cfa55899..9cf836c708f6a 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_loong64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_loong64.go @@ -347,6 +347,14 @@ type Taskstats struct { Wpcopy_delay_min uint64 Irq_delay_max uint64 Irq_delay_min uint64 + Cpu_delay_max_ts KernelTimespec + Blkio_delay_max_ts KernelTimespec + Swapin_delay_max_ts KernelTimespec + Freepages_delay_max_ts KernelTimespec + Thrashing_delay_max_ts KernelTimespec + Compact_delay_max_ts KernelTimespec + Wpcopy_delay_max_ts KernelTimespec + Irq_delay_max_ts KernelTimespec } type cpuMask uint64 @@ -697,3 +705,7 @@ type SysvShmDesc struct { _ uint64 _ uint64 } + +const ( + GPIO_GET_CHIPINFO_IOCTL = 0x8044b401 +) diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_mips.go b/vendor/golang.org/x/sys/unix/ztypes_linux_mips.go index 4133620851752..1d222fcb31224 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_mips.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_mips.go @@ -350,6 +350,14 @@ type Taskstats struct { Wpcopy_delay_min uint64 Irq_delay_max uint64 Irq_delay_min uint64 + Cpu_delay_max_ts KernelTimespec + Blkio_delay_max_ts KernelTimespec + Swapin_delay_max_ts KernelTimespec + Freepages_delay_max_ts KernelTimespec + Thrashing_delay_max_ts KernelTimespec + Compact_delay_max_ts KernelTimespec + Wpcopy_delay_max_ts KernelTimespec + Irq_delay_max_ts KernelTimespec } type cpuMask uint32 @@ -702,3 +710,7 @@ type SysvShmDesc struct { Ctime_high uint16 _ uint16 } + +const ( + GPIO_GET_CHIPINFO_IOCTL = 0x4044b401 +) diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_mips64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_mips64.go index eaa37eb718e2d..912cc4ab63b3e 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_mips64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_mips64.go @@ -349,6 +349,14 @@ type Taskstats struct { Wpcopy_delay_min uint64 Irq_delay_max uint64 Irq_delay_min uint64 + Cpu_delay_max_ts KernelTimespec + Blkio_delay_max_ts KernelTimespec + Swapin_delay_max_ts KernelTimespec + Freepages_delay_max_ts KernelTimespec + Thrashing_delay_max_ts KernelTimespec + Compact_delay_max_ts KernelTimespec + Wpcopy_delay_max_ts KernelTimespec + Irq_delay_max_ts KernelTimespec } type cpuMask uint64 @@ -699,3 +707,7 @@ type SysvShmDesc struct { _ uint64 _ uint64 } + +const ( + GPIO_GET_CHIPINFO_IOCTL = 0x4044b401 +) diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go b/vendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go index 98ae6a1e4ac4d..1e358ef34f280 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go @@ -349,6 +349,14 @@ type Taskstats struct { Wpcopy_delay_min uint64 Irq_delay_max uint64 Irq_delay_min uint64 + Cpu_delay_max_ts KernelTimespec + Blkio_delay_max_ts KernelTimespec + Swapin_delay_max_ts KernelTimespec + Freepages_delay_max_ts KernelTimespec + Thrashing_delay_max_ts KernelTimespec + Compact_delay_max_ts KernelTimespec + Wpcopy_delay_max_ts KernelTimespec + Irq_delay_max_ts KernelTimespec } type cpuMask uint64 @@ -699,3 +707,7 @@ type SysvShmDesc struct { _ uint64 _ uint64 } + +const ( + GPIO_GET_CHIPINFO_IOCTL = 0x4044b401 +) diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_mipsle.go b/vendor/golang.org/x/sys/unix/ztypes_linux_mipsle.go index cae1961594d24..df59f32f5e480 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_mipsle.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_mipsle.go @@ -350,6 +350,14 @@ type Taskstats struct { Wpcopy_delay_min uint64 Irq_delay_max uint64 Irq_delay_min uint64 + Cpu_delay_max_ts KernelTimespec + Blkio_delay_max_ts KernelTimespec + Swapin_delay_max_ts KernelTimespec + Freepages_delay_max_ts KernelTimespec + Thrashing_delay_max_ts KernelTimespec + Compact_delay_max_ts KernelTimespec + Wpcopy_delay_max_ts KernelTimespec + Irq_delay_max_ts KernelTimespec } type cpuMask uint32 @@ -702,3 +710,7 @@ type SysvShmDesc struct { Ctime_high uint16 _ uint16 } + +const ( + GPIO_GET_CHIPINFO_IOCTL = 0x4044b401 +) diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc.go b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc.go index 6ce3b4e02830e..29355aa0bfeff 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc.go @@ -357,6 +357,14 @@ type Taskstats struct { Wpcopy_delay_min uint64 Irq_delay_max uint64 Irq_delay_min uint64 + Cpu_delay_max_ts KernelTimespec + Blkio_delay_max_ts KernelTimespec + Swapin_delay_max_ts KernelTimespec + Freepages_delay_max_ts KernelTimespec + Thrashing_delay_max_ts KernelTimespec + Compact_delay_max_ts KernelTimespec + Wpcopy_delay_max_ts KernelTimespec + Irq_delay_max_ts KernelTimespec } type cpuMask uint32 @@ -710,3 +718,7 @@ type SysvShmDesc struct { _ uint32 _ [4]byte } + +const ( + GPIO_GET_CHIPINFO_IOCTL = 0x4044b401 +) diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go index c7429c6a1461e..c6083a15d7d9f 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go @@ -356,6 +356,14 @@ type Taskstats struct { Wpcopy_delay_min uint64 Irq_delay_max uint64 Irq_delay_min uint64 + Cpu_delay_max_ts KernelTimespec + Blkio_delay_max_ts KernelTimespec + Swapin_delay_max_ts KernelTimespec + Freepages_delay_max_ts KernelTimespec + Thrashing_delay_max_ts KernelTimespec + Compact_delay_max_ts KernelTimespec + Wpcopy_delay_max_ts KernelTimespec + Irq_delay_max_ts KernelTimespec } type cpuMask uint64 @@ -705,3 +713,7 @@ type SysvShmDesc struct { _ uint64 _ uint64 } + +const ( + GPIO_GET_CHIPINFO_IOCTL = 0x4044b401 +) diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go index 4bf4baf4cac56..6321cc7626670 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go @@ -356,6 +356,14 @@ type Taskstats struct { Wpcopy_delay_min uint64 Irq_delay_max uint64 Irq_delay_min uint64 + Cpu_delay_max_ts KernelTimespec + Blkio_delay_max_ts KernelTimespec + Swapin_delay_max_ts KernelTimespec + Freepages_delay_max_ts KernelTimespec + Thrashing_delay_max_ts KernelTimespec + Compact_delay_max_ts KernelTimespec + Wpcopy_delay_max_ts KernelTimespec + Irq_delay_max_ts KernelTimespec } type cpuMask uint64 @@ -705,3 +713,7 @@ type SysvShmDesc struct { _ uint64 _ uint64 } + +const ( + GPIO_GET_CHIPINFO_IOCTL = 0x4044b401 +) diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go index e9709d70afbd8..b44f402feb68c 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go @@ -374,6 +374,14 @@ type Taskstats struct { Wpcopy_delay_min uint64 Irq_delay_max uint64 Irq_delay_min uint64 + Cpu_delay_max_ts KernelTimespec + Blkio_delay_max_ts KernelTimespec + Swapin_delay_max_ts KernelTimespec + Freepages_delay_max_ts KernelTimespec + Thrashing_delay_max_ts KernelTimespec + Compact_delay_max_ts KernelTimespec + Wpcopy_delay_max_ts KernelTimespec + Irq_delay_max_ts KernelTimespec } type cpuMask uint64 @@ -784,3 +792,7 @@ const ( RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE = 0x6 RISCV_HWPROBE_WHICH_CPUS = 0x1 ) + +const ( + GPIO_GET_CHIPINFO_IOCTL = 0x8044b401 +) diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_s390x.go b/vendor/golang.org/x/sys/unix/ztypes_linux_s390x.go index fb44268ca7da3..b22c795a646dd 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_s390x.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_s390x.go @@ -369,6 +369,14 @@ type Taskstats struct { Wpcopy_delay_min uint64 Irq_delay_max uint64 Irq_delay_min uint64 + Cpu_delay_max_ts KernelTimespec + Blkio_delay_max_ts KernelTimespec + Swapin_delay_max_ts KernelTimespec + Freepages_delay_max_ts KernelTimespec + Thrashing_delay_max_ts KernelTimespec + Compact_delay_max_ts KernelTimespec + Wpcopy_delay_max_ts KernelTimespec + Irq_delay_max_ts KernelTimespec } type cpuMask uint64 @@ -719,3 +727,7 @@ type SysvShmDesc struct { _ uint64 _ uint64 } + +const ( + GPIO_GET_CHIPINFO_IOCTL = 0x8044b401 +) diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_sparc64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_sparc64.go index 9c38265c74aff..0b18075b53e36 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_sparc64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_sparc64.go @@ -351,6 +351,14 @@ type Taskstats struct { Wpcopy_delay_min uint64 Irq_delay_max uint64 Irq_delay_min uint64 + Cpu_delay_max_ts KernelTimespec + Blkio_delay_max_ts KernelTimespec + Swapin_delay_max_ts KernelTimespec + Freepages_delay_max_ts KernelTimespec + Thrashing_delay_max_ts KernelTimespec + Compact_delay_max_ts KernelTimespec + Wpcopy_delay_max_ts KernelTimespec + Irq_delay_max_ts KernelTimespec } type cpuMask uint64 @@ -700,3 +708,7 @@ type SysvShmDesc struct { _ uint64 _ uint64 } + +const ( + GPIO_GET_CHIPINFO_IOCTL = 0x4044b401 +) diff --git a/vendor/golang.org/x/sys/windows/syscall_windows.go b/vendor/golang.org/x/sys/windows/syscall_windows.go index 453a7b97f1cb9..9755bca9fd39d 100644 --- a/vendor/golang.org/x/sys/windows/syscall_windows.go +++ b/vendor/golang.org/x/sys/windows/syscall_windows.go @@ -452,6 +452,7 @@ func NewCallbackCDecl(fn interface{}) uintptr { //sys RtlInitString(destinationString *NTString, sourceString *byte) = ntdll.RtlInitString //sys NtCreateFile(handle *Handle, access uint32, oa *OBJECT_ATTRIBUTES, iosb *IO_STATUS_BLOCK, allocationSize *int64, attributes uint32, share uint32, disposition uint32, options uint32, eabuffer uintptr, ealength uint32) (ntstatus error) = ntdll.NtCreateFile //sys NtCreateNamedPipeFile(pipe *Handle, access uint32, oa *OBJECT_ATTRIBUTES, iosb *IO_STATUS_BLOCK, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (ntstatus error) = ntdll.NtCreateNamedPipeFile +//sys NtQueryInformationFile(handle Handle, iosb *IO_STATUS_BLOCK, outBuffer *byte, outBufferLen uint32, class uint32) (ntstatus error) = ntdll.NtQueryInformationFile //sys NtSetInformationFile(handle Handle, iosb *IO_STATUS_BLOCK, inBuffer *byte, inBufferLen uint32, class uint32) (ntstatus error) = ntdll.NtSetInformationFile //sys RtlDosPathNameToNtPathName(dosName *uint16, ntName *NTUnicodeString, ntFileNamePart *uint16, relativeName *RTL_RELATIVE_NAME) (ntstatus error) = ntdll.RtlDosPathNameToNtPathName_U_WithStatus //sys RtlDosPathNameToRelativeNtPathName(dosName *uint16, ntName *NTUnicodeString, ntFileNamePart *uint16, relativeName *RTL_RELATIVE_NAME) (ntstatus error) = ntdll.RtlDosPathNameToRelativeNtPathName_U_WithStatus @@ -460,6 +461,8 @@ func NewCallbackCDecl(fn interface{}) uintptr { //sys NtSetInformationProcess(proc Handle, procInfoClass int32, procInfo unsafe.Pointer, procInfoLen uint32) (ntstatus error) = ntdll.NtSetInformationProcess //sys NtQuerySystemInformation(sysInfoClass int32, sysInfo unsafe.Pointer, sysInfoLen uint32, retLen *uint32) (ntstatus error) = ntdll.NtQuerySystemInformation //sys NtSetSystemInformation(sysInfoClass int32, sysInfo unsafe.Pointer, sysInfoLen uint32) (ntstatus error) = ntdll.NtSetSystemInformation +//sys NtQueryEaFile(handle Handle, iosb *IO_STATUS_BLOCK, outBuffer *byte, outBufferLen uint32, returnSingleEntry bool, eaList *byte, eaListLen uint32, eaIndex *uint32, restartScan bool) (ntstatus error) = ntdll.NtQueryEaFile +//sys NtSetEaFile(handle Handle, iosb *IO_STATUS_BLOCK, inBuffer *byte, inBufferLen uint32) (ntstatus error) = ntdll.NtSetEaFile //sys RtlAddFunctionTable(functionTable *RUNTIME_FUNCTION, entryCount uint32, baseAddress uintptr) (ret bool) = ntdll.RtlAddFunctionTable //sys RtlDeleteFunctionTable(functionTable *RUNTIME_FUNCTION) (ret bool) = ntdll.RtlDeleteFunctionTable diff --git a/vendor/golang.org/x/sys/windows/types_windows.go b/vendor/golang.org/x/sys/windows/types_windows.go index d82299e33f6ee..d2574a73ee00a 100644 --- a/vendor/golang.org/x/sys/windows/types_windows.go +++ b/vendor/golang.org/x/sys/windows/types_windows.go @@ -3043,8 +3043,10 @@ const ( ) const ( - // FileInformationClass for NtSetInformationFile + // FileInformationClass for NtSetInformationFile/NtQueryInformationFile, see + // https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ne-wdm-_file_information_class FileBasicInformation = 4 + FileEaInformation = 7 FileRenameInformation = 10 FileDispositionInformation = 13 FilePositionInformation = 14 diff --git a/vendor/golang.org/x/sys/windows/zsyscall_windows.go b/vendor/golang.org/x/sys/windows/zsyscall_windows.go index a506ac0f1ed4a..192d193002326 100644 --- a/vendor/golang.org/x/sys/windows/zsyscall_windows.go +++ b/vendor/golang.org/x/sys/windows/zsyscall_windows.go @@ -428,8 +428,11 @@ var ( procNetUserGetInfo = modnetapi32.NewProc("NetUserGetInfo") procNtCreateFile = modntdll.NewProc("NtCreateFile") procNtCreateNamedPipeFile = modntdll.NewProc("NtCreateNamedPipeFile") + procNtQueryEaFile = modntdll.NewProc("NtQueryEaFile") + procNtQueryInformationFile = modntdll.NewProc("NtQueryInformationFile") procNtQueryInformationProcess = modntdll.NewProc("NtQueryInformationProcess") procNtQuerySystemInformation = modntdll.NewProc("NtQuerySystemInformation") + procNtSetEaFile = modntdll.NewProc("NtSetEaFile") procNtSetInformationFile = modntdll.NewProc("NtSetInformationFile") procNtSetInformationProcess = modntdll.NewProc("NtSetInformationProcess") procNtSetSystemInformation = modntdll.NewProc("NtSetSystemInformation") @@ -3740,6 +3743,30 @@ func NtCreateNamedPipeFile(pipe *Handle, access uint32, oa *OBJECT_ATTRIBUTES, i return } +func NtQueryEaFile(handle Handle, iosb *IO_STATUS_BLOCK, outBuffer *byte, outBufferLen uint32, returnSingleEntry bool, eaList *byte, eaListLen uint32, eaIndex *uint32, restartScan bool) (ntstatus error) { + var _p0 uint32 + if returnSingleEntry { + _p0 = 1 + } + var _p1 uint32 + if restartScan { + _p1 = 1 + } + r0, _, _ := syscall.SyscallN(procNtQueryEaFile.Addr(), uintptr(handle), uintptr(unsafe.Pointer(iosb)), uintptr(unsafe.Pointer(outBuffer)), uintptr(outBufferLen), uintptr(_p0), uintptr(unsafe.Pointer(eaList)), uintptr(eaListLen), uintptr(unsafe.Pointer(eaIndex)), uintptr(_p1)) + if r0 != 0 { + ntstatus = NTStatus(r0) + } + return +} + +func NtQueryInformationFile(handle Handle, iosb *IO_STATUS_BLOCK, outBuffer *byte, outBufferLen uint32, class uint32) (ntstatus error) { + r0, _, _ := syscall.SyscallN(procNtQueryInformationFile.Addr(), uintptr(handle), uintptr(unsafe.Pointer(iosb)), uintptr(unsafe.Pointer(outBuffer)), uintptr(outBufferLen), uintptr(class)) + if r0 != 0 { + ntstatus = NTStatus(r0) + } + return +} + func NtQueryInformationProcess(proc Handle, procInfoClass int32, procInfo unsafe.Pointer, procInfoLen uint32, retLen *uint32) (ntstatus error) { r0, _, _ := syscall.SyscallN(procNtQueryInformationProcess.Addr(), uintptr(proc), uintptr(procInfoClass), uintptr(procInfo), uintptr(procInfoLen), uintptr(unsafe.Pointer(retLen))) if r0 != 0 { @@ -3756,6 +3783,14 @@ func NtQuerySystemInformation(sysInfoClass int32, sysInfo unsafe.Pointer, sysInf return } +func NtSetEaFile(handle Handle, iosb *IO_STATUS_BLOCK, inBuffer *byte, inBufferLen uint32) (ntstatus error) { + r0, _, _ := syscall.SyscallN(procNtSetEaFile.Addr(), uintptr(handle), uintptr(unsafe.Pointer(iosb)), uintptr(unsafe.Pointer(inBuffer)), uintptr(inBufferLen)) + if r0 != 0 { + ntstatus = NTStatus(r0) + } + return +} + func NtSetInformationFile(handle Handle, iosb *IO_STATUS_BLOCK, inBuffer *byte, inBufferLen uint32, class uint32) (ntstatus error) { r0, _, _ := syscall.SyscallN(procNtSetInformationFile.Addr(), uintptr(handle), uintptr(unsafe.Pointer(iosb)), uintptr(unsafe.Pointer(inBuffer)), uintptr(inBufferLen), uintptr(class)) if r0 != 0 { diff --git a/vendor/modules.txt b/vendor/modules.txt index 8308598f64640..4c819a838a23d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -406,8 +406,8 @@ github.com/containerd/containerd/api/types/runc/options github.com/containerd/containerd/api/types/runtimeoptions/v1 github.com/containerd/containerd/api/types/task github.com/containerd/containerd/api/types/transfer -# github.com/containerd/containerd/v2 v2.2.3 -## explicit; go 1.24.3 +# github.com/containerd/containerd/v2 v2.2.5 +## explicit; go 1.25.0 github.com/containerd/containerd/v2/client github.com/containerd/containerd/v2/cmd/containerd/server/config github.com/containerd/containerd/v2/core/containers @@ -1841,7 +1841,7 @@ go.yaml.in/yaml/v2 # go.yaml.in/yaml/v3 v3.0.4 ## explicit; go 1.16 go.yaml.in/yaml/v3 -# golang.org/x/crypto v0.51.0 +# golang.org/x/crypto v0.53.0 ## explicit; go 1.25.0 golang.org/x/crypto/argon2 golang.org/x/crypto/blake2b @@ -1871,7 +1871,7 @@ golang.org/x/crypto/ssh/internal/bcrypt_pbkdf ## explicit; go 1.25.0 golang.org/x/mod/semver golang.org/x/mod/sumdb/note -# golang.org/x/net v0.54.0 +# golang.org/x/net v0.55.0 ## explicit; go 1.25.0 golang.org/x/net/bpf golang.org/x/net/context @@ -1901,13 +1901,13 @@ golang.org/x/oauth2/google/internal/stsexchange golang.org/x/oauth2/internal golang.org/x/oauth2/jws golang.org/x/oauth2/jwt -# golang.org/x/sync v0.20.0 +# golang.org/x/sync v0.21.0 ## explicit; go 1.25.0 golang.org/x/sync/errgroup golang.org/x/sync/semaphore golang.org/x/sync/singleflight golang.org/x/sync/syncmap -# golang.org/x/sys v0.44.0 +# golang.org/x/sys v0.46.0 ## explicit; go 1.25.0 golang.org/x/sys/cpu golang.org/x/sys/plan9 @@ -1918,10 +1918,10 @@ golang.org/x/sys/windows/svc golang.org/x/sys/windows/svc/debug golang.org/x/sys/windows/svc/eventlog golang.org/x/sys/windows/svc/mgr -# golang.org/x/term v0.43.0 +# golang.org/x/term v0.44.0 ## explicit; go 1.25.0 golang.org/x/term -# golang.org/x/text v0.37.0 +# golang.org/x/text v0.38.0 ## explicit; go 1.25.0 golang.org/x/text/encoding golang.org/x/text/encoding/internal