diff --git a/Makefile b/Makefile index 9ce7c97..efbb9e7 100644 --- a/Makefile +++ b/Makefile @@ -14,3 +14,4 @@ FUZZTIME ?= 30s fuzz: go test -fuzz=FuzzParseSRI -fuzztime=$(FUZZTIME) . go test -fuzz=FuzzSRIRoundTrip -fuzztime=$(FUZZTIME) . + go test -fuzz=FuzzParseChecksum -fuzztime=$(FUZZTIME) . diff --git a/README.md b/README.md index d5241b8..191934a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # integrity -Parse Subresource Integrity metadata and verify package streams while they are read. The module supports SHA-256, SHA-384, and SHA-512 using only the Go standard library. +Parse Subresource Integrity metadata and registry checksums, and verify package streams while they are read. The module supports SHA-256, SHA-384, and SHA-512 using only the Go standard library, and MD5 and SHA-1 for verifying checksums registries already publish. ## Install @@ -75,6 +75,32 @@ SRI verification uses the strongest supported algorithm in the metadata list. An The caller must close the source and keep copied bytes private until verification succeeds. Write cache entries to a temporary file or object, then commit them after `Verify` returns nil. Reporting failures and cache policy remain with the caller. +## Verify a registry checksum + +Many registries publish a single hexadecimal checksum rather than SRI, sometimes in MD5 or SHA-1: npm's `dist.shasum`, Packagist's `dist.shasum`, Conda's `md5`, or a Vagrant box's `checksum_type` and `checksum`. `ParseChecksum` parses an algorithm-prefixed value such as `sha256-` or `sha1:`. `ParseHex` takes the algorithm separately. `Result.VerifyDigest` checks a completed stream against that one exact digest. + +```go +expected, err := integrity.ParseHex(integrity.SHA1, shasum) +if err != nil { + return err +} + +reader, err := integrity.NewReader(response.Body, expected.Algorithm()) +if err != nil { + return err +} +if _, err := io.Copy(temporary, reader); err != nil { + return err +} +if err := reader.Result().VerifyDigest(expected); err != nil { + return err +} +``` + +MD5 and SHA-1 are compatibility checks for metadata a publisher already supplied. Both are broken for collision resistance, so prefer a stronger checksum whenever the registry offers one, and never choose them for new integrity metadata. They are not SRI algorithms: `ParseSRI` rejects them, `Digest.SRI` returns an empty string for them, `FormatSRI` leaves them out, and `Result.Verify` refuses an SRI list containing them. `Digest.Checksum` gives the canonical `-` form for any algorithm. + +SRI verification ranks algorithms by an explicit strength order (SHA-512, then SHA-384, then SHA-256), not by the order of the `Algorithm` constants. + ## Development Run the tests and race detector: @@ -94,6 +120,7 @@ Run each fuzz target: ```bash go test -fuzz=FuzzParseSRI -fuzztime=30s go test -fuzz=FuzzSRIRoundTrip -fuzztime=30s +go test -fuzz=FuzzParseChecksum -fuzztime=30s ``` Run the benchmarks: diff --git a/algorithm.go b/algorithm.go index 26fba4a..c5e047b 100644 --- a/algorithm.go +++ b/algorithm.go @@ -2,6 +2,7 @@ package integrity import ( "fmt" + "slices" "strconv" "strings" ) @@ -16,15 +17,26 @@ const ( SHA384 // SHA512 identifies SHA-512 digests. SHA512 + // MD5 identifies MD5 digests. It is a compatibility check for checksums a + // publisher already supplied and is not a Subresource Integrity + // algorithm. Do not choose it for new integrity metadata. + MD5 + // SHA1 identifies SHA-1 digests. It is a compatibility check for + // checksums a publisher already supplied and is not a Subresource + // Integrity algorithm. Do not choose it for new integrity metadata. + SHA1 ) const ( + md5Size = 16 + sha1Size = 20 sha256Size = 32 sha384Size = 48 sha512Size = 64 ) -// String returns the lower-case SRI name for the algorithm. +// String returns the lower-case name for the algorithm, as SRI and +// algorithm-prefixed checksums spell it. func (a Algorithm) String() string { switch a { case SHA256: @@ -33,11 +45,38 @@ func (a Algorithm) String() string { return "sha384" case SHA512: return "sha512" + case MD5: + return "md5" + case SHA1: + return "sha1" default: return "Algorithm(" + strconv.FormatUint(uint64(a), 10) + ")" } } +// sri reports whether the algorithm may appear in Subresource Integrity +// metadata. +func (a Algorithm) sri() bool { + switch a { + case SHA256, SHA384, SHA512: + return true + default: + return false + } +} + +// byStrength lists the algorithms from weakest to strongest. SRI +// verification ranks by it to pick the strongest algorithm, rather than by +// the order of the constants, which reflects only when each was added. +var byStrength = []Algorithm{MD5, SHA1, SHA256, SHA384, SHA512} + +// strength ranks an algorithm by its place in byStrength, from 1 for the +// weakest; it is 0 for an unsupported algorithm. +func (a Algorithm) strength() int { + return slices.Index(byStrength, a) + 1 +} + +// parseAlgorithm parses any supported algorithm name. func parseAlgorithm(value string) (Algorithm, error) { if len(value) > len("sha512") { return 0, fmt.Errorf("unsupported algorithm") @@ -49,13 +88,33 @@ func parseAlgorithm(value string) (Algorithm, error) { return SHA384, nil case "sha512": return SHA512, nil + case "md5": + return MD5, nil + case "sha1": + return SHA1, nil default: return 0, fmt.Errorf("unsupported algorithm %q", value) } } +// parseSRIAlgorithm parses the name of an algorithm SRI metadata may use. +func parseSRIAlgorithm(value string) (Algorithm, error) { + algorithm, err := parseAlgorithm(value) + if err != nil { + return 0, err + } + if !algorithm.sri() { + return 0, fmt.Errorf("%s is not a Subresource Integrity algorithm", algorithm) + } + return algorithm, nil +} + func digestSize(algorithm Algorithm) (int, error) { switch algorithm { + case MD5: + return md5Size, nil + case SHA1: + return sha1Size, nil case SHA256: return sha256Size, nil case SHA384: diff --git a/checksum_test.go b/checksum_test.go new file mode 100644 index 0000000..5af1da8 --- /dev/null +++ b/checksum_test.go @@ -0,0 +1,238 @@ +package integrity + +import ( + "crypto/md5" //nolint:gosec // MD5 is tested as a compatibility checksum + "crypto/sha1" //nolint:gosec // SHA-1 is tested as a compatibility checksum + "encoding/base64" + "encoding/hex" + "errors" + "strings" + "testing" +) + +func weakDigestBytes(algorithm Algorithm, value string) []byte { + switch algorithm { + case MD5: + sum := md5.Sum([]byte(value)) //nolint:gosec // compatibility checksum + return sum[:] + case SHA1: + sum := sha1.Sum([]byte(value)) //nolint:gosec // compatibility checksum + return sum[:] + default: + return digestBytes(algorithm, value) + } +} + +func checksumHex(algorithm Algorithm, value string) string { + return hex.EncodeToString(weakDigestBytes(algorithm, value)) +} + +var allAlgorithms = []Algorithm{MD5, SHA1, SHA256, SHA384, SHA512} + +func TestAlgorithmNames(t *testing.T) { + for algorithm, want := range map[Algorithm]string{MD5: "md5", SHA1: "sha1", SHA256: "sha256", SHA384: "sha384", SHA512: "sha512"} { + if got := algorithm.String(); got != want { + t.Errorf("String = %q, want %q", got, want) + } + } +} + +func TestParseHexWeakAlgorithms(t *testing.T) { + for _, algorithm := range []Algorithm{MD5, SHA1} { + t.Run(algorithm.String(), func(t *testing.T) { + value := checksumHex(algorithm, "hello") + digest, err := ParseHex(algorithm, strings.ToUpper(value)) + if err != nil { + t.Fatalf("ParseHex: %v", err) + } + if digest.Algorithm() != algorithm || digest.Hex() != value { + t.Errorf("digest = %s %s, want %s %s", digest.Algorithm(), digest.Hex(), algorithm, value) + } + if got, want := digest.Checksum(), algorithm.String()+"-"+value; got != want { + t.Errorf("Checksum = %q, want %q", got, want) + } + }) + } +} + +func TestParseHexWeakDigestLength(t *testing.T) { + for algorithm, value := range map[Algorithm]string{ + MD5: checksumHex(SHA1, "hello"), + SHA1: checksumHex(MD5, "hello"), + } { + if _, err := ParseHex(algorithm, value); err == nil { + t.Errorf("ParseHex(%s, %d hex characters) succeeded, want a length error", algorithm, len(value)) + } + } +} + +func TestParseChecksum(t *testing.T) { + for _, algorithm := range allAlgorithms { + t.Run(algorithm.String(), func(t *testing.T) { + value := checksumHex(algorithm, "hello") + canonical := algorithm.String() + "-" + value + for _, input := range []string{ + canonical, + algorithm.String() + ":" + value, + strings.ToUpper(algorithm.String()) + "-" + strings.ToUpper(value), + " " + canonical + "\n", + } { + digest, err := ParseChecksum(input) + if err != nil { + t.Fatalf("ParseChecksum(%q): %v", input, err) + } + if digest.Algorithm() != algorithm || digest.Checksum() != canonical { + t.Errorf("ParseChecksum(%q) = %s, want %s", input, digest.Checksum(), canonical) + } + } + }) + } +} + +func TestParseChecksumRejectsMalformedValues(t *testing.T) { + sha1Hex := checksumHex(SHA1, "hello") + for _, value := range []string{ + "", + sha1Hex, + "sha1-", + "sha1-" + sha1Hex[:len(sha1Hex)-2], + "sha1-" + sha1Hex + "00", + "sha1-" + strings.Repeat("zz", len(sha1Hex)/2), + "sha1-" + base64.StdEncoding.EncodeToString(weakDigestBytes(SHA1, "hello")), + "crc32-00000000", + "sha1-" + sha1Hex + " sha1-" + sha1Hex, + } { + if _, err := ParseChecksum(value); err == nil { + t.Errorf("ParseChecksum(%q) succeeded, want an error", value) + } + } +} + +// TestParseSRIRejectsWeakAlgorithms keeps MD5 and SHA-1 out of SRI, even as +// valid base64 digests of the right length. +func TestParseSRIRejectsWeakAlgorithms(t *testing.T) { + for _, algorithm := range []Algorithm{MD5, SHA1} { + entry := algorithm.String() + "-" + base64.StdEncoding.EncodeToString(weakDigestBytes(algorithm, "hello")) + for _, value := range []string{entry, sriEntry(SHA512, "hello") + " " + entry} { + if _, err := ParseSRI(value); err == nil { + t.Errorf("ParseSRI(%q) succeeded, want an error", value) + } + } + } +} + +func TestWeakDigestsHaveNoSRIEncoding(t *testing.T) { + sha1Digest, err := ParseHex(SHA1, checksumHex(SHA1, "hello")) + if err != nil { + t.Fatal(err) + } + if got := sha1Digest.SRI(); got != "" { + t.Errorf("SHA-1 SRI = %q, want none", got) + } + strong, err := ParseSRI(sriEntry(SHA256, "hello")) + if err != nil { + t.Fatal(err) + } + if got, want := FormatSRI(append(SRI{sha1Digest}, strong...)), sriEntry(SHA256, "hello"); got != want { + t.Errorf("FormatSRI = %q, want only the SRI entry %q", got, want) + } +} + +func TestReaderCalculatesWeakAlgorithms(t *testing.T) { + result := readResult(t, "hello", MD5, SHA1, SHA256) + if len(result.Digests) != 3 { + t.Fatalf("calculated %d digests, want 3", len(result.Digests)) + } + for _, digest := range result.Digests { + if got, want := digest.Hex(), checksumHex(digest.Algorithm(), "hello"); got != want { + t.Errorf("%s = %s, want %s", digest.Algorithm(), got, want) + } + } +} + +func TestResultVerifyDigest(t *testing.T) { + for _, algorithm := range allAlgorithms { + t.Run(algorithm.String(), func(t *testing.T) { + matching, err := ParseHex(algorithm, checksumHex(algorithm, "hello")) + if err != nil { + t.Fatal(err) + } + other, err := ParseHex(algorithm, checksumHex(algorithm, "other")) + if err != nil { + t.Fatal(err) + } + + result := readResult(t, "hello", algorithm) + if err := result.VerifyDigest(matching); err != nil { + t.Errorf("matching digest: %v", err) + } + if err := result.VerifyDigest(other); err == nil { + t.Error("mismatched digest verified") + } + }) + } +} + +func TestResultVerifyDigestRequiresCompleteStream(t *testing.T) { + for _, algorithm := range []Algorithm{MD5, SHA1} { + reader, err := NewReader(strings.NewReader("hello"), algorithm) + if err != nil { + t.Fatal(err) + } + if _, err := reader.Read(make([]byte, 2)); err != nil { + t.Fatal(err) + } + expected, err := ParseHex(algorithm, checksumHex(algorithm, "hello")) + if err != nil { + t.Fatal(err) + } + var incomplete *ErrIncomplete + if err := reader.Result().VerifyDigest(expected); !errors.As(err, &incomplete) { + t.Errorf("%s: VerifyDigest after a partial read = %v, want ErrIncomplete", algorithm, err) + } + } +} + +func TestResultVerifyDigestRequiresCalculatedAlgorithm(t *testing.T) { + expected, err := ParseHex(SHA1, checksumHex(SHA1, "hello")) + if err != nil { + t.Fatal(err) + } + if err := readResult(t, "hello", SHA256).VerifyDigest(expected); err == nil { + t.Error("verified a SHA-1 digest that was not calculated") + } + if err := readResult(t, "hello", SHA256).VerifyDigest(Digest{}); err == nil { + t.Error("verified the zero Digest") + } +} + +// TestResultVerifyRejectsWeakDigests keeps SRI verification to SRI +// algorithms, even when a weak digest is put into an SRI list by hand. +func TestResultVerifyRejectsWeakDigests(t *testing.T) { + sha1Digest, err := ParseHex(SHA1, checksumHex(SHA1, "hello")) + if err != nil { + t.Fatal(err) + } + strong, err := ParseSRI(sriEntry(SHA256, "hello")) + if err != nil { + t.Fatal(err) + } + result := readResult(t, "hello", SHA1, SHA256) + if err := result.Verify(SRI{sha1Digest}); err == nil { + t.Error("Verify accepted a SHA-1 digest as SRI") + } + if err := result.Verify(append(SRI{sha1Digest}, strong...)); err == nil { + t.Error("Verify accepted an SRI list holding a SHA-1 digest") + } +} + +// TestAlgorithmStrength checks the ranking SRI verification picks the +// strongest algorithm by, which does not follow the order of the constants. +func TestAlgorithmStrength(t *testing.T) { + for i := 1; i < len(allAlgorithms); i++ { + weaker, stronger := allAlgorithms[i-1], allAlgorithms[i] + if weaker.strength() >= stronger.strength() { + t.Errorf("%s ranks %d, not below %s at %d", weaker, weaker.strength(), stronger, stronger.strength()) + } + } +} diff --git a/digest.go b/digest.go index beb0852..5095ed7 100644 --- a/digest.go +++ b/digest.go @@ -5,11 +5,12 @@ import ( "encoding/base64" "encoding/hex" "fmt" + "strings" ) // Digest is an immutable algorithm and digest-byte pair. // -// Use ParseHex or ParseSRI to construct a Digest. Bytes returns a copy of the +// Use ParseHex, ParseChecksum or ParseSRI to construct a Digest. Bytes returns a copy of the // stored bytes. type Digest struct { algorithm Algorithm @@ -27,7 +28,9 @@ func newDigest(algorithm Algorithm, raw []byte) (Digest, error) { return Digest{algorithm: algorithm, bytes: append([]byte(nil), raw...)}, nil } -// ParseHex parses a hexadecimal digest for algorithm. +// ParseHex parses a hexadecimal digest for algorithm. It accepts every +// supported algorithm, MD5 and SHA-1 included, for callers that already know +// which algorithm a registry's checksum uses. func ParseHex(algorithm Algorithm, value string) (Digest, error) { want, err := digestSize(algorithm) if err != nil { @@ -48,6 +51,27 @@ func ParseHex(algorithm Algorithm, value string) (Digest, error) { return digest, nil } +// ParseChecksum parses a single algorithm-prefixed hexadecimal checksum, such +// as "sha256-" or "sha1:", the form many registries publish instead +// of SRI. It accepts every supported algorithm, MD5 and SHA-1 included, and +// surrounding whitespace. Verify the result with Result.VerifyDigest. +func ParseChecksum(value string) (Digest, error) { + value = strings.TrimSpace(value) + separator := strings.IndexAny(value, "-:") + if separator < 0 { + return Digest{}, fmt.Errorf("parse checksum: missing algorithm prefix") + } + algorithm, err := parseAlgorithm(value[:separator]) + if err != nil { + return Digest{}, fmt.Errorf("parse checksum: %w", err) + } + digest, err := ParseHex(algorithm, value[separator+1:]) + if err != nil { + return Digest{}, fmt.Errorf("parse checksum: %w", err) + } + return digest, nil +} + // Algorithm returns the digest algorithm. func (d Digest) Algorithm() Algorithm { return d.algorithm @@ -63,11 +87,21 @@ func (d Digest) Hex() string { return hex.EncodeToString(d.bytes) } -// SRI returns the canonical Subresource Integrity encoding. +// SRI returns the canonical Subresource Integrity encoding, or "" for an +// MD5 or SHA-1 digest, which SRI does not allow. func (d Digest) SRI() string { + if !d.algorithm.sri() { + return "" + } return d.algorithm.String() + "-" + base64.StdEncoding.EncodeToString(d.bytes) } +// Checksum returns the canonical algorithm-prefixed hexadecimal encoding, +// which ParseChecksum accepts. +func (d Digest) Checksum() string { + return d.algorithm.String() + "-" + d.Hex() +} + // Equal reports whether two digests use the same algorithm and contain the // same raw bytes. The byte comparison takes constant time for equal-length // values. diff --git a/doc.go b/doc.go index 6ae16cf..e4026c4 100644 --- a/doc.go +++ b/doc.go @@ -2,5 +2,8 @@ // against cryptographic digests. // // It supports SHA-256, SHA-384, and SHA-512 without depending on HTTP, -// storage, or package-manager types. +// storage, or package-manager types. MD5 and SHA-1 are supported for +// verification only, to check checksums that registries already publish in +// those algorithms. They are never valid SRI and must not be chosen for new +// integrity metadata. package integrity diff --git a/fuzz_test.go b/fuzz_test.go index 3ecee5f..35e597b 100644 --- a/fuzz_test.go +++ b/fuzz_test.go @@ -45,3 +45,26 @@ func FuzzSRIRoundTrip(f *testing.F) { } }) } + +func FuzzParseChecksum(f *testing.F) { + f.Add("") + f.Add("sha1-") + f.Add("md5-" + checksumHex(MD5, "hello")) + f.Add("sha1:" + checksumHex(SHA1, "hello")) + f.Add("SHA512-" + checksumHex(SHA512, "hello")) + f.Add("sha256-" + checksumHex(SHA256, "hello") + " extra") + + f.Fuzz(func(t *testing.T, value string) { + digest, err := ParseChecksum(value) + if err != nil { + return + } + again, err := ParseChecksum(digest.Checksum()) + if err != nil { + t.Fatalf("ParseChecksum accepted %q but rejected its canonical form: %v", value, err) + } + if !digest.Equal(again) { + t.Fatalf("round trip of %q changed the digest", value) + } + }) +} diff --git a/reader.go b/reader.go index 0973968..c5a851d 100644 --- a/reader.go +++ b/reader.go @@ -1,6 +1,8 @@ package integrity import ( + "crypto/md5" //nolint:gosec // verification-only compatibility checksum + "crypto/sha1" //nolint:gosec // verification-only compatibility checksum "crypto/sha256" "crypto/sha512" "fmt" @@ -60,6 +62,10 @@ func NewReader(source io.Reader, algorithms ...Algorithm) (*Reader, error) { calculator = sha512.New384() case SHA512: calculator = sha512.New() + case MD5: + calculator = md5.New() //nolint:gosec // verification-only compatibility checksum + case SHA1: + calculator = sha1.New() //nolint:gosec // verification-only compatibility checksum default: return nil, fmt.Errorf("new integrity reader: unsupported algorithm %q", algorithm) } @@ -102,7 +108,8 @@ func (r *Reader) Result() Result { // Verify applies the W3C SRI matching rule to a completed result. Only the // strongest algorithm in expected is considered, and any digest using that -// algorithm may match. +// algorithm may match. expected must hold only SRI algorithms; use +// VerifyDigest for an MD5 or SHA-1 checksum. func (r Result) Verify(expected SRI) error { if !r.Complete { return &ErrIncomplete{} @@ -134,3 +141,26 @@ func (r Result) Verify(expected SRI) error { return fmt.Errorf("integrity mismatch: expected %s, calculated no %s digest", FormatSRI(expectedStrongest), algorithm) } + +// VerifyDigest checks a completed result against one exact digest, such as a +// registry checksum from ParseChecksum or ParseHex. It accepts every +// supported algorithm, MD5 and SHA-1 included, and requires that the reader +// calculated that algorithm. +func (r Result) VerifyDigest(expected Digest) error { + if !r.Complete { + return &ErrIncomplete{} + } + if _, err := digestSize(expected.Algorithm()); err != nil || len(expected.bytes) == 0 { + return fmt.Errorf("verify digest: no expected digest") + } + for _, calculated := range r.Digests { + if calculated.Algorithm() != expected.Algorithm() { + continue + } + if calculated.Equal(expected) { + return nil + } + return fmt.Errorf("integrity mismatch: expected %s, calculated %s", expected.Checksum(), calculated.Checksum()) + } + return fmt.Errorf("integrity mismatch: expected %s, calculated no %s digest", expected.Checksum(), expected.Algorithm()) +} diff --git a/sri.go b/sri.go index b03a477..ccbf5ed 100644 --- a/sri.go +++ b/sri.go @@ -25,7 +25,7 @@ func ParseSRI(value string) (SRI, error) { if !ok { return nil, fmt.Errorf("parse SRI entry %d: missing algorithm prefix", index+1) } - algorithm, err := parseAlgorithm(name) + algorithm, err := parseSRIAlgorithm(name) if err != nil { return nil, fmt.Errorf("parse SRI entry %d: %w", index+1, err) } @@ -63,11 +63,14 @@ func decodeBase64Digest(algorithm Algorithm, encoded string) ([]byte, error) { } // FormatSRI formats a metadata list with lower-case algorithm names, padded -// standard base64, and one space between entries. +// standard base64, and one space between entries. MD5 and SHA-1 digests have +// no SRI encoding and are left out. func FormatSRI(sri SRI) string { - entries := make([]string, len(sri)) - for i, digest := range sri { - entries[i] = digest.SRI() + entries := make([]string, 0, len(sri)) + for _, digest := range sri { + if entry := digest.SRI(); entry != "" { + entries = append(entries, entry) + } } return strings.Join(entries, " ") } @@ -77,15 +80,12 @@ func strongestAlgorithm(sri SRI) (Algorithm, error) { return 0, fmt.Errorf("verify integrity: no expected digests") } strongest := sri[0].Algorithm() - if _, err := digestSize(strongest); err != nil { - return 0, fmt.Errorf("verify integrity: %w", err) - } - for _, digest := range sri[1:] { + for _, digest := range sri { algorithm := digest.Algorithm() - if _, err := digestSize(algorithm); err != nil { - return 0, fmt.Errorf("verify integrity: %w", err) + if !algorithm.sri() { + return 0, fmt.Errorf("verify integrity: %s is not a Subresource Integrity algorithm", algorithm) } - if algorithm > strongest { + if algorithm.strength() > strongest.strength() { strongest = algorithm } }