diff --git a/CHANGELOG.md b/CHANGELOG.md index d1a90fda..84314b0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,8 +49,12 @@ because it turns other people's test suites red. land on one file on Windows or macOS, names a system cannot write, archives built to explode, contents that are not what the archive says. `--unpacked_limit` reports an archive that unpacks to more than you allow. - Safe ends with code 0 and anything found with 7, the whole report on the - error channel. 7z and RAR are recognised and not read yet. + Archives inside it are read too, in memory and up to 16 deep, and a trap + in any of them makes the whole unsafe, said with the way to it + (`lib 0.zip > lib 3.zip > ../x`). Safe ends with code 0 and anything found + with 7, the whole report on the error channel. A report lists 100 findings + of each kind and counts the rest. 7z and RAR are recognised and not read + yet, and one inside another archive is reported as not checked. - **Documents: the `rtf` format.** A Rich Text Format document at the exact size you ask for, read by LibreOffice, pandoc, the RichEdit control of Windows, striprtf and Apache Tika. It holds headings, paragraphs with bold diff --git a/README.md b/README.md index 5cd0aacb..78ba35b5 100644 --- a/README.md +++ b/README.md @@ -424,9 +424,15 @@ the checksum file is in. Files it does not list are not looked at, the same as `archive-scan` looks into a ZIP, TAR, TAR.GZ or gzip file without unpacking it and says whether unpacking it is safe: no entry written outside the folder, no link, no device, no two names landing on one file, no name a system cannot -write, nothing built to explode and nothing damaged. `--unpacked_limit 500mb` -also reports an archive that unpacks to more than that. Safe ends with code 0, -anything found with 7. +write, nothing built to explode and nothing damaged. Archives inside it - a +ZIP in a ZIP, a JAR in a WAR, a DOCX in a TAR - are read the same way, in +memory, up to 16 deep, and a trap in one of them is a trap in the whole: a +program that unpacks everything inside unpacks those too. An archive inside +larger than 256 MB, or than what is left of it while the archives around it +are held, or one this tool does not read, is reported as not checked. `--unpacked_limit 500mb` also reports an archive that unpacks to more +than that, counting the archives inside. Safe ends with code 0, anything found +with 7. A report lists 100 findings of each kind and counts the rest, and what +the tool holds in memory has a ceiling whatever the archive holds. ### `tfg damage` diff --git a/internal/guard/archivescan_test.go b/internal/guard/archivescan_test.go index abe27cf3..72506851 100644 --- a/internal/guard/archivescan_test.go +++ b/internal/guard/archivescan_test.go @@ -241,6 +241,9 @@ func TestTheArchiveScanEndsWithTheCodeOfWhatHappened(t *testing.T) { leaves := archiveFile(t, "leaves.zip", zipOf(t, zipPart{name: "../a.txt"})) sevenZip := archiveFile(t, "x.7z", append([]byte{'7', 'z', 0xBC, 0xAF, 0x27, 0x1C}, make([]byte, 32)...)) text := archiveFile(t, "notes.txt", []byte("not an archive")) + // Two bytes are all gzip and many other files share: the third, the one + // method gzip has, tells them apart. + gzipLike := archiveFile(t, "data.bin", append([]byte{0x1f, 0x8b, 0x00}, make([]byte, 32)...)) stopped, stop := context.WithCancel(context.Background()) stop() for _, c := range []struct { @@ -254,6 +257,7 @@ func TestTheArchiveScanEndsWithTheCodeOfWhatHappened(t *testing.T) { {"more than the limit", context.Background(), []string{clean, "--unpacked_limit", "0"}, cli.ExitVerify}, {"a limit that is not a size", context.Background(), []string{clean, "--unpacked_limit", "10xb"}, cli.ExitUsage}, {"a file that is not an archive", context.Background(), []string{text}, cli.ExitUsage}, + {"a file beginning as gzip does and not one", context.Background(), []string{gzipLike}, cli.ExitUsage}, {"an archive this tool does not read", context.Background(), []string{sevenZip}, cli.ExitUsage}, {"a directory", context.Background(), []string{filepath.Dir(clean)}, cli.ExitUsage}, {"no file", context.Background(), nil, cli.ExitUsage}, @@ -317,6 +321,11 @@ func FuzzArchiveScan(f *testing.F) { f.Add(tarOf(t, true, tarPart{name: "a", body: "abc"}, tarPart{name: "l", typ: '2', link: "/x"})) f.Add(tarOf(t, false, tarPart{name: "a", body: "abc"})) f.Add(handZip("first", "first", true)) + // Archives inside archives, so the reading of one held in memory is + // fuzzed too. + inner := zipOf(t, zipPart{name: "in.txt", body: "abc"}) + f.Add(zipOf(t, zipPart{name: "inner.zip", body: string(inner)}, zipPart{name: "t.tar", body: string(tarOf(t, false, tarPart{name: "z.zip", body: string(inner)}))})) + f.Add(gzipOf(t, inner)) d, err := tool.Get(archivescan.ID) if err != nil { f.Fatal(err) diff --git a/internal/guard/archivescaninside_test.go b/internal/guard/archivescaninside_test.go new file mode 100644 index 00000000..d4c7a10c --- /dev/null +++ b/internal/guard/archivescaninside_test.go @@ -0,0 +1,296 @@ +package guard + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "context" + "fmt" + "io" + "runtime" + "slices" + "strings" + "testing" + "unicode/utf8" + + "github.com/donislawdev/TestingFilesGenerator/internal/tool" + "github.com/donislawdev/TestingFilesGenerator/internal/tool/archivescan" +) + +// The archives inside an archive, and the bounds of what the scanner holds: +// a trap in an archive inside is a trap, the archives together can be built to +// explode, and nothing an archive holds makes the report, or what is held to +// read it, grow without a bound. docs/SKANER-W-GLAB-2026-10-19.md. + +// noteItems is every item of the note with this key in a result, as the +// command line prints it. +func noteItems(r tool.Result, id string) []string { + for _, n := range r.Notes { + if n.ID == id { + return n.Items + } + } + return nil +} + +// holdsText says whether one item of a list holds this text. +func holdsText(items []string, text string) bool { + return slices.ContainsFunc(items, func(item string) bool { return strings.Contains(item, text) }) +} + +// chainOf is an archive inside an archive, depth times, around one file. +func chainOf(t *testing.T, depth int) []byte { + t.Helper() + data := zipOf(t, zipPart{name: "bottom.txt", body: "hello"}) + for level := range depth { + data = zipOf(t, zipPart{name: fmt.Sprintf("level%02d.zip", level), body: string(data)}) + } + return data +} + +// storedZip is a ZIP of one entry stored as it is, so the archive is as large +// as what it holds. +func storedZip(t *testing.T, name string, data []byte) []byte { + t.Helper() + var out strings.Builder + z := zip.NewWriter(&out) + w, err := z.CreateHeader(&zip.FileHeader{Name: name, Method: zip.Store}) + if err == nil { + _, err = w.Write(data) + } + if err == nil { + err = z.Close() + } + if err != nil { + t.Fatal(err) + } + return []byte(out.String()) +} + +// bigInside is a gzip of a ZIP of 300 MiB of zeros: an archive inside larger +// than the scanner holds, whose size nothing says before it is read. +func bigInside(t *testing.T) []byte { + t.Helper() + var out strings.Builder + g := gzip.NewWriter(&out) + z := zip.NewWriter(g) + w, err := z.CreateHeader(&zip.FileHeader{Name: "zeros.bin", Method: zip.Store}) + if err != nil { + t.Fatal(err) + } + chunk := make([]byte, 1<<20) + for range 300 { + if _, err := w.Write(chunk); err != nil { + t.Fatal(err) + } + } + if err := z.Close(); err != nil { + t.Fatal(err) + } + if err := g.Close(); err != nil { + t.Fatal(err) + } + return []byte(out.String()) +} + +// roomInside is a gzip of a.zip, which holds 120 MiB of zeros and b.zip, which +// holds 100 MiB: b.zip fits in what the scanner holds, and not in what is +// left of it while a.zip is held. +func roomInside(t *testing.T) []byte { + t.Helper() + var out strings.Builder + g := gzip.NewWriter(&out) + a := zip.NewWriter(g) + zeros := func(w io.Writer, mib int) { + chunk := make([]byte, 1<<20) + for range mib { + if _, err := w.Write(chunk); err != nil { + t.Fatal(err) + } + } + } + filler, err := a.CreateHeader(&zip.FileHeader{Name: "filler.bin", Method: zip.Store}) + if err != nil { + t.Fatal(err) + } + zeros(filler, 120) + inner, err := a.CreateHeader(&zip.FileHeader{Name: "b.zip", Method: zip.Store}) + if err != nil { + t.Fatal(err) + } + bz := zip.NewWriter(inner) + w, err := bz.CreateHeader(&zip.FileHeader{Name: "zeros.bin", Method: zip.Store}) + if err != nil { + t.Fatal(err) + } + zeros(w, 100) + for _, c := range []io.Closer{bz, a, g} { + if err := c.Close(); err != nil { + t.Fatal(err) + } + } + return []byte(out.String()) +} + +// TestTheArchiveScanLooksIntoTheArchivesInside holds the scanner to reading +// every archive inside the one given, and to saying a trap in one with the way +// to it - in the notes both surfaces show, and in the report of the archive +// inside that --json carries. +func TestTheArchiveScanLooksIntoTheArchivesInside(t *testing.T) { + bad := zipOf(t, zipPart{name: "../evil.txt", body: "x"}) + emptyBzip2 := string([]byte{'B', 'Z', 'h', '9', 0x17, 0x72, 0x45, 0x38, 0x50, 0x90, 0, 0, 0, 0}) + // Bytes no two pieces of a megabyte share, so an archive read back from + // the wrong piece fails its checksum. + pattern := make([]byte, 3<<20+7) + for i := range pattern { + pattern[i] = byte(i % 251) + } + member, gz := gzipOf(t, make([]byte, 64<<20)), []byte(nil) + for range 10 { + gz = append(gz, member...) + } + for _, c := range []struct { + what, file string + data []byte + // note and item are a note the result has to hold, and words one + // of its items has to hold. + note, item string + safe bool + }{ + {what: "a trap in an archive inside", file: "inside.zip", data: zipOf(t, zipPart{name: "inner.zip", body: string(bad)}), + note: "leaves_folder", item: "inner.zip > ../evil.txt"}, + {what: "a trap two archives deep", file: "twice.zip", data: zipOf(t, zipPart{name: "a.tar", body: string(tarOf(t, false, + tarPart{name: "b.zip", body: string(bad)}))}), note: "leaves_folder", item: "a.tar > b.zip > ../evil.txt"}, + {what: "archives inside, deeper than the scanner looks", file: "chain.zip", data: chainOf(t, archivescan.MostDepth+2), + note: "unchecked", item: "deeper than this tool looks"}, + {what: "an archive inside larger than the scanner holds", file: "big.zip.gz", data: bigInside(t), + note: "unchecked", item: "larger than the"}, + {what: "an archive inside larger than what is left while the one around it is held", file: "room.zip.gz", data: roomInside(t), + note: "unchecked", item: "what is left of the"}, + {what: "an archive inside the scanner does not read", file: "bz.zip", data: zipOf(t, zipPart{name: "x.bz2", body: emptyBzip2}), + note: "unchecked", item: "bzip2 archive"}, + {what: "an archive inside held in several pieces", file: "pieces.zip", safe: true, note: "nested", item: "big.zip", + data: zipOf(t, zipPart{name: "big.zip", body: string(storedZip(t, "data.bin", pattern))})}, + {what: "text that begins with the letters of bzip2", file: "text.zip", data: zipOf(t, zipPart{name: "x.txt", body: "BZh9 is a word, not bzip2"}), + safe: true}, + // Two archives of 640 MiB of zeros each: neither is built to explode + // on its own, and together they are. + {what: "archives inside that explode only together", file: "together.zip", data: zipOf(t, zipPart{name: "a.gz", body: string(gz)}, + zipPart{name: "b.gz", body: string(gz)}), note: "bomb", item: "with the archives inside it"}, + } { + report, r := scanArchive(t, archiveFile(t, c.file, c.data), nil) + if report.Safe != c.safe || (r.Concluded == "safe") != c.safe { + t.Errorf("%s (%s): safe is %v and the conclusion %q, and it is %v", c.what, c.file, report.Safe, r.Concluded, c.safe) + } + if c.note != "" && !holdsText(noteItems(r, c.note), c.item) { + t.Errorf("%s (%s): the note %s says %q, and has to say %q", c.what, c.file, c.note, noteItems(r, c.note), c.item) + } + } +} + +// TestTheArchiveScanReportsAnArchiveInsideOnItsOwn holds --json to naming in +// found what is in the archive given, and carrying the archive inside that is +// not safe as a report of its own, with the names in it. +func TestTheArchiveScanReportsAnArchiveInsideOnItsOwn(t *testing.T) { + inner := zipOf(t, zipPart{name: "../evil.txt", body: "x"}) + report, _ := scanArchive(t, archiveFile(t, "outer.zip", zipOf(t, zipPart{name: "ok.txt"}, zipPart{name: "inner.zip", body: string(inner)})), nil) + if _, ok := report.Found["leaves_folder"]; ok || !slices.Equal(report.Found["nested"], []string{"inner.zip"}) { + t.Errorf("found is %v: it names what is in the archive given, so inner.zip and nothing about ../evil.txt", report.Found) + } + if len(report.Inside) != 1 || report.Inside[0].File != "inner.zip" || !slices.Equal(report.Inside[0].Found["leaves_folder"], []string{"../evil.txt"}) { + t.Errorf("inside is %+v, and has to carry inner.zip with ../evil.txt leaving the folder", report.Inside) + } + if report.Safe || report.Inside[0].Safe { + t.Errorf("an archive with a trap inside is not safe, and neither is the archive inside") + } +} + +// TestTheArchiveScanReportStaysTheSameSizeWhateverAnArchiveHolds holds the +// report to its bounds: a note lists so many findings and counts the rest, +// --json lists as many and says how many more, and a name is cut where a +// report stops showing it, with its length. +func TestTheArchiveScanReportStaysTheSameSizeWhateverAnArchiveHolds(t *testing.T) { + var parts []zipPart + for i := range archivescan.MostListed + 50 { + parts = append(parts, zipPart{name: fmt.Sprintf("q?%03d", i)}) + } + long := strings.Repeat("x", 2000) + // One byte, then characters of two: the cut at 1024 falls inside one. + polish := "a" + strings.Repeat(string(rune(0x105)), 600) + parts = append(parts, zipPart{name: long}) + report, r := scanArchive(t, archiveFile(t, "many.zip", zipOf(t, parts...)), nil) + listed := report.Found["unwritable"] + if len(listed) != archivescan.MostListed || report.More["unwritable"] != 51 { + t.Errorf("--json lists %d names that cannot be written and says %d more, and there are %d", len(listed), report.More["unwritable"], len(parts)) + } + items := noteItems(r, "unwritable") + if len(items) != archivescan.MostListed+1 || items[len(items)-1] != "and 51 more" { + t.Errorf("the note has %d items ending %q, and has to list %d and count the rest", len(items), items[len(items)-1], archivescan.MostListed) + } + cut, _ := scanArchive(t, archiveFile(t, "long.zip", zipOf(t, zipPart{name: long})), nil) + if got := cut.Found["unwritable"]; len(got) != 1 || len(got[0]) != archivescan.MostShownName { + t.Errorf("a name of 2000 bytes is listed as %d bytes, and is cut at %d", len(got[0]), archivescan.MostShownName) + } + _, r = scanArchive(t, archiveFile(t, "long.zip", zipOf(t, zipPart{name: long})), nil) + if !holdsText(noteItems(r, "unwritable"), "(2 000 B in all)") { + t.Errorf("the note shows %q, and has to say the name is cut and how long it is", noteItems(r, "unwritable")) + } + whole, _ := scanArchive(t, archiveFile(t, "polish.zip", zipOf(t, zipPart{name: polish})), nil) + if got := whole.Found["unwritable"]; len(got) != 1 || len(got[0]) != archivescan.MostShownName-1 || !utf8.ValidString(got[0]) { + t.Errorf("a name cut inside a character is listed as %d bytes, valid UTF-8 %v - the cut goes back to where the character starts", len(got[0]), utf8.ValidString(got[0])) + } +} + +// TestTheArchiveScanHoldsNoMoreOfANameThanTheName measures the heap while a +// TAR.GZ is read whose every entry is named in a header of a megabyte, and is +// an archive with a trap in it. archive/tar hands such a name back as a slice +// of the whole header, and holding the slice - to compare names, or as the +// name of an archive inside kept for the report - held every header: 200 of +// them were 200 MB, measured at 1.3 GB on 900 +// (docs/SKANER-W-GLAB-2026-10-19.md §1). +func TestTheArchiveScanHoldsNoMoreOfANameThanTheName(t *testing.T) { + var buf strings.Builder + g := gzip.NewWriter(&buf) + tw := tar.NewWriter(g) + pad := strings.Repeat("p", 1_000_000) + bad := zipOf(t, zipPart{name: "../evil.txt", body: "x"}) + for i := range 200 { + name := fmt.Sprintf("f%03d.zip", i) + if err := tw.WriteHeader(&tar.Header{Name: name, Typeflag: tar.TypeReg, Mode: 0o644, Size: int64(len(bad)), Format: tar.FormatPAX, + PAXRecords: map[string]string{"comment": pad, "path": name}}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(bad); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := g.Close(); err != nil { + t.Fatal(err) + } + path := archiveFile(t, "padded.tar.gz", []byte(buf.String())) + d, err := tool.Get(archivescan.ID) + if err != nil { + t.Fatal(err) + } + runtime.GC() + var before, now runtime.MemStats + runtime.ReadMemStats(&before) + most, calls := uint64(0), 0 + sample := func(int64, int64) { + if calls++; calls%4 == 0 { + runtime.GC() + runtime.ReadMemStats(&now) + most = max(most, now.HeapAlloc) + } + } + r, err := d.Start(context.Background(), tool.Request{Inputs: map[string]string{archivescan.InputArchive: path}}, sample) + if err != nil || r.Concluded != "unsafe" || calls < 4 { + t.Fatalf("the padded archive: %v, concluded %q, %d reports of progress", err, r.Concluded, calls) + } + if grew := int64(most) - int64(before.HeapAlloc); grew > 64<<20 { + t.Errorf("reading 200 names of 8 bytes held %d MB at the most - a name kept holds its whole header", grew>>20) + } +} diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index 4ec7e9f3..e4db14f5 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -15140,15 +15140,15 @@ ( "archive-scan calls an archive unsafe for holding another archive", "internal/tool/archivescan/findings.go", - "{tool.Note{ID: noteNested, Says: \"Archives inside it, which were not opened:\"}, true},", - "{tool.Note{ID: noteNested, Says: \"Archives inside it, which were not opened:\"}, false},", + "{tool.Note{ID: noteNested, Says: \"Archives inside it, read as well:\"}, true},", + "{tool.Note{ID: noteNested, Says: \"Archives inside it, read as well:\"}, false},", "TestTheArchiveScanFindsEveryTrapItNames", ), ( "archive-scan never compares what an archive unpacks to with the limit given", "internal/tool/archivescan/findings.go", - "if s.limit >= 0 && s.unpacked > uint64(s.limit) {", - "if s.limit < 0 && s.unpacked > uint64(s.limit) {", + "if limit := s.tree.limit; limit >= 0 && all > uint64(limit) {", + "if limit := s.tree.limit; limit < 0 && all > uint64(limit) {", "TestTheArchiveScanFindsEveryTrapItNames", ), ( @@ -15235,4 +15235,123 @@ "\t_ = src\n", "TestTheArchiveScanFindsEveryTrapItNames", ), + ( + "archive-scan never finds the archives inside one built to explode together", + "internal/tool/archivescan/inside.go", + "\tif t.unpacked == r.unpacked || t.unpacked < BombBytes || t.unpacked <= uint64(r.size)*BombRatio {\n", + "\tif true || t.unpacked == r.unpacked || t.unpacked < BombBytes || t.unpacked <= uint64(r.size)*BombRatio {\n", + "TestTheArchiveScanLooksIntoTheArchivesInside", + ), + ( + "archive-scan looks into archives inside a ZIP however deep they go", + "internal/tool/archivescan/zip.go", + "\tz.hold.start(z.tree, e.unpacked, look && z.depth < MostDepth)\n", + "\tz.hold.start(z.tree, e.unpacked, look)\n", + "TestTheArchiveScanLooksIntoTheArchivesInside", + ), + ( + "archive-scan holds an archive inside however large it grows", + "internal/tool/archivescan/inside.go", + "\tif want <= 0 || want > int64(MostInside-h.t.held) {\n", + "\tif want <= 0 {\n", + "TestTheArchiveScanLooksIntoTheArchivesInside", + ), + ( + "archive-scan passes over an archive inside it does not read, as if nothing were there", + "internal/tool/archivescan/inside.go", + "\tcase !readable(h.kind):\n", + "\tcase false:\n", + "TestTheArchiveScanLooksIntoTheArchivesInside", + ), + ( + "archive-scan keeps what it finds in an archive inside out of the notes", + "internal/tool/archivescan/inside.go", + "\t\t\ts.tree.add(s.found.deep, s.found.deepMore, d.ID, deeper(name, f))\n", + "\t\t\t_ = f\n", + "TestTheArchiveScanLooksIntoTheArchivesInside", + ), + ( + "archive-scan calls an archive safe with an archive inside that is not", + "internal/tool/archivescan/inside.go", + "\ts.nest.unsafe = true\n", + "\ts.nest.unsafe = false\n", + "TestTheArchiveScanReportsAnArchiveInsideOnItsOwn", + ), + ( + "archive-scan reads an archive held in pieces from the first piece only", + "internal/tool/archivescan/inside.go", + "\t\tc := copy(p[n:], h.parts[off/piece][off%piece:])\n", + "\t\tc := copy(p[n:], h.parts[0][off%piece:])\n", + "TestTheArchiveScanLooksIntoTheArchivesInside", + ), + ( + "archive-scan reads text that begins with the letters of bzip2 as bzip2", + "internal/tool/archivescan/sniff.go", + "\treturn bytes.Equal(magic, []byte{0x31, 0x41, 0x59, 0x26, 0x53, 0x59}) || bytes.Equal(magic, []byte{0x17, 0x72, 0x45, 0x38, 0x50, 0x90})\n", + "\treturn len(magic) >= 0\n", + "TestTheArchiveScanLooksIntoTheArchivesInside", + ), + ( + "archive-scan takes any file beginning with the two bytes of gzip for gzip", + "internal/tool/archivescan/sniff.go", + "gzipMagic = []byte{0x1f, 0x8b, 0x08}", + "gzipMagic = []byte{0x1f, 0x8b}", + "TestTheArchiveScanEndsWithTheCodeOfWhatHappened", + ), + ( + "archive-scan lists every finding in a note, however many there are", + "internal/tool/archivescan/report.go", + "\tif len(list) >= MostListed || t.listed >= MostListedBytes {\n", + "\tif t.listed >= MostListedBytes {\n", + "TestTheArchiveScanReportStaysTheSameSizeWhateverAnArchiveHolds", + ), + ( + "archive-scan lists a name whole, however long it is", + "internal/tool/archivescan/report.go", + "\tn := min(len(name), MostShownName)\n", + "\tn := len(name)\n", + "TestTheArchiveScanReportStaysTheSameSizeWhateverAnArchiveHolds", + ), + ( + "archive-scan stops a note at its limit without saying how many more there are", + "internal/tool/archivescan/report.go", + "\tif more > 0 {\n\t\twords := core.Says(\"archivescan.More\"", + "\tif false {\n\t\twords := core.Says(\"archivescan.More\"", + "TestTheArchiveScanReportStaysTheSameSizeWhateverAnArchiveHolds", + ), + ( + "archive-scan leaves the archive inside out of what --json reports", + "internal/tool/archivescan/report.go", + "\t\tr.Inside = append(r.Inside, c.report())\n", + "\t\t_ = c\n", + "TestTheArchiveScanReportsAnArchiveInsideOnItsOwn", + ), + ( + "archive-scan holds a name set in a PAX header as a slice of the whole header", + "internal/tool/archivescan/names.go", + "\tname = strings.Clone(name)\n", + "\tname = name + \"\"\n", + "TestTheArchiveScanHoldsNoMoreOfANameThanTheName", + ), + ( + "archive-scan keeps the name of an archive inside as a slice of a PAX header", + "internal/tool/archivescan/inside.go", + "\tc := blank(cutName(name), h.size, s.tree)\n", + "\tc := blank(name, h.size, s.tree)\n", + "TestTheArchiveScanHoldsNoMoreOfANameThanTheName", + ), + ( + "archive-scan cuts a name inside a character", + "internal/tool/archivescan/report.go", + "\tfor back := 1; back < utf8.UTFMax && n < len(name) && !utf8.RuneStart(name[n]); back++ {\n", + "\tfor back := 1; false && back < utf8.UTFMax && n < len(name) && !utf8.RuneStart(name[n]); back++ {\n", + "TestTheArchiveScanReportStaysTheSameSizeWhateverAnArchiveHolds", + ), + ( + "archive-scan calls an archive inside too large when only the room left is too small", + "internal/tool/archivescan/inside.go", + "\tcase h.need > MostInside:\n", + "\tcase h.need > 0:\n", + "TestTheArchiveScanLooksIntoTheArchivesInside", + ), ] diff --git a/internal/gui/text/locale/registry/en.json b/internal/gui/text/locale/registry/en.json index 9547f5f0..d135a3d9 100644 --- a/internal/gui/text/locale/registry/en.json +++ b/internal/gui/text/locale/registry/en.json @@ -1941,8 +1941,8 @@ }, "Detail.tool/archive-scan.unpacked_limit": { "description": "The sentence under the unpacked_limit setting of the archive-scan tool, after what it takes.", - "hash": "sha256-db62592b44cd", - "other": "The most the archive may unpack to, all its files together - for a server or a disk that takes no more. Left empty, only an archive built to explode is reported: 1 GB or more, and more than 100 times its own size." + "hash": "sha256-2d979a19400d", + "other": "The most the archive may unpack to, all its files together with those of the archives inside it - for a server or a disk that takes no more. Left empty, only an archive built to explode is reported: 1 GB or more, and more than 100 times its own size." }, "Detail.tool/checksum-write.algorithm": { "description": "The sentence under the algorithm setting of the checksum-write tool, after what it takes.", @@ -2926,8 +2926,8 @@ }, "Tool.archive-scan.Note.nested": { "description": "A line of the result of the archive-scan tool, ending in a colon - what follows it is a list of names or one number.", - "hash": "sha256-d256c3caa62f", - "other": "Archives inside it, which were not opened:" + "hash": "sha256-bafd9d1c251d", + "other": "Archives inside it, read as well:" }, "Tool.archive-scan.Note.over_limit": { "description": "A line of the result of the archive-scan tool, ending in a colon - what follows it is a list of names or one number.", @@ -2969,6 +2969,11 @@ "hash": "sha256-75d57d9929a3", "other": "Unpacked:" }, + "Tool.archive-scan.Note.unpacked_all": { + "description": "A line of the result of the archive-scan tool, ending in a colon - what follows it is a list of names or one number.", + "hash": "sha256-5bfde17d9bd9", + "other": "Unpacked, with the archives inside it:" + }, "Tool.archive-scan.Note.unwritable": { "description": "A line of the result of the archive-scan tool, ending in a colon - what follows it is a list of names or one number.", "hash": "sha256-40ba7750552a", diff --git a/internal/gui/text/locale/registry/pl.json b/internal/gui/text/locale/registry/pl.json index eb8f02c7..e4119284 100644 --- a/internal/gui/text/locale/registry/pl.json +++ b/internal/gui/text/locale/registry/pl.json @@ -387,7 +387,7 @@ "Detail.preset/upload-validation.deny": { "hash": "sha256-f9e5c633d218", "other": "Jakie rozszerzenia twój formularz ma odrzucać. Rozszerzenie, dla którego ta wersja nie ma formatu, i tak dostaje plik o tej nazwie, ze zwykłym tekstem w środku." }, "Detail.preset/upload-validation.far-over": { "hash": "sha256-6a0093c6d7a0", "other": "Jak daleko ponad limit sięga jeden duży plik. Wyłącz tam, gdzie zapis kilkukrotności limitu nie jest wart miejsca na dysku." }, "Detail.preset/upload-validation.limit": { "hash": "sha256-be929d76067d", "other": "Limit rozmiaru, który deklaruje twój formularz przesyłania. Ten zestaw robi jeden krok po obu stronach - żeby dostać plik w każdej odległości, uruchom preset size-boundaries." }, - "Detail.tool/archive-scan.unpacked_limit": { "hash": "sha256-db62592b44cd", "other": "Najwięcej, ile archiwum może zająć po rozpakowaniu, wszystkie pliki razem - dla serwera albo dysku, który więcej nie przyjmie. Puste zgłasza tylko archiwum zbudowane, żeby eksplodować: 1 GB albo więcej i ponad 100 razy więcej niż ono samo." }, + "Detail.tool/archive-scan.unpacked_limit": { "hash": "sha256-2d979a19400d", "other": "Najwięcej, ile archiwum może zająć po rozpakowaniu, wszystkie pliki razem z plikami archiwów w środku - dla serwera albo dysku, który więcej nie przyjmie. Puste zgłasza tylko archiwum zbudowane, żeby eksplodować: 1 GB albo więcej i ponad 100 razy więcej niż ono samo." }, "Detail.tool/checksum-write.algorithm": { "hash": "sha256-bc3d55ed0c3c", "other": "Który skrót zapisać. sha256 to ten, którego używa sha256sum i większość stron z wydaniami." }, "Detail.tool/checksum.algorithm": { "hash": "sha256-b2c6c5ddeaa1", "other": "Które sumy kontrolne policzyć: md5, sha1, sha256, sha512 albo crc32. Tylko sha256 i sha512 nadal pokazują, że nikt celowo nie zmienił pliku. crc32 to ta, której używają ZIP i PNG, a nie ta, którą wypisuje cksum." }, "Detail.tool/checksum.expected": { "hash": "sha256-d2a5105f8783", "other": "Suma kontrolna, jaką plik powinien mieć. Algorytm rozpoznaje się po jej długości i liczy go, nawet jeśli nie wybrano go wyżej." }, @@ -584,7 +584,7 @@ "Tool.archive-scan.Note.leaves_folder": { "hash": "sha256-5d69160ea44c", "other": "Wpisy, które zostałyby zapisane poza folderem:" }, "Tool.archive-scan.Note.links": { "hash": "sha256-ec9bee24c930", "other": "Dowiązania, przez które może zostać zapisany późniejszy wpis:" }, "Tool.archive-scan.Note.longer_than_said": { "hash": "sha256-f68f6d40c1ff", "other": "Wpisy zawierające więcej, niż podaje archiwum:" }, - "Tool.archive-scan.Note.nested": { "hash": "sha256-d256c3caa62f", "other": "Archiwa w środku, których nie otwierano:" }, + "Tool.archive-scan.Note.nested": { "hash": "sha256-bafd9d1c251d", "other": "Archiwa w środku, również przeczytane:" }, "Tool.archive-scan.Note.over_limit": { "hash": "sha256-6179368adab1", "other": "Po rozpakowaniu więcej niż podany limit:" }, "Tool.archive-scan.Note.overlapping": { "hash": "sha256-cd5cedc1af84", "other": "Wpisy dzielące bajty z innym, co robi tylko archiwum zbudowane, żeby eksplodować:" }, "Tool.archive-scan.Note.size": { "hash": "sha256-c987f668d963", "other": "Rozmiar:" }, @@ -593,6 +593,7 @@ "Tool.archive-scan.Note.two_names": { "hash": "sha256-672e79c37984", "other": "Wpisy nazwane inaczej w spisie zawartości, a inaczej przy samym wpisie:" }, "Tool.archive-scan.Note.unchecked": { "hash": "sha256-94238136c3ba", "other": "Wpisy, których zawartości nie dało się sprawdzić:" }, "Tool.archive-scan.Note.unpacked": { "hash": "sha256-75d57d9929a3", "other": "Po rozpakowaniu:" }, + "Tool.archive-scan.Note.unpacked_all": { "hash": "sha256-5bfde17d9bd9", "other": "Po rozpakowaniu, razem z archiwami w środku:" }, "Tool.archive-scan.Note.unwritable": { "hash": "sha256-40ba7750552a", "other": "Nazwy, których Windows, macOS albo Linux nie zapisze w tej postaci:" }, "Tool.archive-scan.Question": { "hash": "sha256-d91795de08fb", "other": "Czy to archiwum można bezpiecznie rozpakować?" }, "Tool.checksum-check.Detail": { "hash": "sha256-3480ee8eb9ae", "other": "Czyta plik sum kontrolnych - SHA256SUMS, MD5SUMS albo w postaci ze znacznikami, którą pisze shasum - i sprawdza każdy wymieniony w nim plik w folderze, w którym ten plik leży. Plików, których nie wymienia, nie ogląda, tak samo jak sha256sum -c." }, diff --git a/internal/gui/text/locale/said/en.json b/internal/gui/text/locale/said/en.json index 8721d2e4..5924a3df 100644 --- a/internal/gui/text/locale/said/en.json +++ b/internal/gui/text/locale/said/en.json @@ -64,6 +64,11 @@ "hash": "sha256-1b4a0ab80849", "other": "{{.Unpacked}} from {{.Size}}" }, + "archivescan.BombInside": { + "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Unpacked}}, {{.Size}}, written the way the program writes them - keep each spelled exactly that way.", + "hash": "sha256-34337d75d598", + "other": "with the archives inside it, {{.Unpacked}} from {{.Size}}" + }, "archivescan.Bytes": { "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Size}}, written the way the program writes them - keep each spelled exactly that way.", "hash": "sha256-a7df568dcb25", @@ -84,6 +89,11 @@ "hash": "sha256-7ec127709e04", "other": "the record at its end gives the number of entries as {{.Counted}} and the list holds {{.Listed}}" }, + "archivescan.Cut": { + "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Name}}, {{.Length}}, written the way the program writes them - keep each spelled exactly that way.", + "hash": "sha256-54d0a6ef8fec", + "other": "{{.Name}}... ({{.Length}} in all)" + }, "archivescan.Device": { "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run.", "hash": "sha256-679cb400e806", @@ -114,6 +124,16 @@ "hash": "sha256-7a245481ab58", "other": "its gzip header is broken" }, + "archivescan.Inside": { + "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Archive}}, {{.Item}}, written the way the program writes them - keep each spelled exactly that way.", + "hash": "sha256-9fd8e4fbbb4a", + "other": "{{.Archive}} > {{.Item}}" + }, + "archivescan.InsideUnread": { + "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Kind}}, written the way the program writes them - keep each spelled exactly that way.", + "hash": "sha256-1276ea1eefc0", + "other": "it is a {{.Kind}} archive, which this tool does not read" + }, "archivescan.IsADirectory": { "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Path}}, written the way the program writes them - keep each spelled exactly that way.", "hash": "sha256-99f387f3e133", @@ -164,6 +184,11 @@ "hash": "sha256-909cb4c9c048", "other": "it is packed with method {{.Method}}, which this tool does not read" }, + "archivescan.More": { + "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.More}}, written the way the program writes them - keep each spelled exactly that way.", + "hash": "sha256-906bd54a09e0", + "other": "and {{.More}} more" + }, "archivescan.Name": { "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Name}}, written the way the program writes them - keep each spelled exactly that way.", "hash": "sha256-1a0545a47535", @@ -184,6 +209,11 @@ "hash": "sha256-638b1e0460e7", "other": "the record at its end, which says where its list of contents is, is missing" }, + "archivescan.NoRoom": { + "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Most}}, written the way the program writes them - keep each spelled exactly that way.", + "hash": "sha256-968601ad8080", + "other": "it is an archive larger than what is left of the {{.Most}} this tool holds, with the archives around it held" + }, "archivescan.NotReadYet": { "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Path}}, {{.Kind}}, written the way the program writes them - keep each spelled exactly that way.", "hash": "sha256-bfe67470dde3", @@ -244,20 +274,30 @@ "hash": "sha256-66a4b1365362", "other": "it says it is one part of an archive split across several files" }, + "archivescan.StoppedAround": { + "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run.", + "hash": "sha256-80e28d403609", + "other": "the archive given was found built to explode with the archives inside it, and no more is read" + }, "archivescan.StoppedBomb": { "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Ratio}}, written the way the program writes them - keep each spelled exactly that way.", "hash": "sha256-a9e09bc34953", "other": "it unpacks to more than {{.Ratio}} times what was read of it, and no more is read" }, + "archivescan.StoppedBombInside": { + "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Ratio}}, written the way the program writes them - keep each spelled exactly that way.", + "hash": "sha256-51faf1bd773e", + "other": "with the archives inside it, it unpacks to more than {{.Ratio}} times its size, and no more is read" + }, "archivescan.StoppedEntries": { "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Most}}, written the way the program writes them - keep each spelled exactly that way.", - "hash": "sha256-f587645b5f7b", - "other": "it lists more than {{.Most}} entries, and no more are read" + "hash": "sha256-20cc83a69acd", + "other": "with the archives inside it, it lists more than {{.Most}} entries, and no more are read" }, "archivescan.StoppedNames": { "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run.", - "hash": "sha256-21ea12750101", - "other": "its names take more room than this tool sets aside to compare them, and no more are read" + "hash": "sha256-2643637bee68", + "other": "its names, with those of the archives inside it, take more room than this tool sets aside to compare them, and no more are read" }, "archivescan.StreamBroken": { "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run.", @@ -269,6 +309,21 @@ "hash": "sha256-0350e128590c", "other": "a header is broken, or it ends in the middle of one" }, + "archivescan.TheFileInside": { + "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run.", + "hash": "sha256-aa69584a8a46", + "other": "the file it unpacks to" + }, + "archivescan.TooDeep": { + "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Depth}}, written the way the program writes them - keep each spelled exactly that way.", + "hash": "sha256-d1630ce6d4a7", + "other": "it is an archive more than {{.Depth}} archives deep, deeper than this tool looks" + }, + "archivescan.TooLarge": { + "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Most}}, written the way the program writes them - keep each spelled exactly that way.", + "hash": "sha256-b52e73423bba", + "other": "it is an archive larger than the {{.Most}} this tool holds to look into" + }, "archivescan.TwoNames": { "description": "Said by internal/tool/archivescan - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Listed}}, {{.Local}}, written the way the program writes them - keep each spelled exactly that way.", "hash": "sha256-8816d7bc260e", diff --git a/internal/gui/text/locale/said/pl.json b/internal/gui/text/locale/said/pl.json index 01fc9836..f9c3600a 100644 --- a/internal/gui/text/locale/said/pl.json +++ b/internal/gui/text/locale/said/pl.json @@ -12,16 +12,20 @@ "archivescan.ADirectoryIsUnpacked": { "hash": "sha256-625960667a52", "other": "katalog to miejsce, do którego rozpakowuje się archiwum, a nie archiwum" }, "archivescan.AnswerAboutNoVersion": { "hash": "sha256-a9d2669e6062", "other": "odpowiedź o pliku, który właśnie jest zapisywany, nie dotyczy żadnej jego wersji" }, "archivescan.Bomb": { "hash": "sha256-1b4a0ab80849", "other": "{{.Unpacked}} z {{.Size}}" }, + "archivescan.BombInside": { "hash": "sha256-34337d75d598", "other": "razem z archiwami w środku {{.Unpacked}} z {{.Size}}" }, "archivescan.Bytes": { "hash": "sha256-a7df568dcb25", "other": "{{.Size}}" }, "archivescan.Changed": { "hash": "sha256-7b323b86f3c7", "other": "{{.Path}} zmienił się w trakcie czytania" }, "archivescan.Checksum": { "hash": "sha256-046f31c01563", "other": "jego zawartość nie zgadza się ze swoją sumą kontrolną" }, "archivescan.Counted": { "hash": "sha256-7ec127709e04", "other": "rekord na jego końcu podaje liczbę wpisów {{.Counted}}, a spis zawiera ich {{.Listed}}" }, + "archivescan.Cut": { "hash": "sha256-54d0a6ef8fec", "other": "{{.Name}}... (w całości {{.Length}})" }, "archivescan.Device": { "hash": "sha256-679cb400e806", "other": "to urządzenie" }, "archivescan.Encrypted": { "hash": "sha256-f93c3b3dbeb0", "other": "jest zaszyfrowany" }, "archivescan.EndsEarly": { "hash": "sha256-d7126b8d60eb", "other": "jego zawartość kończy się przed rozmiarem podanym w spisie" }, "archivescan.EntryWhy": { "hash": "sha256-13b3c18bff10", "other": "{{.Name}}: {{.Why}}" }, "archivescan.Global": { "hash": "sha256-2464a8a64571", "other": "nagłówek ustawia {{.Key}} każdemu następnemu wpisowi, czego to narzędzie nie uwzględnia" }, "archivescan.GzipHeader": { "hash": "sha256-7a245481ab58", "other": "jego nagłówek gzip jest uszkodzony" }, + "archivescan.Inside": { "hash": "sha256-9fd8e4fbbb4a", "other": "{{.Archive}} > {{.Item}}" }, + "archivescan.InsideUnread": { "hash": "sha256-1276ea1eefc0", "other": "to archiwum {{.Kind}}, którego to narzędzie nie czyta" }, "archivescan.IsADirectory": { "hash": "sha256-99f387f3e133", "other": "{{.Path}} to katalog, a to narzędzie sprawdza jedno archiwum" }, "archivescan.IsNotAFile": { "hash": "sha256-28957d41213f", "other": "{{.Path}} nie jest plikiem, tylko urządzeniem, potokiem albo gniazdem" }, "archivescan.IsNotAnArchive": { "hash": "sha256-4ab90d477d74", "other": "{{.Path}} nie jest plikiem ZIP, TAR, TAR.GZ ani gzip" }, @@ -32,10 +36,12 @@ "archivescan.ListEndsEarly": { "hash": "sha256-13e077022aaf", "other": "jego spis zawartości urywa się w środku wpisu" }, "archivescan.ListHoldsOther": { "hash": "sha256-294d9d38d2bf", "other": "jego spis zawartości zawiera coś, co nie jest wpisem" }, "archivescan.Method": { "hash": "sha256-909cb4c9c048", "other": "jest spakowany metodą {{.Method}}, której to narzędzie nie czyta" }, + "archivescan.More": { "hash": "sha256-906bd54a09e0", "other": "i {{.More}} więcej" }, "archivescan.Name": { "hash": "sha256-1a0545a47535", "other": "{{.Name}}" }, "archivescan.NameAZipOrTar": { "hash": "sha256-17713db18b11", "other": "wskaż plik ZIP, TAR, TAR.GZ albo gzip" }, "archivescan.NameOneArchive": { "hash": "sha256-b3119045be6e", "other": "wskaż jeden plik archiwum" }, "archivescan.NoEnd": { "hash": "sha256-638b1e0460e7", "other": "brakuje rekordu na jego końcu, który mówi, gdzie jest spis zawartości" }, + "archivescan.NoRoom": { "hash": "sha256-968601ad8080", "other": "to archiwum większe niż to, co zostaje z {{.Most}}, które to narzędzie trzyma, gdy archiwa wokół niego są już trzymane" }, "archivescan.NotReadYet": { "hash": "sha256-bfe67470dde3", "other": "{{.Path}} to archiwum {{.Kind}}, którego to narzędzie jeszcze nie czyta" }, "archivescan.NotWhereListed": { "hash": "sha256-8e9d4c7346e8", "other": "nie zaczyna się tam, gdzie podaje spis zawartości" }, "archivescan.OverLimit": { "hash": "sha256-31b786bac0bd", "other": "{{.Unpacked}} przy limicie {{.Limit}}" }, @@ -48,11 +54,16 @@ "archivescan.Size": { "hash": "sha256-8deb8dd02c4a", "other": "{{.Size}} ({{.Exactly}})" }, "archivescan.Socket": { "hash": "sha256-a0d8c46d99ca", "other": "to gniazdo" }, "archivescan.Split": { "hash": "sha256-66a4b1365362", "other": "podaje, że jest jedną częścią archiwum podzielonego na kilka plików" }, + "archivescan.StoppedAround": { "hash": "sha256-80e28d403609", "other": "podane archiwum razem z archiwami w środku okazało się zbudowane, żeby eksplodować, więc dalej nie jest czytane" }, "archivescan.StoppedBomb": { "hash": "sha256-a9e09bc34953", "other": "rozpakowuje się do ponad {{.Ratio}} razy więcej, niż z niego przeczytano, więc dalej nie jest czytane" }, - "archivescan.StoppedEntries": { "hash": "sha256-f587645b5f7b", "other": "wymienia ponad {{.Most}} wpisów, więc dalsze nie są czytane" }, - "archivescan.StoppedNames": { "hash": "sha256-21ea12750101", "other": "jego nazwy zajmują więcej miejsca, niż to narzędzie przeznacza na ich porównanie, więc dalsze nie są czytane" }, + "archivescan.StoppedBombInside": { "hash": "sha256-51faf1bd773e", "other": "razem z archiwami w środku rozpakowuje się do ponad {{.Ratio}} razy więcej niż jego rozmiar, więc dalej nie jest czytane" }, + "archivescan.StoppedEntries": { "hash": "sha256-20cc83a69acd", "other": "razem z archiwami w środku wymienia ponad {{.Most}} wpisów, więc dalsze nie są czytane" }, + "archivescan.StoppedNames": { "hash": "sha256-2643637bee68", "other": "jego nazwy razem z nazwami archiwów w środku zajmują więcej miejsca, niż to narzędzie przeznacza na ich porównanie, więc dalsze nie są czytane" }, "archivescan.StreamBroken": { "hash": "sha256-afe268a16a8f", "other": "kończy się przedwcześnie albo jego zawartość się nie rozpakowuje" }, "archivescan.TarHeader": { "hash": "sha256-0350e128590c", "other": "nagłówek jest uszkodzony albo archiwum urywa się w jego środku" }, + "archivescan.TheFileInside": { "hash": "sha256-aa69584a8a46", "other": "plik, do którego się rozpakowuje" }, + "archivescan.TooDeep": { "hash": "sha256-d1630ce6d4a7", "other": "to archiwum zagnieżdżone głębiej niż {{.Depth}} archiwów, głębiej niż to narzędzie zagląda" }, + "archivescan.TooLarge": { "hash": "sha256-b52e73423bba", "other": "to archiwum większe niż {{.Most}}, które to narzędzie trzyma, żeby do niego zajrzeć" }, "archivescan.TwoNames": { "hash": "sha256-8816d7bc260e", "other": "{{.Listed}} w spisie, {{.Local}} przy wpisie" }, "archivescan.Type": { "hash": "sha256-ea85c62d6ca3", "other": "to wpis typu {{.Type}}, którego większość programów nie rozpakowuje" }, "archivescan.WaitAndRunAgain": { "hash": "sha256-161d36039e9c", "other": "poczekaj, aż to, co go zapisuje, skończy, i uruchom to jeszcze raz" }, diff --git a/internal/tool/archivescan/archivescan.go b/internal/tool/archivescan/archivescan.go index 78b1c3a4..cc943bff 100644 --- a/internal/tool/archivescan/archivescan.go +++ b/internal/tool/archivescan/archivescan.go @@ -16,7 +16,6 @@ import ( "context" "errors" "io" - "os" "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" @@ -33,7 +32,10 @@ const ( SettingLimit = "unpacked_limit" ) -// The limits, fixed. The one a person sets is SettingLimit. +// The limits, fixed. The one a person sets is SettingLimit. Each is the limit +// of the archive given and every archive inside it together, not of each one, +// so an archive of a thousand archives holds no more than one archive may +// (docs/SKANER-W-GLAB-2026-10-19.md §2.1). const ( // MostEntries is where reading stops. A million names held to find two // that land on one file is a few hundred megabytes, and an archive with @@ -45,6 +47,21 @@ const ( // 64 GB held in memory to answer a question about a file. MostPlaces = 2 * MostEntries MostNameBytes = 256 << 20 + // MostDepth is how many archives deep the tool looks. An archive can + // hold itself, and that ends here rather than never. + MostDepth = 16 + // MostInside is the most the archives inside hold in memory at once to + // be read: a ZIP is read from its end, so it has to be held whole. + MostInside = 256 << 20 + // MostListed, MostShownName and MostListedBytes bound the report: the + // items of one note, the bytes of one name in it, and the bytes of every + // name in it. Past them a finding is counted and not listed - a million + // findings, or a name of 64 KiB, are no report anybody reads. + MostListed = 100 + MostShownName = 1024 + MostListedBytes = 4 << 20 + // MostReports is how many reports of archives inside --json carries. + MostReports = 1000 // BombBytes and BombRatio are an archive built to explode: at least this // much when unpacked, and more than this many times its own size. Both, // because either alone is innocent - a file of zeros packs a thousand to @@ -73,7 +90,7 @@ func init() { }}, Settings: []format.Property{{ Name: SettingLimit, Kind: format.PropertySize, - Detail: "The most the archive may unpack to, all its files together - for a server or a disk that takes no more. " + + Detail: "The most the archive may unpack to, all its files together with those of the archives inside it - for a server or a disk that takes no more. " + "Left empty, only an archive built to explode is reported: 1 GB or more, and more than 100 times its own size.", }}, Notes: declaredNotes(), @@ -107,6 +124,7 @@ func run(ctx context.Context, in tool.Request, progress tool.Progress) (tool.Res if err := s.look(f); err != nil { return tool.Result{}, err } + s.overLimit() after, err := f.Stat() if err != nil { return tool.Result{}, err @@ -117,10 +135,11 @@ func run(ctx context.Context, in tool.Request, progress tool.Progress) (tool.Res return s.result(), nil } -// look tells what the file is and reads it as that. -func (s *scan) look(f *os.File) error { +// look tells what an archive is and reads it as that: the file given, or an +// archive inside it held in memory. +func (s *scan) look(r io.ReaderAt) error { head := make([]byte, headLength) - n, err := f.ReadAt(head, 0) + n, err := r.ReadAt(head, 0) if err != nil && !errors.Is(err, io.EOF) { return err } @@ -128,12 +147,12 @@ func (s *scan) look(f *os.File) error { switch k, other := sniff(head); { case k == kindZip: s.kind = kindZip - return s.zip(f) + return s.zip(r) case k == kindTar: s.kind = kindTar - return s.tar(io.NewSectionReader(f, 0, s.size)) + return s.tar(io.NewSectionReader(r, 0, s.size)) case k == kindGzip: - return s.gzip(io.NewSectionReader(f, 0, s.size)) + return s.gzip(io.NewSectionReader(r, 0, s.size)) case other != "": return &NotReadYetError{Path: s.path, Kind: other} } diff --git a/internal/tool/archivescan/findings.go b/internal/tool/archivescan/findings.go index 1a42d59b..3da2e70b 100644 --- a/internal/tool/archivescan/findings.go +++ b/internal/tool/archivescan/findings.go @@ -13,10 +13,11 @@ import ( // The keys of the notes. Also the keys of what --json reports as found, so // each is a public name (untouchable rule 10). const ( - noteKind = "kind" - noteEntries = "entries" - notePacked = "size" - noteUnpacked = "unpacked" + noteKind = "kind" + noteEntries = "entries" + notePacked = "size" + noteUnpacked = "unpacked" + noteUnpackedAll = "unpacked_all" noteLeaves = "leaves_folder" noteLinks = "links" @@ -47,6 +48,7 @@ var declared = []struct { {tool.Note{ID: noteEntries, Says: "Entries:"}, true}, {tool.Note{ID: notePacked, Says: "Size:"}, true}, {tool.Note{ID: noteUnpacked, Says: "Unpacked:"}, true}, + {tool.Note{ID: noteUnpackedAll, Says: "Unpacked, with the archives inside it:"}, true}, {tool.Note{ID: noteLeaves, Says: "Entries that would be written outside the folder:"}, false}, {tool.Note{ID: noteLinks, Says: "Links, which a later entry can be written through:"}, false}, {tool.Note{ID: noteSpecial, Says: "Entries that are not plain files or folders:"}, false}, @@ -60,7 +62,7 @@ var declared = []struct { {tool.Note{ID: noteDamaged, Says: "Damaged, so programs that unpack it may disagree on what it holds:"}, false}, {tool.Note{ID: noteUnchecked, Says: "Entries whose contents could not be checked:"}, false}, {tool.Note{ID: noteStopped, Says: "Reading stopped before the end:"}, false}, - {tool.Note{ID: noteNested, Says: "Archives inside it, which were not opened:"}, true}, + {tool.Note{ID: noteNested, Says: "Archives inside it, read as well:"}, true}, } // declaredNotes is what the registry is told. @@ -80,64 +82,112 @@ type found struct { } // Report is what --json prints. Names are the names in the archive, byte for -// byte, so a script compares them. The window and the text show them escaped. +// byte, so a script compares them - up to MostShownName bytes, where a longer +// one is cut where a character ends, the name of an archive inside too. The +// window and the text show them escaped. type Report struct { File string `json:"file"` Kind string `json:"kind"` Bytes int64 `json:"bytes"` Entries int `json:"entries"` Unpacked uint64 `json:"unpacked_bytes"` - Safe bool `json:"safe"` - // Found is every finding by its key, each with the entries it is about. - // A finding about the archive as a whole has no entry and an empty list. + // UnpackedAll is what it unpacks to with every archive inside it. + UnpackedAll uint64 `json:"unpacked_all_bytes"` + Safe bool `json:"safe"` + // Found is every finding in this archive by its key, each with the + // entries it is about, at most MostListed of them. A finding about the + // archive as a whole has no entry and an empty list. Found map[string][]string `json:"found"` + // More is how many findings of a key there are past what Found lists. + More map[string]int `json:"more,omitempty"` + // Inside are the reports of the archives in this one that are not safe, + // each File the name it has here, and InsideMore how many more of them + // there are than Inside carries. + Inside []*Report `json:"inside,omitempty"` + InsideMore int `json:"inside_more,omitempty"` } -// scan is one run of the tool over one archive. +// scan is the reading of one archive: the one given, or one inside it. type scan struct { - ctx context.Context - path string - size int64 - limit int64 - progress tool.Progress - read int64 + path string + size int64 + read int64 kind kind entries int unpacked uint64 names *book - nameRoom int stopped bool // exploded is an archive found built to explode, noted once. exploded bool - found map[string][]found - // buf is what every entry's contents are read into, one for the scan. - buf []byte + found findings + nest nest + + tree *tree + depth int + // hold keeps what an entry holds while it is read, when that is an + // archive to read in turn. + hold holding +} + +// findings are what a scan found: own in this archive, deep in the archives +// inside it, each said with the way to it, and in more and deepMore how many +// of each were found past what a report lists. +type findings struct { + own map[string][]found + more map[string]int + deep map[string][]found + deepMore map[string]int } +// nest is what the archives inside one came to: those kept for the report, +// how many more were not safe, whether any was not safe, and what they all +// unpack to. +type nest struct { + kept []*scan + more int + unsafe bool + unpacked uint64 +} + +// newScan is the reading of the archive given. func newScan(ctx context.Context, path string, size, limit int64, progress tool.Progress) *scan { + t := &tree{ + ctx: ctx, limit: limit, progress: progress, + placeRoom: MostPlaces, nameRoom: MostNameBytes, buf: make([]byte, 64<<10), + } + s := blank(path, size, t) + t.root = s + return s +} + +// blank is the reading of an archive in the run t, before anything is read. +func blank(path string, size int64, t *tree) *scan { return &scan{ - ctx: ctx, path: path, size: size, limit: limit, progress: progress, - names: newBook(), nameRoom: MostNameBytes, found: map[string][]found{}, - buf: make([]byte, 64<<10), + path: path, size: size, tree: t, names: newBook(&t.placeRoom), + found: findings{own: map[string][]found{}, more: map[string]int{}, deep: map[string][]found{}, deepMore: map[string]int{}}, } } // note records a finding about an entry, shown as the entry's name. func (s *scan) note(id, name string) { - s.found[id] = append(s.found[id], found{name: name}) + s.noteSaid(id, name, core.Said{}) } -// noteSaid records a finding with words of its own. +// noteSaid records a finding with words of its own, or counts it once the +// report has no room left to list it. func (s *scan) noteSaid(id, name string, words core.Said) { - s.found[id] = append(s.found[id], found{name: name, words: words}) + s.tree.add(s.found.own, s.found.more, id, found{name: name, words: words}) } -// counted tells the progress how much more of the archive has been read. +// counted tells the progress how much more of the archive has been read - +// of the archive given: the ones inside it are read from memory. func (s *scan) counted(n int64) { s.read += n - s.progress(min(s.read, s.size), s.size) + if s == s.tree.root { + s.tree.progress(min(s.read, s.size), s.size) + } } // stop ends reading, saying why. Everything after this point is unseen, so @@ -150,18 +200,34 @@ func (s *scan) stop(why core.Said) { s.noteSaid(noteStopped, "", why) } +// going says whether reading goes on: not once this archive has stopped, nor +// once the run is found built to explode. The archive given says why then, and +// every archive between it and the one that found out stops with it. +func (s *scan) going() bool { + if s.tree.exploded { + s.stopped = true + } + return !s.stopped +} + // entry is what every kind of archive does for each entry it lists: count it, // judge its name, and record where it lands. False when reading has to stop. func (s *scan) entry(name string, folder bool) bool { - if s.entries >= MostEntries { - s.stop(core.Says("archivescan.StoppedEntries", "it lists more than %s entries, and no more are read", core.A("Most", core.Exactly(MostEntries)))) + if !s.going() { return false } - s.nameRoom -= len(name) - if s.nameRoom < 0 || s.names.full() { - s.stop(core.Says("archivescan.StoppedNames", "its names take more room than this tool sets aside to compare them, and no more are read")) + t := s.tree + if t.entries >= MostEntries { + s.stop(core.Says("archivescan.StoppedEntries", "with the archives inside it, it lists more than %s entries, and no more are read", + core.A("Most", core.Exactly(MostEntries)))) return false } + t.nameRoom -= len(name) + if t.nameRoom < 0 || s.names.full() { + s.stop(core.Says("archivescan.StoppedNames", "its names, with those of the archives inside it, take more room than this tool sets aside to compare them, and no more are read")) + return false + } + t.entries++ s.entries++ if leaves(name) { s.note(noteLeaves, name) @@ -170,20 +236,33 @@ func (s *scan) entry(name string, folder bool) bool { s.note(noteUnwritable, name) } if first, ok := s.names.add(name, folder); ok { - s.noteSaid(noteCollides, name, core.Says("archivescan.Same", "%s = %s", core.A("Name", core.Shown(name)), core.A("First", core.Shown(first)))) + s.noteSaid(noteCollides, name, core.Says("archivescan.Same", "%s = %s", core.A("Name", shownName(name)), core.A("First", shownName(first)))) } return true } -// weigh is the size of what an archive unpacks to, against the archive and -// against the limit given. +// grow adds to what an archive unpacks to, and to what the run does. +func (s *scan) grow(n uint64) { + s.unpacked = addCapped(s.unpacked, n) + s.tree.unpacked = addCapped(s.tree.unpacked, n) +} + +// weigh is the size of what an archive unpacks to, against the archive and, +// with every archive read so far, against the archive given. func (s *scan) weigh() { if !s.exploded && s.unpacked >= BombBytes && s.unpacked > uint64(s.size)*BombRatio { s.exploded = true s.noteSaid(noteBomb, "", core.Says("archivescan.Bomb", "%s from %s", core.A("Unpacked", bytesOf(s.unpacked)), core.A("Size", core.Bytes(s.size)))) } - if s.limit >= 0 && s.unpacked > uint64(s.limit) { - s.noteSaid(noteOverLimit, "", core.Says("archivescan.OverLimit", "%s against a limit of %s", core.A("Unpacked", bytesOf(s.unpacked)), core.A("Limit", core.Bytes(s.limit)))) + s.tree.check(s) +} + +// overLimit is what the archive given unpacks to, with every archive inside +// it, against the limit given. +func (s *scan) overLimit() { + all := addCapped(s.unpacked, s.nest.unpacked) + if limit := s.tree.limit; limit >= 0 && all > uint64(limit) { + s.noteSaid(noteOverLimit, "", core.Says("archivescan.OverLimit", "%s against a limit of %s", core.A("Unpacked", bytesOf(all)), core.A("Limit", core.Bytes(limit)))) } } @@ -196,50 +275,6 @@ func bytesOf(n uint64) core.Bytes { return core.Bytes(n) } -// result is what the scan found, as the registry hands it to both surfaces. -func (s *scan) result() tool.Result { - report := Report{ - File: s.path, Kind: string(s.kind), Bytes: s.size, Entries: s.entries, - Unpacked: s.unpacked, Safe: true, Found: map[string][]string{}, - } - summary := map[string]tool.Noted{ - noteKind: {ID: noteKind, Items: []string{string(s.kind)}}, - noteEntries: {ID: noteEntries, Items: []string{core.Exactly(int64(s.entries))}}, - notePacked: sized(notePacked, uint64(s.size)), - noteUnpacked: sized(noteUnpacked, s.unpacked), - } - r := tool.Result{Data: &report, Concluded: concludedSafe} - for _, d := range declared { - if n, ok := summary[d.ID]; ok { - r.Notes = append(r.Notes, n) - continue - } - list := s.found[d.ID] - if len(list) == 0 { - continue - } - r.Notes = append(r.Notes, noted(d.ID, list)) - report.Found[d.ID] = namesOf(list) - if !d.harmless { - report.Safe = false - r.Concluded = concludedUnsafe - } - } - return r -} - -// namesOf is the entries a list of findings is about, for --json: a finding -// about the archive as a whole names none. -func namesOf(list []found) []string { - names := make([]string, 0, len(list)) - for _, f := range list { - if f.name != "" { - names = append(names, f.name) - } - } - return names -} - // special notes an entry that is not a plain file or folder, and why. func (s *scan) special(name string, why core.Said) { s.noteSaid(noteSpecial, name, entryWhy(name, why)) @@ -247,39 +282,5 @@ func (s *scan) special(name string, why core.Said) { // entryWhy is an entry and what is wrong with it, as one item of a note. func entryWhy(name string, why core.Said) core.Said { - return core.Says("archivescan.EntryWhy", "%s: %s", core.A("Name", core.Shown(name)), core.A("Why", why)) -} - -// sized is a note of one size, said the way a person reads it with the exact -// count beside it. Below a kilobyte the two are one number, said once. -func sized(id string, n uint64) tool.Noted { - words := core.Says("archivescan.Size", "%s (%s)", core.A("Size", bytesOf(n)), core.A("Exactly", core.ExactBytes(int64(bytesOf(n))))) - if n < 1024 { - words = core.Says("archivescan.Bytes", "%s", core.A("Size", bytesOf(n))) - } - return tool.Noted{ID: id, Items: []string{words.String()}, Words: []core.Said{words}} -} - -// noted is a note as the registry carries it: each finding as data, and in -// words where any of them has words - then every one, since a window shows -// the words of all or of none. -func noted(id string, list []found) tool.Noted { - n := tool.Noted{ID: id} - worded := false - for _, f := range list { - worded = worded || !f.words.IsZero() - } - for _, f := range list { - words := f.words - if words.IsZero() { - words = core.Says("archivescan.Name", "%s", core.A("Name", core.Shown(f.name))) - } - if worded { - n.Items = append(n.Items, words.String()) - n.Words = append(n.Words, words) - continue - } - n.Items = append(n.Items, f.name) - } - return n + return core.Says("archivescan.EntryWhy", "%s: %s", core.A("Name", shownName(name)), core.A("Why", why)) } diff --git a/internal/tool/archivescan/inside.go b/internal/tool/archivescan/inside.go new file mode 100644 index 00000000..4d6c7c36 --- /dev/null +++ b/internal/tool/archivescan/inside.go @@ -0,0 +1,309 @@ +package archivescan + +import ( + "context" + "errors" + "io" + "math" + "slices" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/tool" +) + +// The archives inside an archive: each is read the way the one given is, from +// memory, and what is wrong in it is wrong in the one given - a program that +// unpacks everything inside unpacks those too (docs/SKANER-W-GLAB-2026-10-19.md). +// +// Nothing here grows with what an archive holds. The limits are the run's, +// shared by the archive given and every archive inside it, so an archive of a +// thousand archives is allowed no more than one archive alone. How deep the +// tool looks is counted before it looks, and an archive inside is read only +// once the entry holding it has been read whole, so one is read at a time. + +// tree is what one run of the tool shares across every archive it reads. +type tree struct { + ctx context.Context + root *scan + // limit and progress are the archive given's: the limit a person set, + // or -1, and where the reading of it is told. + limit int64 + progress tool.Progress + // entries, placeRoom and nameRoom are counted once for the whole run. + entries int + placeRoom int + nameRoom int + // held is the bytes of archives inside held in memory right now. + held int + // unpacked is what everything read unpacks to, on every level. + unpacked uint64 + // listed is the bytes of names the report keeps, and reports how many + // reports of archives inside it keeps. + listed int + reports int + // exploded is a run whose archives, together, were found built to + // explode. + exploded bool + // buf is the one buffer every archive's contents are read into. + buf []byte +} + +// explodes says whether what everything read so far unpacks to, the archive +// given and every one inside it, makes the archive given one built to explode, +// and says so on it the first time. Only once something inside has unpacked: +// before that the archive given asks it of itself. +func (t *tree) explodes() bool { + if t.exploded { + return true + } + r := t.root + if t.unpacked == r.unpacked || t.unpacked < BombBytes || t.unpacked <= uint64(r.size)*BombRatio { + return false + } + t.exploded = true + r.exploded = true + r.noteSaid(noteBomb, "", core.Says("archivescan.BombInside", "with the archives inside it, %s from %s", + core.A("Unpacked", bytesOf(t.unpacked)), core.A("Size", core.Bytes(r.size)))) + r.stop(core.Says("archivescan.StoppedBombInside", "with the archives inside it, it unpacks to more than %d times its size, and no more is read", + core.A("Ratio", BombRatio))) + return true +} + +// check asks the run's rule once an archive has asked its own, and ends the +// reading of that archive when the run is found built to explode - saying so +// in an archive inside that had not found it of itself, whose own reading +// ends unfinished. +func (t *tree) check(s *scan) { + if !t.explodes() || s.exploded { + return + } + s.exploded = true + if s != t.root { + s.stop(core.Says("archivescan.StoppedAround", "the archive given was found built to explode with the archives inside it, and no more is read")) + } +} + +// holding is what an entry holds, kept while it is read when it begins as an +// archive this tool reads, to be read in turn once it has been read whole. It +// is kept in pieces that are never copied to grow - growing by copying holds +// the old copy and the new one at once - and read back through ReadAt. +type holding struct { + t *tree + // said is what the entry says it holds, or -1 when nothing says. + said int64 + // open is whether an archive found here may be read: not one already as + // deep as the tool looks. + open bool + decided bool + head []byte + // kind is the archive the contents begin as, or nothing. + kind string + // parts are what is held, each of piece bytes but the last, size how + // much that is, and held whether the contents are being held. + parts [][]byte + size int64 + held bool + // need is what an archive inside needed to be held and was not given: + // more than is ever held, or more than is left while the archives around + // it are held. Zero for one that was. + need int64 +} + +// piece is how much of an archive inside one allocation holds. +const piece = 1 << 20 + +// start readies the holding for the contents of one entry. +func (h *holding) start(t *tree, said uint64, open bool) { + h.drop() + h.t, h.open, h.decided, h.head, h.kind, h.need = t, open, false, h.head[:0], "", 0 + h.said = int64(min(said, math.MaxInt64)) +} + +// unsaid readies the holding for contents nothing gives the size of. +func (h *holding) unsaid(t *tree, open bool) { + h.start(t, 0, open) + h.said = -1 +} + +// Write is a piece of the contents, in order. It never fails: what does not +// fit is not held, and the reading goes on. +func (h *holding) Write(p []byte) (int, error) { + n := len(p) + if !h.decided { + take := min(len(p), headLength-len(h.head)) + h.head = append(h.head, p[:take]...) + if len(h.head) < headLength { + return n, nil + } + h.decide() + p = p[take:] + } + h.keep(p) + return n, nil +} + +// decide tells from the first bytes whether the contents are an archive, and +// holds them when they are one this tool reads and there is room for them. +func (h *holding) decide() { + h.decided = true + h.kind = archiveIn(h.head) + if !h.open || !readable(h.kind) || (h.said >= 0 && h.said < int64(len(h.head))) { + return + } + if h.said > int64(MostInside-h.t.held) { + h.need = h.said + return + } + h.held = true + h.keep(h.head) +} + +// keep holds more of the contents, while they are being held. +func (h *holding) keep(p []byte) { + for len(p) > 0 && h.held { + if h.full() && !h.grow() { + return + } + last := len(h.parts) - 1 + part := h.parts[last] + n := copy(part[len(part):cap(part)], p) + h.parts[last] = part[:len(part)+n] + h.size += int64(n) + p = p[n:] + } +} + +// full says whether there is no room left in the pieces held so far. +func (h *holding) full() bool { + last := len(h.parts) - 1 + return last < 0 || len(h.parts[last]) == cap(h.parts[last]) +} + +// grow adds a piece. Past what the entry said it holds, nothing more is held +// and it is let go - the entry is reported for holding more than it says, and +// never read. Past what is left to hold, the same, and the archive is too +// large to look into. +func (h *holding) grow() bool { + want := int64(piece) + if h.said >= 0 { + want = min(want, h.said-h.size) + } + if want <= 0 || want > int64(MostInside-h.t.held) { + if want > 0 { + h.need = h.size + want + } + h.drop() + return false + } + h.parts = append(h.parts, make([]byte, 0, want)) + h.t.held += int(want) + return true +} + +// drop lets go of what is held. +func (h *holding) drop() { + for _, part := range h.parts { + h.t.held -= cap(part) + } + h.parts, h.size, h.held = nil, 0, false +} + +// errBefore is a read before the first byte held, which nothing in this +// package asks for. +var errBefore = core.Defect(errors.New("an archive held in memory was read before its first byte")) + +// ReadAt reads what is held, the way a file of it would be read. +func (h *holding) ReadAt(p []byte, off int64) (int, error) { + if off < 0 { + return 0, errBefore + } + n := 0 + for n < len(p) && off < h.size { + c := copy(p[n:], h.parts[off/piece][off%piece:]) + n += c + off += int64(c) + } + if n < len(p) { + return n, io.EOF + } + return n, nil +} + +// within reads the archive an entry held, the last step of reading an entry +// whose contents were whole - or says why it is not read. +func (s *scan) within(name string, h *holding) error { + if !h.decided { + // Contents shorter than a TAR header, read whole: their size is + // known now. + if h.said < 0 { + h.said = int64(len(h.head)) + } + h.decide() + } + defer h.drop() + var why core.Said + switch { + case h.kind == "": + return nil + case !readable(h.kind): + why = core.Says("archivescan.InsideUnread", "it is a %s archive, which this tool does not read", core.A("Kind", h.kind)) + case !h.open: + why = core.Says("archivescan.TooDeep", "it is an archive more than %d archives deep, deeper than this tool looks", core.A("Depth", MostDepth)) + case h.need > MostInside: + why = core.Says("archivescan.TooLarge", "it is an archive larger than the %s this tool holds to look into", core.A("Most", core.Bytes(MostInside))) + case h.need > 0: + why = core.Says("archivescan.NoRoom", "it is an archive larger than what is left of the %s this tool holds, with the archives around it held", + core.A("Most", core.Bytes(MostInside))) + case !h.held: + return nil + } + if !why.IsZero() { + s.noteSaid(noteUnchecked, name, entryWhy(name, why)) + return nil + } + c := blank(cutName(name), h.size, s.tree) + c.depth = s.depth + 1 + err := c.look(h) + c.hold.drop() + if err != nil { + return err + } + s.took(name, c) + return nil +} + +// took is an archive read inside this one: what is wrong in it is wrong here, +// each said with the way to it. +func (s *scan) took(name string, c *scan) { + c.names = nil + s.note(noteNested, name) + s.nest.unpacked = addCapped(s.nest.unpacked, addCapped(c.unpacked, c.nest.unpacked)) + for _, d := range declared { + for _, f := range slices.Concat(c.found.own[d.ID], c.found.deep[d.ID]) { + s.tree.add(s.found.deep, s.found.deepMore, d.ID, deeper(name, f)) + } + s.found.deepMore[d.ID] += c.found.more[d.ID] + c.found.deepMore[d.ID] + } + if c.safe() { + return + } + s.nest.unsafe = true + if len(s.nest.kept) >= MostListed || s.tree.reports >= MostReports || s.tree.listed+len(c.path) > MostListedBytes { + s.nest.more++ + return + } + s.nest.kept = append(s.nest.kept, c) + s.tree.reports++ + s.tree.listed += len(c.path) +} + +// deeper is a finding of an archive inside, said with the name of the entry +// that holds it. Its own name is in the words, not the data: --json names what +// is in one archive, and the archives inside have reports of their own. +func deeper(name string, f found) found { + words := f.words + if words.IsZero() { + words = core.Says("archivescan.Name", "%s", core.A("Name", shownName(f.name))) + } + return found{words: core.Says("archivescan.Inside", "%s > %s", core.A("Archive", shownName(name)), core.A("Item", words))} +} diff --git a/internal/tool/archivescan/names.go b/internal/tool/archivescan/names.go index f0262d41..e9f5d977 100644 --- a/internal/tool/archivescan/names.go +++ b/internal/tool/archivescan/names.go @@ -119,21 +119,28 @@ type book struct { at map[place]int32 file []bool named []string + // room is the places the books of one run may still hold, all of them + // together. + room *int } // root is the folder an archive is unpacked into. const root = -1 -func newBook() *book { - return &book{at: map[place]int32{}} +func newBook(room *int) *book { + return &book{at: map[place]int32{}, room: room} } -// full says the book holds as many places as it is allowed to. -func (b *book) full() bool { return len(b.named) >= MostPlaces } +// full says the books of the run hold as many places as they are allowed to. +func (b *book) full() bool { return *b.room <= 0 } // add records an entry, and says which entry seen before it lands on the same // place - or put a file where this one needs a folder - if one did. func (b *book) add(name string, folder bool) (string, bool) { + // A copy, held: archive/tar hands back a name set in a PAX header as a + // slice of the whole header, and holding the slice holds up to a + // megabyte for every entry (docs/SKANER-W-GLAB-2026-10-19.md §1). + name = strings.Clone(name) in := int32(root) ps := parts(name) for i, p := range ps { @@ -161,6 +168,7 @@ func (b *book) add(name string, folder bool) (string, bool) { // put records a place first reached by the entry with this name. func (b *book) put(here place, name string, file bool) int32 { id := int32(len(b.named)) + *b.room-- b.at[here] = id b.file = append(b.file, file) b.named = append(b.named, name) diff --git a/internal/tool/archivescan/report.go b/internal/tool/archivescan/report.go new file mode 100644 index 00000000..fe35c2d2 --- /dev/null +++ b/internal/tool/archivescan/report.go @@ -0,0 +1,190 @@ +package archivescan + +import ( + "slices" + "strings" + "unicode/utf8" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/tool" +) + +// What a scan found, as the registry hands it to both surfaces and as --json +// prints it. A report is bounded however much an archive holds: a note lists +// MostListed findings and counts the rest, a name is shown up to +// MostShownName bytes, and every name a report keeps, in the archive given and +// every one inside it, adds up to MostListedBytes at most. + +// result is what the scan found, as the registry hands it to both surfaces. +func (s *scan) result() tool.Result { + report := s.report() + r := tool.Result{Data: report, Concluded: concludedSafe} + if !report.Safe { + r.Concluded = concludedUnsafe + } + summary := map[string]tool.Noted{ + noteKind: {ID: noteKind, Items: []string{string(s.kind)}}, + noteEntries: {ID: noteEntries, Items: []string{core.Exactly(int64(s.entries))}}, + notePacked: sized(notePacked, uint64(s.size)), + noteUnpacked: sized(noteUnpacked, s.unpacked), + } + if len(s.found.own[noteNested])+s.found.more[noteNested] > 0 { + summary[noteUnpackedAll] = sized(noteUnpackedAll, report.UnpackedAll) + } + for _, d := range declared { + if n, ok := summary[d.ID]; ok { + r.Notes = append(r.Notes, n) + continue + } + list := slices.Concat(s.found.own[d.ID], s.found.deep[d.ID]) + more := s.found.more[d.ID] + s.found.deepMore[d.ID] + if len(list) > 0 || more > 0 { + r.Notes = append(r.Notes, noted(d.ID, list, more)) + } + } + return r +} + +// report is what --json says of this archive and of the archives inside it +// that are not safe. +func (s *scan) report() *Report { + r := &Report{ + File: s.path, Kind: string(s.kind), Bytes: s.size, Entries: s.entries, Unpacked: s.unpacked, + UnpackedAll: addCapped(s.unpacked, s.nest.unpacked), Safe: s.safe(), + Found: map[string][]string{}, More: map[string]int{}, InsideMore: s.nest.more, + } + for _, d := range declared { + if len(s.found.own[d.ID])+s.found.more[d.ID] > 0 { + r.Found[d.ID] = namesOf(s.found.own[d.ID]) + } + if s.found.more[d.ID] > 0 { + r.More[d.ID] = s.found.more[d.ID] + } + } + if len(r.More) == 0 { + r.More = nil + } + for _, c := range s.nest.kept { + r.Inside = append(r.Inside, c.report()) + } + return r +} + +// safe says whether nothing in this archive, nor in any archive inside it, +// makes unpacking it unsafe. +func (s *scan) safe() bool { + if s.nest.unsafe { + return false + } + for _, d := range declared { + if !d.harmless && len(s.found.own[d.ID])+s.found.more[d.ID] > 0 { + return false + } + } + return true +} + +// namesOf is the entries a list of findings is about, for --json: a finding +// about the archive as a whole names none. +func namesOf(list []found) []string { + names := make([]string, 0, len(list)) + for _, f := range list { + if f.name != "" { + names = append(names, f.name) + } + } + return names +} + +// sized is a note of one size, said the way a person reads it with the exact +// count beside it. Below a kilobyte the two are one number, said once. +func sized(id string, n uint64) tool.Noted { + words := core.Says("archivescan.Size", "%s (%s)", core.A("Size", bytesOf(n)), core.A("Exactly", core.ExactBytes(int64(bytesOf(n))))) + if n < 1024 { + words = core.Says("archivescan.Bytes", "%s", core.A("Size", bytesOf(n))) + } + return tool.Noted{ID: id, Items: []string{words.String()}, Words: []core.Said{words}} +} + +// noted is a note as the registry carries it: each finding as data, and in +// words where any of them has words - then every one, since a window shows +// the words of all or of none - and how many more were found than it lists. +func noted(id string, list []found, more int) tool.Noted { + n := tool.Noted{ID: id} + worded := more > 0 + for _, f := range list { + worded = worded || !f.words.IsZero() + } + for _, f := range list { + words := f.words + if words.IsZero() { + words = core.Says("archivescan.Name", "%s", core.A("Name", shownName(f.name))) + } + if worded { + n.Items = append(n.Items, words.String()) + n.Words = append(n.Words, words) + continue + } + n.Items = append(n.Items, f.name) + } + if more > 0 { + words := core.Says("archivescan.More", "and %s more", core.A("More", core.Exactly(int64(more)))) + n.Items = append(n.Items, words.String()) + n.Words = append(n.Words, words) + } + return n +} + +// add keeps a finding under its key, or counts it under the same key in more +// when the report has no room left to list it. +func (t *tree) add(lists map[string][]found, more map[string]int, id string, f found) { + list, kept := t.keep(lists[id], f) + lists[id] = list + if !kept { + more[id]++ + } +} + +// keep adds a finding to a list a report shows, while the list and the report +// have room for it, and says whether they had. What it keeps of a name is a +// copy of its first MostShownName bytes, never the name itself: a name can be +// a slice of something far larger - archive/tar hands back a name set in a +// PAX header as a slice of the whole header, up to a megabyte. +func (t *tree) keep(list []found, f found) ([]found, bool) { + if len(list) >= MostListed || t.listed >= MostListedBytes { + return list, false + } + if len(f.name) > MostShownName && f.words.IsZero() { + f.words = core.Says("archivescan.Name", "%s", core.A("Name", shownName(f.name))) + } + f.name = cutName(f.name) + t.listed += len(f.name) + len(f.words.String()) + return append(list, f), true +} + +// shownName is a name as a sentence shows it: escaped, and past MostShownName +// bytes cut, with how long it is. Nothing in an archive is named "" - that is +// the one file a gzip file unpacks to. +func shownName(name string) any { + switch { + case name == "": + return core.Says("archivescan.TheFileInside", "the file it unpacks to") + case len(name) <= MostShownName: + return core.Shown(strings.Clone(name)) + } + return core.Says("archivescan.Cut", "%s... (%s in all)", + core.A("Name", core.Shown(cutName(name))), core.A("Length", core.ExactBytes(int64(len(name))))) +} + +// cutName is a copy of the first MostShownName bytes of a name, or of all of +// it when it is shorter, ending on a whole character: a cut through one +// leaves half of it, which --json writes as a character the name does not +// have. A name that is not UTF-8 has no characters to keep whole, so the cut +// goes back no further than one character is long. +func cutName(name string) string { + n := min(len(name), MostShownName) + for back := 1; back < utf8.UTFMax && n < len(name) && !utf8.RuneStart(name[n]); back++ { + n-- + } + return strings.Clone(name[:n]) +} diff --git a/internal/tool/archivescan/sniff.go b/internal/tool/archivescan/sniff.go index b5b29e9c..96b38d8b 100644 --- a/internal/tool/archivescan/sniff.go +++ b/internal/tool/archivescan/sniff.go @@ -20,9 +20,10 @@ const ( const headLength = 512 var ( - zipLocal = []byte("PK\x03\x04") - zipEmpty = []byte("PK\x05\x06") - gzipMagic = []byte{0x1f, 0x8b} + zipLocal = []byte("PK\x03\x04") + zipEmpty = []byte("PK\x05\x06") + // gzipMagic is the two bytes of gzip and its one method, deflate. + gzipMagic = []byte{0x1f, 0x8b, 0x08} tarMagic = []byte("ustar") ) @@ -36,7 +37,6 @@ var others = []struct { {"7z", []byte{'7', 'z', 0xBC, 0xAF, 0x27, 0x1C}}, {"RAR", []byte("Rar!\x1a\x07")}, {"xz", []byte{0xFD, '7', 'z', 'X', 'Z', 0x00}}, - {"bzip2", []byte("BZh")}, {"zstd", []byte{0x28, 0xB5, 0x2F, 0xFD}}, {"CAB", []byte("MSCF")}, } @@ -57,9 +57,23 @@ func sniff(head []byte) (kind, string) { return "", o.name } } + if isBzip2(head) { + return "", "bzip2" + } return "", "" } +// isBzip2 says whether a file begins as bzip2 does: "BZh", a block size from 1 +// to 9, and the magic of a block or of the end of the stream - three letters +// alone begin too many other files. +func isBzip2(head []byte) bool { + if len(head) < 10 || !bytes.HasPrefix(head, []byte("BZh")) || head[3] < '1' || head[3] > '9' { + return false + } + magic := head[4:10] + return bytes.Equal(magic, []byte{0x31, 0x41, 0x59, 0x26, 0x53, 0x59}) || bytes.Equal(magic, []byte{0x17, 0x72, 0x45, 0x38, 0x50, 0x90}) +} + // isTar says whether a block is a TAR header, or the block of zeros an empty // TAR is. POSIX and GNU both write "ustar" at 257. The old V7 layout has no // magic at all and is not told apart from any other file. @@ -71,7 +85,7 @@ func isTar(head []byte) bool { } // archiveIn names the archive a piece of contents begins as, or nothing - for -// an entry holding another archive, which is reported and never opened. +// an entry holding another archive. func archiveIn(head []byte) string { if k, other := sniff(head); other != "" { return other @@ -90,3 +104,8 @@ func allZero(b []byte) bool { } return true } + +// readable says whether an archive named by archiveIn is one this tool reads. +func readable(name string) bool { + return name == string(kindZip) || name == string(kindTar) || name == string(kindGzip) +} diff --git a/internal/tool/archivescan/tar.go b/internal/tool/archivescan/tar.go index 0f97deee..2114cc78 100644 --- a/internal/tool/archivescan/tar.go +++ b/internal/tool/archivescan/tar.go @@ -74,7 +74,7 @@ func (st stream) gzipped(in io.Reader, head []byte) error { } // What follows the end of the TAR, which a TAR program never reads and // gunzip writes all the same. - _, _, err := drain(st.ctx, st.buf, in, ^uint64(0), nil, nil) + _, _, err := drain(st.tree.ctx, st.tree.buf, in, ^uint64(0), nil, nil) return st.ended(err, "") } @@ -82,20 +82,20 @@ func (st stream) gzipped(in io.Reader, head []byte) error { // names what it unpacks after the archive, not after the name inside. func (st stream) one(in io.Reader) error { st.scan.entries = 1 - _, head, err := drain(st.ctx, st.buf, in, ^uint64(0), nil, st.unpacking) + st.tree.entries++ + st.hold.unsaid(st.tree, st.depth < MostDepth) + _, _, err := drain(st.tree.ctx, st.tree.buf, in, ^uint64(0), &st.hold, st.unpacking) if err := st.ended(err, ""); err != nil || st.stopped { + st.hold.drop() return err } - if archiveIn([]byte(head)) != "" { - st.note(noteNested, "") - } - return nil + return st.within("", &st.hold) } // walk reads the entries of a TAR one after another. func (st stream) walk(r io.Reader) error { tr := tar.NewReader(r) - for !st.stopped { + for st.going() { h, err := st.next(tr) if err != nil || h == nil { return err @@ -111,21 +111,29 @@ func (st stream) walk(r io.Reader) error { if h.Size <= 0 { continue } - _, head, err := drain(st.ctx, st.buf, tr, uint64(h.Size), nil, st.unpacking) - if err := st.ended(err, h.Name); err != nil { + if err := st.contents(tr, h); err != nil { return err } - if archiveIn([]byte(head)) != "" { - st.note(noteNested, h.Name) - } } return nil } +// contents reads what an entry holds, and the archive it holds when it holds +// one. +func (st stream) contents(tr *tar.Reader, h *tar.Header) error { + st.hold.start(st.tree, uint64(h.Size), st.depth < MostDepth) + _, _, err := drain(st.tree.ctx, st.tree.buf, tr, uint64(h.Size), &st.hold, st.unpacking) + if err := st.ended(err, h.Name); err != nil || st.stopped { + st.hold.drop() + return err + } + return st.within(h.Name, &st.hold) +} + // next is the header of the next entry, or nothing at the end of the archive // or where a broken one ends the reading. func (st stream) next(tr *tar.Reader) (*tar.Header, error) { - if err := st.ctx.Err(); err != nil { + if err := st.tree.ctx.Err(); err != nil { return nil, err } h, err := tr.Next() @@ -195,7 +203,7 @@ var globalHarmless = map[string]bool{ // Contents and not only a gzip layer, because a sparse entry in a bare TAR // unpacks to its holes too. func (st stream) unpacking(n int) bool { - st.unpacked = addCapped(st.unpacked, uint64(n)) + st.grow(uint64(n)) return !st.explodes(st.unpacked) } @@ -206,8 +214,8 @@ func (st stream) ended(err error, name string) error { switch { case err == nil, st.exploded: return nil - case st.ctx.Err() != nil: - return st.ctx.Err() + case st.tree.ctx.Err() != nil: + return st.tree.ctx.Err() case failedToRead(err): return err } @@ -224,17 +232,19 @@ func (st stream) ended(err error, name string) error { // archive to get there is an archive built to explode - a gigabyte, and more // than a hundred times what was read - and notes it the first time. Reading // stops there: going on is what such an archive is built to make a program do. +// So it does when the archive given, with every archive read inside it, is +// found to be one. func (s *scan) explodes(unpacked uint64) bool { if s.exploded { return true } - if unpacked < BombBytes || unpacked <= uint64(s.read)*BombRatio { - return false + if unpacked >= BombBytes && unpacked > uint64(s.read)*BombRatio { + s.exploded = true + s.noteSaid(noteBomb, "", core.Says("archivescan.Bomb", "%s from %s", core.A("Unpacked", bytesOf(unpacked)), core.A("Size", core.Bytes(s.read)))) + s.stop(core.Says("archivescan.StoppedBomb", "it unpacks to more than %d times what was read of it, and no more is read", core.A("Ratio", BombRatio))) } - s.exploded = true - s.noteSaid(noteBomb, "", core.Says("archivescan.Bomb", "%s from %s", core.A("Unpacked", bytesOf(unpacked)), core.A("Size", core.Bytes(s.read)))) - s.stop(core.Says("archivescan.StoppedBomb", "it unpacks to more than %d times what was read of it, and no more is read", core.A("Ratio", BombRatio))) - return true + s.tree.check(s) + return s.exploded } // errExploded unwinds a reader once the archive has been found built to diff --git a/internal/tool/archivescan/zip.go b/internal/tool/archivescan/zip.go index 51a722a9..241fa3a5 100644 --- a/internal/tool/archivescan/zip.go +++ b/internal/tool/archivescan/zip.go @@ -88,7 +88,7 @@ func (z *zipScan) list() ([]span, error) { if !z.entry(e.name, isFolder(e)) { return false } - z.unpacked = addCapped(z.unpacked, e.unpacked) + z.grow(e.unpacked) at := z.header(e) z.data = append(z.data, at) if at != 0 { @@ -167,7 +167,7 @@ func (z *zipScan) contents() error { index := 0 var failed error err := z.each(false, func(e zipEntry) bool { - if index >= len(z.data) { + if index >= len(z.data) || !z.going() { return false } failed = z.inside(e, index) @@ -196,7 +196,7 @@ func (z *zipScan) inside(e zipEntry, index int) error { target := "" var err error if readable { - target, err = z.read(e, at) + target, _, err = z.read(e, at, false) } z.noteSaid(noteLinks, e.name, linked(e.name, target)) return err @@ -204,47 +204,52 @@ func (z *zipScan) inside(e zipEntry, index int) error { if !readable { return nil } - head, err := z.read(e, at) - if err == nil && archiveIn([]byte(head)) != "" { - z.note(noteNested, e.name) + _, whole, err := z.read(e, at, true) + if err != nil || !whole { + z.hold.drop() + return err } - return err + return z.within(e.name, &z.hold) } // read reads what an entry holds, through its checksum and no further than -// one byte past what the list says it holds, and hands back its first bytes. -// What is wrong with the contents is noted, and only a stop or a disk that -// fails to read is an error. -func (z *zipScan) read(e zipEntry, at uint64) (string, error) { +// one byte past what the list says it holds, and hands back its first bytes +// and whether it read them whole and as the list says - and holds them, when +// asked to look and they are an archive. What is wrong with the contents is +// noted, and only a stop or a disk that fails to read is an error. +func (z *zipScan) read(e zipEntry, at uint64, look bool) (string, bool, error) { switch { case e.flags&1 != 0 || e.method == methodAES: z.noteSaid(noteUnchecked, e.name, entryWhy(e.name, core.Says("archivescan.Encrypted", "it is encrypted"))) - return "", nil + return "", false, nil case e.method != methodStored && e.method != methodDeflated: z.noteSaid(noteUnchecked, e.name, entryWhy(e.name, core.Says("archivescan.Method", "it is packed with method %d, which this tool does not read", core.A("Method", int(e.method))))) - return "", nil + return "", false, nil } var src io.Reader = &counting{r: io.NewSectionReader(z.r, int64(at), int64(e.packed)), s: z.scan} if e.method == methodDeflated { src = z.inflated(src) } sum := crc32.NewIEEE() - got, head, err := drain(z.ctx, z.buf, src, e.unpacked, sum, nil) + z.hold.start(z.tree, e.unpacked, look && z.depth < MostDepth) + got, head, err := drain(z.tree.ctx, z.tree.buf, src, e.unpacked, io.MultiWriter(sum, &z.hold), nil) var corrupt flate.CorruptInputError switch { - case z.ctx.Err() != nil: - return "", z.ctx.Err() + case z.tree.ctx.Err() != nil: + return "", false, z.tree.ctx.Err() case got > e.unpacked: z.note(noteLonger, e.name) case errors.As(err, &corrupt), errors.Is(err, io.ErrUnexpectedEOF), got < e.unpacked: z.broken(e, core.Says("archivescan.EndsEarly", "its contents end before the size the list gives them")) case err != nil: - return "", err + return "", false, err case sum.Sum32() != e.crc: z.broken(e, core.Says("archivescan.Checksum", "its contents do not match their checksum")) + default: + return head, true, nil } - return head, nil + return head, false, nil } // inflated is what deflated contents unpack to, through the archive's one @@ -266,7 +271,7 @@ func (z *zipScan) inflated(src io.Reader) io.Reader { func (z *zipScan) each(first bool, see func(zipEntry) bool) error { in := bufio.NewReaderSize(io.NewSectionReader(z.r, int64(z.end.listAt), int64(z.end.listSize)), 64<<10) for { - if err := z.ctx.Err(); err != nil { + if err := z.tree.ctx.Err(); err != nil { return err } e, size, problem, err := readListed(in)