diff --git a/nzbparser.go b/nzbparser.go index 888d55c..1fa968e 100644 --- a/nzbparser.go +++ b/nzbparser.go @@ -4,9 +4,9 @@ import ( "bytes" "encoding/xml" "fmt" - "html" "io" "sort" + "strings" "golang.org/x/net/html/charset" ) @@ -68,7 +68,7 @@ func (s NzbSegments) Swap(i, j int) { s[i], s[j] = s[j], s[i] } type NzbSegment struct { Bytes int `xml:"bytes,attr"` Number int `xml:"number,attr"` - ID string `xml:",innerxml"` + ID string `xml:",chardata"` // the message-id, XML-decoded and trimmed } // parse nzb file provided as string @@ -95,20 +95,29 @@ func ParseWithOptions(buf io.Reader, opts ParseOptions) (*Nzb, error) { decoder.Strict = false // ignore unknown or malformed character entities if err := decoder.Decode(xnzb); err != nil { - return nil, fmt.Errorf("unable to parse NZB file: %s", err.Error()) + return nil, fmt.Errorf("unable to parse NZB file: %w", err) } // convert to nicer format nzb := new(Nzb) - // copy elements - nzb.Comment = xnzb.Comment + // copy elements. Comment, ids, groups and meta values are trimmed: pretty + // printed NZBs put newlines around them, and Write pads the comment. + nzb.Comment = strings.TrimSpace(xnzb.Comment) nzb.Files = xnzb.Files + for i := range nzb.Files { + for j := range nzb.Files[i].Groups { + nzb.Files[i].Groups[j] = strings.TrimSpace(nzb.Files[i].Groups[j]) + } + for j := range nzb.Files[i].Segments { + nzb.Files[i].Segments[j].ID = strings.TrimSpace(nzb.Files[i].Segments[j].ID) + } + } // convert metadata nzb.Meta = make(map[string]string) for _, md := range xnzb.Metadata { - nzb.Meta[md.Type] = md.Value + nzb.Meta[md.Type] = strings.TrimSpace(md.Value) } // conditionally remove duplicate entries @@ -119,11 +128,12 @@ func ParseWithOptions(buf io.Reader, opts ParseOptions) (*Nzb, error) { // scan the nzb for the additional information ScanNzbFile(nzb) - // sort the files and segments - sort.Sort(nzb.Files) + // sort the files and segments; stable, so files without a number in the + // subject (all 1) and repeated segment numbers keep the NZB's order + sort.Stable(nzb.Files) for id := range nzb.Files { - sort.Sort(nzb.Files[id].Segments) + sort.Stable(nzb.Files[id].Segments) } return nzb, nil @@ -150,9 +160,14 @@ func Write(nzb *Nzb) ([]byte, error) { // add namespace xnzb.Xmlns = Xmlns - // add metadata - for t, v := range nzb.Meta { - xnzb.Metadata = append(xnzb.Metadata, xNzbMeta{Type: t, Value: v}) + // add metadata, in a fixed order (map order made the output differ per run) + types := make([]string, 0, len(nzb.Meta)) + for t := range nzb.Meta { + types = append(types, t) + } + sort.Strings(types) + for _, t := range types { + xnzb.Metadata = append(xnzb.Metadata, xNzbMeta{Type: t, Value: nzb.Meta[t]}) } // Marshal and prepend header @@ -187,6 +202,7 @@ func ScanNzbFile(nzb *Nzb) { } else { nzb.Files[id].Filename = subject.Header } + nzb.Files[id].Basefilename = subject.Basefilename // was never set totalFileSegments = subject.TotalSegments @@ -195,14 +211,13 @@ func ScanNzbFile(nzb *Nzb) { } } - for i, segment := range file.Segments { + for _, segment := range file.Segments { if segment.Number > totalFileSegments { totalFileSegments = segment.Number } totalBytes = totalBytes + int64(segment.Bytes) totalFileBytes = totalFileBytes + int64(segment.Bytes) - nzb.Files[id].Segments[i].ID = html.UnescapeString(segment.ID) } segments = segments + file.Segments.Len() @@ -278,5 +293,5 @@ type xNzb struct { // temp raw meta data for (un)marshalling type xNzbMeta struct { Type string `xml:"type,attr"` - Value string `xml:",innerxml"` + Value string `xml:",chardata"` } diff --git a/review_test.go b/review_test.go new file mode 100644 index 0000000..b0a93ad --- /dev/null +++ b/review_test.go @@ -0,0 +1,131 @@ +package nzbparser + +import ( + "encoding/xml" + "fmt" + "strings" + "testing" +) + +func reviewDoc(head, files string) string { + return Header + `` + head + files + `` +} + +func reviewFile(subject, id string) string { + return `alt.test` + + `` + id + `` +} + +// A parsed NZB written back is well-formed XML with the same ids +// (ids were unescaped after parsing, then written raw). +func TestWriteAfterParseIsWellFormed(t *testing.T) { + nzb, err := ParseString(reviewDoc("", reviewFile(""a.bin" yEnc (1/1)", "part1&x<y>@example.test"))) + if err != nil { + t.Fatal(err) + } + out, err := WriteString(nzb) + if err != nil { + t.Fatal(err) + } + if err := xml.Unmarshal([]byte(out[len(Header):]), new(xNzb)); err != nil { + t.Fatalf("Write output is not well-formed: %v\n%s", err, out) + } + again, err := ParseString(out) + if err != nil { + t.Fatal(err) + } + if got := again.Files[0].Segments[0].ID; got != "part1&x@example.test" { + t.Fatalf("id after a round trip: %q", got) + } +} + +// Pretty-printed NZBs put whitespace around ids, groups and meta values. +func TestWhitespaceIsTrimmed(t *testing.T) { + doc := reviewDoc("\n A Title\n", + "\n alt.test \n"+ + "\n p@example.test\n ") + nzb, err := ParseString(doc) + if err != nil { + t.Fatal(err) + } + if id := nzb.Files[0].Segments[0].ID; id != "p@example.test" { + t.Errorf("id %q", id) + } + if g := nzb.Files[0].Groups[0]; g != "alt.test" { + t.Errorf("group %q", g) + } + if m := nzb.Meta["title"]; m != "A Title" { + t.Errorf("meta %q", m) + } +} + +// A segment count too big for an int counts as missing; the real segments decide. +func TestOverflowingSubjectNumbers(t *testing.T) { + nzb, err := ParseString(reviewDoc("", reviewFile(""b.bin" yEnc (1/99999999999999999999)", "b@example.test"))) + if err != nil { + t.Fatal(err) + } + if nzb.TotalSegments != 1 || nzb.Files[0].TotalSegments != 1 { + t.Fatalf("TotalSegments = %d / %d, want 1", nzb.TotalSegments, nzb.Files[0].TotalSegments) + } +} + +func TestBasefilenameIsSet(t *testing.T) { + nzb, err := ParseString(reviewDoc("", reviewFile("[1/2] "release.part01.rar" yEnc (1/1)", "c@example.test"))) + if err != nil { + t.Fatal(err) + } + if f := nzb.Files[0]; f.Filename != "release.part01.rar" || f.Basefilename != "release" { + t.Fatalf("Filename %q Basefilename %q", f.Filename, f.Basefilename) + } +} + +// Parse → Write → Parse keeps the comment and the output is the same every time. +func TestWriteIsStable(t *testing.T) { + doc := reviewDoc("213", + reviewFile(""d.bin" (1/1)", "d@example.test")) + nzb, err := ParseString(doc) + if err != nil { + t.Fatal(err) + } + first, _ := WriteString(nzb) + for i := 0; i < 20; i++ { + again, err := ParseString(first) + if err != nil { + t.Fatal(err) + } + if again.Comment != "hi" { + t.Fatalf("comment after %d round trips: %q", i+1, again.Comment) + } + out, _ := WriteString(again) + if out != first { + t.Fatalf("Write output changed on round trip %d:\n%s\n---\n%s", i+1, first, out) + } + } +} + +// Files without a number in the subject keep the NZB's order (sort.Sort is not stable). +func TestFilesWithoutNumbersKeepOrder(t *testing.T) { + var files []string + for i := 0; i < 200; i++ { + files = append(files, reviewFile(fmt.Sprintf("post "f%03d.bin" yEnc", i), fmt.Sprintf("m%03d@example.test", i))) + } + nzb, err := ParseString(reviewDoc("", strings.Join(files, ""))) + if err != nil { + t.Fatal(err) + } + for i, f := range nzb.Files { + if want := fmt.Sprintf("f%03d.bin", i); f.Filename != want { + t.Fatalf("file %d is %q, want %q", i, f.Filename, want) + } + } +} + +func BenchmarkParseSubject(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if _, err := ParseSubject(`[1/7] "ubuntu-22.04-live-server-amd64.iso" yEnc (1/2047)`); err != nil { + b.Fatal(err) + } + } +} diff --git a/subject.go b/subject.go index d3a8eef..cd1ae51 100644 --- a/subject.go +++ b/subject.go @@ -6,6 +6,36 @@ import ( "strings" ) +// The regular expressions, compiled once: ParseSubject compiled them on every +// call (~80 µs and ~250 allocations per subject). +var ( + // file and segment numbers, [X/Y] and (X/Y) + reNumbers = regexp.MustCompile(`(?i)(?:(?P.*?) *(?:(?P(?:"?\[|[<[]? *)(?P\d+) */ *(?P\d+) *(?:\]"?|[>\]])?)|(?P"?\((?P\d+) */ *(?P\d+)\)"?))|.*$)`) + // "x of y" file numbers + reXofY = regexp.MustCompile(`(?i)(?:(?P.*?) *(?P(?:\[|[<[]? *(?:file|datei)?) *(?P\d+) *(?:of|von) *(?P\d+) *(?:\]|[>\]])?)(?P.*$))`) + // a filename between quotes, the header before it + reQuotedName = regexp.MustCompile(`(?i)^(?P
.*?)?[- ]*"+(?P(?P.*?)(?:\.(?P(?:7z\.)?(?:vol\d+\+\d+\.par2?|part\d+\.[^ "\.]*|[^ "\.]*\.\d+|[^ "\.]*)))?)"+`) + // an unquoted filename with a known extension (greedy: the last one) + reKnownExt = regexp.MustCompile(`(?i)^(?P(?P.*)\.(?P(?:vol\d+\+\d+\.par2?|part\d+\.rar|r\d{2,3}|mkv|avi|mp4|mov|wmv|flv|webm|m4v|mpg|mpeg|rar|zip|7z|tar|gz|bz2|nfo|sfv|par2?|txt|md|log|jpg|jpeg|png|gif|bmp|svg|pdf|doc|docx|xls|xlsx|ppt|pptx|mp3|flac|m4a|wav|ogg|aac|wma)))(?:\s|$)`) + // every quoted string + reQuoted = regexp.MustCompile(`"([^\"]+)"`) + // a quoted candidate that has an extension + reFileWithExt = regexp.MustCompile(`(?i)^(?P.*?)\.(?:7z\.)?(?:vol\d+\+\d+\.par2?|part\d+\.[^ "\.]*|[^ "\.]*\.\d+|[^ "\.]*?)$`) + // rXX archive volume extensions + reDynArchive = regexp.MustCompile(`(?i)r\d{2}$`) + // a quoted candidate with a real archive/media extension + reCandidate = regexp.MustCompile(`(?i)^(?P.+?)\.(?P(?:vol\d+\+\d+\.par2|par2|part\d+\.rar|r\d{2}|rar|nfo|sfv|zip|7z|mp4|mkv|avi|mov|mp3|flac|m4a|jpg|jpeg|png|gif|pdf|txt))$`) + // a filename in square brackets + reBracketName = regexp.MustCompile(`\[(?P(?P[^\[\]/]+?)\.(?: 7z\.)?(?:vol\d+\+\d+\.par2?|part\d+\.[^\s\"\.\[\]]*|r\d{2,3}|[^\s\"\.\[\]]+))\]`) + // leading number pairs and the rest of the subject + reLeadBrackets = regexp.MustCompile(`^(?: *(?:"?\[|[<[]?)\d+ */ *\d+ *(?:\]"?|[>\]])?)+ *(?P.*)$`) + // a quoted filename in that rest + reQuotedTail = regexp.MustCompile(`(?i)"+(?P(?P.*?)(?:\.(?P(?:7z\.)?(?:vol\d+\+\d+\.par2?|part\d+\.[^ "\.]*|[^ "\.]*\.\d+|[^ "\.]*)))?)"+`) +) + +// extensions the refinement below treats as real file extensions +var knownExt = map[string]struct{}{"rar": {}, "r00": {}, "r01": {}, "r02": {}, "par2": {}, "nfo": {}, "sfv": {}, "zip": {}, "7z": {}, "mp4": {}, "mkv": {}, "avi": {}, "mov": {}, "mp3": {}, "flac": {}, "m4a": {}, "jpg": {}, "jpeg": {}, "png": {}, "gif": {}, "pdf": {}, "txt": {}, "r03": {}, "r04": {}, "r05": {}} + type Subject struct { Subject string // full subject Header string // header (remaining text before the filename) @@ -34,7 +64,7 @@ func ParseSubject(s string) (Subject, error) { // we also assume that max. two number pairs are present (segment numbers or segment and file numbers) // we further assume that if both number pairs are present the last one are the segment numbers, // and that no relevant info apperars in the subject after them (usualy only "yEnc" and the size is stated after the segment numbers) - r := regexp.MustCompile(`(?i)(?:(?P.*?) *(?:(?P(?:"?\[|[<[]? *)(?P\d+) */ *(?P\d+) *(?:\]"?|[>\]])?)|(?P"?\((?P\d+) */ *(?P\d+)\)"?))|.*$)`) + r := reNumbers matches := findAllNamedMatches(r, subject.Subject) foundNumbers := false if matches != nil { @@ -44,12 +74,12 @@ func ParseSubject(s string) (Subject, error) { // if neither segments nor files are assigned yet, we just assign if matches[counter]["files"] != "" { foundNumbers = true - subject.File, _ = strconv.Atoi(matches[counter]["file"]) - subject.TotalFiles, _ = strconv.Atoi(matches[counter]["totalfiles"]) + subject.File = num(matches[counter]["file"]) + subject.TotalFiles = num(matches[counter]["totalfiles"]) } else if matches[counter]["segments"] != "" { // we have either [files] or (segments) in a match but not both foundNumbers = true - subject.Segment, _ = strconv.Atoi(matches[counter]["segment"]) - subject.TotalSegments, _ = strconv.Atoi(matches[counter]["totalsegments"]) + subject.Segment = num(matches[counter]["segment"]) + subject.TotalSegments = num(matches[counter]["totalsegments"]) } } else if subject.TotalFiles == 0 || subject.TotalSegments == 0 { // one of them is already assigned, we do some checks @@ -67,11 +97,11 @@ func ParseSubject(s string) (Subject, error) { // if not, probably only segment numbers are present but use square brackets // in this case we assign the "file" numbers to the segments if subject.TotalSegments == 0 { - subject.Segment, _ = strconv.Atoi(matches[counter]["file"]) - subject.TotalSegments, _ = strconv.Atoi(matches[counter]["totalfiles"]) + subject.Segment = num(matches[counter]["file"]) + subject.TotalSegments = num(matches[counter]["totalfiles"]) } else { - subject.File, _ = strconv.Atoi(matches[counter]["file"]) - subject.TotalFiles, _ = strconv.Atoi(matches[counter]["totalfiles"]) + subject.File = num(matches[counter]["file"]) + subject.TotalFiles = num(matches[counter]["totalfiles"]) } } else if matches[counter]["segments"] != "" { // we have either [files] or (segments) in a match but not both foundNumbers = true @@ -80,11 +110,11 @@ func ParseSubject(s string) (Subject, error) { // in this case we assume that segment numbers are at the end // and assign these "segment" numbers to the files if subject.TotalSegments != 0 { - subject.File, _ = strconv.Atoi(matches[counter]["segment"]) - subject.TotalFiles, _ = strconv.Atoi(matches[counter]["totalsegments"]) + subject.File = num(matches[counter]["segment"]) + subject.TotalFiles = num(matches[counter]["totalsegments"]) } else { - subject.Segment, _ = strconv.Atoi(matches[counter]["segment"]) - subject.TotalSegments, _ = strconv.Atoi(matches[counter]["totalsegments"]) + subject.Segment = num(matches[counter]["segment"]) + subject.TotalSegments = num(matches[counter]["totalsegments"]) } } } @@ -110,11 +140,11 @@ func ParseSubject(s string) (Subject, error) { // check if we have found some file numbers if subject.TotalFiles == 0 { // if no file numbers were found we first try some edge cases like "x of y" - r := regexp.MustCompile(`(?i)(?:(?P.*?) *(?P(?:\[|[<[]? *(?:file|datei)?) *(?P\d+) *(?:of|von) *(?P\d+) *(?:\]|[>\]])?)(?P.*$))`) + r := reXofY matches := findAllNamedMatches(r, remainder) if matches != nil && matches[0]["files"] != "" { - subject.File, _ = strconv.Atoi(matches[0]["file"]) - subject.TotalFiles, _ = strconv.Atoi(matches[0]["totalfiles"]) + subject.File = num(matches[0]["file"]) + subject.TotalFiles = num(matches[0]["totalfiles"]) remainder = strings.TrimSpace(strings.TrimSpace(matches[0]["remainder1"]) + " " + strings.TrimSpace(matches[0]["remainder2"])) } else { // if still nothing was found, we assume it is a single file post @@ -127,7 +157,7 @@ func ParseSubject(s string) (Subject, error) { // we first assume that the filename is between quotes and may or may not end with an extension // we also assume that there is no more relevant information after the filename // everything before the filename is considered to be the header - r = regexp.MustCompile(`(?i)^(?P
.*?)?[- ]*"+(?P(?P.*?)(?:\.(?P(?:7z\.)?(?:vol\d+\+\d+\.par2?|part\d+\.[^ "\.]*|[^ "\.]*\.\d+|[^ "\.]*)))?)"+`) + r = reQuotedName matches = findAllNamedMatches(r, remainder) if matches != nil { subject.Header = strings.Trim(matches[0]["header"], " -") @@ -136,7 +166,7 @@ func ParseSubject(s string) (Subject, error) { } else { // if no filename was found between quotes, we assume the whole remaining subject is the filename and we only search for an extension // Use greedy matching (.*) to prefer the LAST occurrence of a known extension, avoiding false matches on dots in the middle of filenames - r = regexp.MustCompile(`(?i)^(?P(?P.*)\.(?P(?:vol\d+\+\d+\.par2?|part\d+\.rar|r\d{2,3}|mkv|avi|mp4|mov|wmv|flv|webm|m4v|mpg|mpeg|rar|zip|7z|tar|gz|bz2|nfo|sfv|par2?|txt|md|log|jpg|jpeg|png|gif|bmp|svg|pdf|doc|docx|xls|xlsx|ppt|pptx|mp3|flac|m4a|wav|ogg|aac|wma)))(?:\s|$)`) + r = reKnownExt matches = findAllNamedMatches(r, remainder) if matches != nil { subject.Filename = strings.Trim(matches[0]["filename"], " -") @@ -154,11 +184,11 @@ func ParseSubject(s string) (Subject, error) { // Example: [04/23] "Release.Name" - "release.name.r00" - yEnc(1/140) // If we only captured a filename without extension while there are multiple quoted strings, prefer the later quoted string that has an extension. if subject.Filename != "" && subject.Filename == subject.Basefilename { // no extension captured - quoteRE := regexp.MustCompile(`"([^\"]+)"`) + quoteRE := reQuoted quoted := quoteRE.FindAllStringSubmatch(remainder, -1) if len(quoted) > 1 { // pattern to detect filename with extension (reuse existing extension logic) - fileWithExtRE := regexp.MustCompile(`(?i)^(?P.*?)\.(?:7z\.)?(?:vol\d+\+\d+\.par2?|part\d+\.[^ "\.]*|[^ "\.]*\.\d+|[^ "\.]*?)$`) + fileWithExtRE := reFileWithExt for i := 1; i < len(quoted); i++ { // skip first, as it was already treated as filename/header candidate := strings.TrimSpace(quoted[i][1]) m := fileWithExtRE.FindStringSubmatch(candidate) @@ -186,7 +216,7 @@ func ParseSubject(s string) (Subject, error) { // Heuristic: even if we captured an extension, determine if it's a real file extension. If not and multiple quoted strings exist, try to pick a better candidate (e.g. second quoted string with real archive/media extension) if subject.Filename != "" { // attempt refinement - quoteRE := regexp.MustCompile(`"([^\"]+)"`) + quoteRE := reQuoted quoted := quoteRE.FindAllStringSubmatch(remainder, -1) if len(quoted) > 1 { // Extract extension from current filename @@ -196,12 +226,11 @@ func ParseSubject(s string) (Subject, error) { if idx != -1 { curExt = strings.ToLower(cur[idx+1:]) } - known := map[string]struct{}{"rar": {}, "r00": {}, "r01": {}, "r02": {}, "par2": {}, "nfo": {}, "sfv": {}, "zip": {}, "7z": {}, "mp4": {}, "mkv": {}, "avi": {}, "mov": {}, "mp3": {}, "flac": {}, "m4a": {}, "jpg": {}, "jpeg": {}, "png": {}, "gif": {}, "pdf": {}, "txt": {}, "r03": {}, "r04": {}, "r05": {}} - _, curKnown := known[curExt] + _, curKnown := knownExt[curExt] // treat dynamic rXX, partXX.rar as known even if not enumerated - isDynamicArchive := regexp.MustCompile(`(?i)r\d{2}$`).MatchString(curExt) + isDynamicArchive := reDynArchive.MatchString(curExt) if !curKnown && !isDynamicArchive { // try to find better candidate - candidateRE := regexp.MustCompile(`(?i)^(?P.+?)\.(?P(?:vol\d+\+\d+\.par2|par2|part\d+\.rar|r\d{2}|rar|nfo|sfv|zip|7z|mp4|mkv|avi|mov|mp3|flac|m4a|jpg|jpeg|png|gif|pdf|txt))$`) + candidateRE := reCandidate for i := 1; i < len(quoted); i++ { // skip first cand := strings.TrimSpace(quoted[i][1]) if m := candidateRE.FindStringSubmatch(cand); m != nil { @@ -221,7 +250,7 @@ func ParseSubject(s string) (Subject, error) { // Additional handling: some subjects have the filename within square brackets instead of quotes // Example: [PRiVATE]-[WtFnZb]-[test.h264-tripel.r10]-[13/21] - "" yEnc if subject.Filename == "" { - bracketRE := regexp.MustCompile(`\[(?P(?P[^\[\]/]+?)\.(?: 7z\.)?(?:vol\d+\+\d+\.par2?|part\d+\.[^\s\"\.\[\]]*|r\d{2,3}|[^\s\"\.\[\]]+))\]`) + bracketRE := reBracketName bracketMatches := findAllNamedMatches(bracketRE, remainder) if bracketMatches != nil { // Use the last match (typically right before file numbers) @@ -244,7 +273,7 @@ func ParseSubject(s string) (Subject, error) { // Fallback: if we still have no filename but there are quoted parts after leading bracket pairs, attempt to extract. if subject.Filename == "" { - leadBrackets := regexp.MustCompile(`^(?: *(?:"?\[|[<[]?)\d+ */ *\d+ *(?:\]"?|[>\]])?)+ *(?P.*)$`) + leadBrackets := reLeadBrackets if m := leadBrackets.FindStringSubmatch(subject.Subject); m != nil { // identify tail index idxTail := -1 @@ -257,7 +286,7 @@ func ParseSubject(s string) (Subject, error) { if idxTail != -1 { tail := strings.TrimSpace(m[idxTail]) // look for quoted filename in tail - q := regexp.MustCompile(`(?i)"+(?P(?P.*?)(?:\.(?P(?:7z\.)?(?:vol\d+\+\d+\.par2?|part\d+\.[^ "\.]*|[^ "\.]*\.\d+|[^ "\.]*)))?)"+`) + q := reQuotedTail if qm := q.FindStringSubmatch(tail); qm != nil { groups := make(map[string]string) for i, val := range qm { @@ -279,6 +308,17 @@ func ParseSubject(s string) (Subject, error) { } +// num converts a matched number. One that doesn't fit an int counts as +// missing (0): strconv.Atoi returned the largest int, which ScanNzbFile sums +// into TotalSegments and callers used as an allocation size. +func num(s string) int { + n, err := strconv.Atoi(s) + if err != nil { + return 0 + } + return n +} + // helper function for easier handling of named sub matches func findAllNamedMatches(regex *regexp.Regexp, str string) map[int]map[string]string { matches := regex.FindAllStringSubmatch(str, -1)