From cef0d6532acdfb5101ee83de3886565dad207c51 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 19:38:37 +0100 Subject: [PATCH 001/152] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 12f3814..64e335c 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ test*.* *.pem *.key *.crt +.*passwd* bak/* log/ From c81f3fc23b5be10b6f34ddedaeddf9425aac5534 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 19:40:10 +0100 Subject: [PATCH 002/152] add basic Proxy module --- Config.go | 21 +- ConnPool.go | 6 +- Flags.go | 88 +++++ NetConn.go | 63 ++-- Routines.go | 14 +- Server.go | 921 ++++++++++++++++++++++++++++++++++++++++++++++++++++ Session.go | 218 +++++++------ Utils.go | 40 +-- go.mod | 3 +- go.sum | 4 + main.go | 33 +- 11 files changed, 1235 insertions(+), 176 deletions(-) create mode 100644 Server.go diff --git a/Config.go b/Config.go index c268cd1..6e590bb 100644 --- a/Config.go +++ b/Config.go @@ -101,14 +101,19 @@ type CFG struct { DebugFlags bool `json:"DebugFlags"` // if true, enable printing item flags DebugRapidYenc bool `json:"DebugRapidYenc"` // if true, enable rapidyenc debug output //DebugSTREAM bool `json:"DebugSTREAM"` // if true, enable STREAM debug output - Verbose bool `json:"Verbose"` // if true, enable verbose output - Bar bool `json:"Bar"` // if true, show progress bar - Colors bool `json:"Colors"` // if true, enable colored output - MaxArtSize int `json:"MaxArtSize"` // maximum article size in bytes - SloMoC int `json:"SloMoC"` // slow motion for checking articles - SloMoD int `json:"SloMoD"` // slow motion for downloading articles - SloMoU int `json:"SloMoU"` // slow motion for uploading articles - SessThreshold int `json:"SessThreshold"` // max number of sessions a processor keeps open + Verbose bool `json:"Verbose"` // if true, enable verbose output + Bar bool `json:"Bar"` // if true, show progress bar + Colors bool `json:"Colors"` // if true, enable colored output + MaxArtSize int `json:"MaxArtSize"` // maximum article size in bytes + SloMoC int `json:"SloMoC"` // slow motion for checking articles + SloMoD int `json:"SloMoD"` // slow motion for downloading articles + SloMoU int `json:"SloMoU"` // slow motion for uploading articles + SessThreshold int `json:"SessThreshold"` // max number of sessions a processor keeps open + TLSCertPem string `json:"TLSCertPem"` // path to the TLS certificate in PEM format: /etc/letsencrypt/live/sub.domain.com/fullchain.pem + TLSPrivKey string `json:"TLSPrivKey"` // path to the TLS key in PEM format: /etc/letsencrypt/live/sub.domain.com/privkey.pem + ProxyTCP int `json:"ProxyTCP"` // port for the TCP proxy, 0 = no proxy + ProxyTLS int `json:"ProxyTLS"` // port for the TLS proxy, 0 = no proxy + ProxyPasswdFile string `json:"ProxyPasswdFile"` // path to the file with proxy passwords, format: "username|bcryptHash|MaxConns|ExpireAt(unix timestamp)|post" per line } // end CFG struct type Provider struct { diff --git a/ConnPool.go b/ConnPool.go index b4dc0d9..c874b55 100644 --- a/ConnPool.go +++ b/ConnPool.go @@ -72,7 +72,7 @@ type ConnPool struct { nextconnId uint64 // unique connection id //mux sync.RWMutex mux *loggedrwmutex.LoggedSyncRWMutex - counter *Counter_uint64 // used to count connections, parked conns, etc. + Counter *Counter_uint64 // used to count connections, parked conns, etc. pool chan *ConnItem // idle/parked conns are in here poolmap map[uint64]*ConnItem // idle/parked conns are in here //wait []chan *ConnItem @@ -125,7 +125,7 @@ func NewConnPool(s *SESSION, provider *Provider, workerWGconnReady *sync.WaitGro wants_auth := (provider.Username != "" && provider.Password != "") provider.ConnPool = &ConnPool{ - counter: NewCounter(10), // used to count connections, parked conns, etc. + Counter: NewCounter(10), // used to count connections, parked conns, etc. pool: make(chan *ConnItem, provider.MaxConns), poolmap: make(map[uint64]*ConnItem, provider.MaxConns), // use map rserver: rserver, wants_auth: wants_auth, @@ -850,7 +850,7 @@ func Speedmeter(byteSize int64, cp *ConnPool, cnt *Counter_uint64, workerWGconnR case cp == nil && cnt == nil: name, group, counter = "Global", "Speedmeter", GCounter case cp != nil && cnt == nil: - name, group, counter = cp.provider.Name, cp.provider.Group, cp.counter + name, group, counter = cp.provider.Name, cp.provider.Group, cp.Counter case cp == nil && cnt != nil: name, group, counter = "Global", "Speedmeter", cnt default: diff --git a/Flags.go b/Flags.go index ac41c16..c327d6a 100644 --- a/Flags.go +++ b/Flags.go @@ -7,15 +7,20 @@ import ( "fmt" "io" "log" + mrand "math/rand" "os" "path/filepath" "runtime" + "strconv" + "strings" "time" "github.com/go-while/NZBreX/rapidyenc" prof "github.com/go-while/go-cpu-mem-profiler" ) +var AddUserDataToProxy string // AddUserToProxy is a flag to add user to proxy + func ParseFlags() { flag.BoolVar(&version, "version", false, "prints app version") // essentials @@ -80,6 +85,10 @@ func ParseFlags() { flag.IntVar(&cfg.opt.MaxArtSize, "maxartsize", DefaultMaxArticleSize, "limits article size to 1M (mostly articles have ~700K only)") flag.BoolVar(&testmode, "zzz-shr-testmode", false, "[true|false] only used to test compilation on self-hosted runners (default: false)") flag.BoolVar(&testrapidyenc, "testrapidyenc", false, "[true|false] will test rapidyenc testfiles on boot and exit (default: false)") + flag.IntVar(&cfg.opt.ProxyTCP, "proxytcp", 0, "if set: use this port (e.g.: 1119) for TCP proxying (default: 0 = no proxy)") + flag.IntVar(&cfg.opt.ProxyTLS, "proxytls", 0, "if set: use this port (e.g.: 1563) for TLS proxying (default: 0 = no proxy)") + flag.StringVar(&AddUserDataToProxy, "proxyadduser", "", "if set: adds this user to proxy ( \ne.g.: -proxyadduser='username1234|password4567|maxconns|expires'\n ) \nPassword entered MUST be cleartext, hashing is done when writing to .passwd file!\n Expiration: you can use a 'number of days' with 'd' at the end (e.g. 7d, 30d, 365d) or 'h' for hours (1h, 12h, ...) or 'm' minutes (1m, 30m, ...), it will be converted to a valid unixtimestamp expiring in the future from now on! (default: empty = no user added)\nExample -proxyadduser \"HelloWorld|NotAsecurePassword|5|42d\" creates a user with 5 conns and 42 days to expiration") + flag.StringVar(&cfg.opt.ProxyPasswdFile, "proxypasswdfile", ".proxypasswd", "if set: use this file for proxy (default: .proxypasswd in current dir)") // cosmetics: segmentBar needs fixing: only when everything else works! //flag.BoolVar(&cfg.opt.Bar, "bar", false, "show progress bars") // FIXME TODO //flag.BoolVar(&cfg.opt.Colors, "colors", false, "adds colors to s") // FIXME TODO @@ -94,6 +103,81 @@ func ParseFlags() { Prof = prof.NewProf() RunProf() } + if cfg.opt.ProxyPasswdFile == "" { + cfg.opt.ProxyPasswdFile = ".proxypasswd" // default passwd file for proxy + } + if AddUserDataToProxy != "" { + // Format: 'user|password|maxconns|expires|(no)post' (expires is unix timestamp) + parts := strings.SplitN(AddUserDataToProxy, "|", 4) + if len(parts) != 5 { + log.Fatalf("-proxyadduser expects format: user|password|maxconns|expires|(no)post (got: %q)", AddUserDataToProxy) + } + var username, password string + Ausername := strings.TrimSpace(parts[0]) + Apassword := strings.TrimSpace(parts[1]) + if Ausername == "auto" && Apassword == "auto" { + // create auto user with random password + username, password, _ = generateRandomHexCredentials() + } else { + username = Ausername + password = Apassword + } + if len(username) < 10 { + log.Fatalf("-proxyadduser: username must be at least 10 characters long, got %d characters", len(username)) + } + if len(password) < 10 { + log.Fatalf("-proxyadduser: password must be at least 10 characters long, got %d characters", len(password)) + } + maxconns, err1 := strconv.Atoi(parts[2]) + expiresStr := strings.TrimSpace(parts[3]) + var expires int64 + if strings.HasSuffix(expiresStr, "d") { + days, err := strconv.ParseInt(strings.TrimSuffix(expiresStr, "d"), 10, 64) + if err != nil { + log.Fatalf("-adduser: invalid expires (days): %s", expiresStr) + } + expires = time.Now().Add(time.Duration(days) * 24 * time.Hour).Unix() + } else if strings.HasSuffix(expiresStr, "h") { + hours, err := strconv.ParseInt(strings.TrimSuffix(expiresStr, "h"), 10, 64) + if err != nil { + log.Fatalf("-adduser: invalid expires (hours): %s", expiresStr) + } + expires = time.Now().Add(time.Duration(hours) * time.Hour).Unix() + } else if strings.HasSuffix(expiresStr, "m") { + minutes, err := strconv.ParseInt(strings.TrimSuffix(expiresStr, "m"), 10, 64) + if err != nil { + log.Fatalf("-adduser: invalid expires (minutes): %s", expiresStr) + } + expires = time.Now().Add(time.Duration(minutes) * time.Minute).Unix() + } else { + expiresInt, err2 := strconv.ParseInt(parts[3], 10, 64) + if err1 != nil || err2 != nil { + log.Fatalf("-adduser: invalid maxconns or expires: maxconns='%s' expires='%s'", parts[2], parts[3]) + } + expires = expiresInt + } + if username == "" || password == "" { + log.Fatalf("-adduser: username and password must not be empty") + } + if expires < time.Now().Unix() { + log.Fatalf("-adduser: expires must be a future timestamp, got %d", expires) + } + // Compose new passwd file name + newPasswdFile := fmt.Sprintf("%s.new.%d.%d", cfg.opt.ProxyPasswdFile, time.Now().Unix(), mrand.Intn(65535)) + userData := &UserData{ + Username: username, + Password: password, + MaxConns: maxconns, + ExpireAt: expires, + Posting: strings.HasPrefix(strings.ToLower(parts[4]), "post"), + } + if err := addUserToProxyPasswdFile(userData, newPasswdFile); err != nil { + log.Fatalf("Failed to add user to %s: %v", newPasswdFile, err) + os.Exit(1) + } + log.Printf("User '%s' added to new passwd file: '%s'. Next reload in <60s", username, newPasswdFile) + os.Exit(0) + } // test rapidyenc decoder if testrapidyenc { // this is only for testing rapidyenc decoder @@ -218,6 +302,10 @@ func ParseFlags() { os.Exit(1) } + if cfg.opt.ProxyTCP > 0 || cfg.opt.ProxyTLS > 0 { + proxy = true + } + dlog(cfg.opt.Verbose, "Settings: '%#v'", *cfg.opt) } // end func ParseFlags diff --git a/NetConn.go b/NetConn.go index 9137595..bedd201 100644 --- a/NetConn.go +++ b/NetConn.go @@ -47,16 +47,16 @@ var itemReadLineCommands = map[string]struct{}{ } // CMD_STAT checks if the article exists on the server -// and returns the status code or an error if it fails. -func CMD_STAT(connitem *ConnItem, item *segmentChanItem) (int, error) { +// and returns the status code, message and an error if it fails. +func CMD_STAT(connitem *ConnItem, item *segmentChanItem) (int, string, error) { if connitem == nil || connitem.conn == nil || connitem.srvtp == nil { - return 0, fmt.Errorf("error CMD_STAT srvtp=nil") + return 0, "", fmt.Errorf("error CMD_STAT srvtp=nil") } start := time.Now() - id, err := connitem.srvtp.Cmd("STAT <%s>", item.segment.Id) + id, err := connitem.srvtp.Cmd("STAT %s", item.segment.Id) if err != nil { dlog(always, "ERROR checkMessageID @ '%s' srvtp.Cmd err='%v'", connitem.c.provider.Name, err) - return 0, err + return 0, "", err } connitem.srvtp.StartResponse(id) code, msg, err := connitem.srvtp.ReadCodeLine(223) @@ -65,67 +65,72 @@ func CMD_STAT(connitem *ConnItem, item *segmentChanItem) (int, error) { case 223: // article exists... or should! dlog(cfg.opt.DebugSTAT, "CMD_STAT +OK+ seg.Id='%s' @ '%s'", item.segment.Id, connitem.c.provider.Name) - return code, nil + return code, msg, nil case 430: // "430 No Such Article" dlog(cfg.opt.DebugSTAT, "CMD_STAT -NO- seg.Id='%s' @ '%s'", item.segment.Id, connitem.c.provider.Name) - return code, nil + return code, msg, nil case 451: dlog(cfg.opt.DebugSTAT, "CMD_STAT got DMCA code=451 seg.Id='%s' @ '%s' msg='%s'", item.segment.Id, connitem.c.provider.Name, msg) - return code, nil + return code, msg, nil } - return code, fmt.Errorf("error CMD_STAT returned unknown code=%d msg='%s' @ '%s' reqTook='%v' err='%v'", code, msg, connitem.c.provider.Name, time.Since(start), err) + return code, msg, fmt.Errorf("error CMD_STAT returned unknown code=%d msg='%s' @ '%s' reqTook='%v' err='%v'", code, msg, connitem.c.provider.Name, time.Since(start), err) } // end func CMD_STAT // CMD_ARTICLE fetches the article from the server. // It returns the response code, message, size of the article, and any error encountered. // If the article is successfully fetched, it will be stored in item.article. -func CMD_ARTICLE(connitem *ConnItem, item *segmentChanItem) (int, string, uint64, error) { +func CMD(connitem *ConnItem, item *segmentChanItem, command string) (int, string, uint64, error) { if connitem == nil || connitem.conn == nil || connitem.srvtp == nil { - return 0, "", 0, fmt.Errorf("error in CMD_ARTICLE srvtp=nil") + return 0, "", 0, fmt.Errorf("error in CMD srvtp=nil") } - if debugthis { - return 220, "fake article", 1234, nil + rcode := 0 + switch command { + case cmdARTICLE: + rcode = 220 + case cmdHEAD: + rcode = 221 + case cmdBODY: + rcode = 222 } - start := time.Now() - id, aerr := connitem.srvtp.Cmd("ARTICLE <%s>", item.segment.Id) + id, aerr := connitem.srvtp.Cmd("%s %s", command, item.segment.Id) if aerr != nil { - dlog(always, "ERROR in CMD_ARTICLE srvtp.Cmd @ '%s' err='%v'", connitem.c.provider.Name, aerr) + dlog(always, "ERROR in CMD srvtp.Cmd @ '%s' err='%v'", connitem.c.provider.Name, aerr) return 0, "", 0, aerr } connitem.srvtp.StartResponse(id) - code, msg, err := connitem.srvtp.ReadCodeLine(220) + code, msg, err := connitem.srvtp.ReadCodeLine(rcode) connitem.srvtp.EndResponse(id) switch code { - case 220: - // article is coming + case 220, 221, 222: + // article/head/body is coming // old textproto.ReadDotLines replaced with new function: readArticleDotLines // to clean up headers directly while fetching from network // and decoding yenc on the fly - rcode, rxb, _, err := readDotLines(connitem, item, cmdARTICLE) + rcode, rxb, _, err := readDotLines(connitem, item, command) if err != nil { - dlog(always, "ERROR in CMD_ARTICLE srvtp.ReadDotLines @ '%s' err='%v' code=%d rcode=%d", connitem.c.provider.Name, err, code, rcode) + dlog(always, "ERROR in CMD %s srvtp.ReadDotLines @ '%s' err='%v' code=%d rcode=%d", command, connitem.c.provider.Name, err, code, rcode) return code, "", uint64(rxb), err } - dlog(cfg.opt.DebugARTICLE, "CMD_ARTICLE seg.Id='%s' @ '%s' msg='%s' rxb=%d lines=%d code=%d dlcnt=%d fails=%d", item.segment.Id, connitem.c.provider.Name, msg, item.size, len(item.article), code, item.dlcnt, item.fails) + dlog(cfg.opt.DebugARTICLE, "CMD %s seg.Id='%s' @ '%s' msg='%s' rxb=%d lines=%d code=%d dlcnt=%d fails=%d", command, item.segment.Id, connitem.c.provider.Name, msg, item.size, len(item.article), code, item.dlcnt, item.fails) if rcode == 99932 { code = 99932 // bad crc32 } return code, msg, uint64(rxb), nil case 430: - dlog(cfg.opt.DebugARTICLE, "INFO CMD_ARTICLE:430 seg.Id='%s' @ '%s' msg='%s' err='%v' dlcnt=%d fails=%d", item.segment.Id, connitem.c.provider.Name, msg, err, item.dlcnt, item.fails) + dlog(cfg.opt.DebugARTICLE, "INFO CMD %s:430 seg.Id='%s' @ '%s' msg='%s' err='%v' dlcnt=%d fails=%d", command, item.segment.Id, connitem.c.provider.Name, msg, err, item.dlcnt, item.fails) return code, msg, 0, nil // not an error, just no such article case 451: - dlog((cfg.opt.Verbose || cfg.opt.Print430), "INFO CMD_ARTICLE:451 seg.Id='%s' @ '%s' msg='%s' err='%v' dlcnt=%d fails=%d", item.segment.Id, connitem.c.provider.Name, msg, err, item.dlcnt, item.fails) + dlog((cfg.opt.Verbose || cfg.opt.Print430), "INFO CMD %s:451 seg.Id='%s' @ '%s' msg='%s' err='%v' dlcnt=%d fails=%d", command, item.segment.Id, connitem.c.provider.Name, msg, err, item.dlcnt, item.fails) return code, msg, 0, nil // not an error, just DMCA default: // returns the unknown code with an error! } - return code, msg, 0, fmt.Errorf("error in CMD_ARTICLE got unknown code=%d msg='%s' @ '%s' reqTook='%v' err='%v'", code, msg, connitem.c.provider.Name, time.Since(start), err) + return code, msg, 0, fmt.Errorf("error in CMD %s got unknown code=%d msg='%s' @ '%s' reqTook='%v' err='%v'", command, code, msg, connitem.c.provider.Name, time.Since(start), err) } // end func CMD_ARTICLE // CMD_IHAVE sends an article to the server using the IHAVE command. @@ -154,7 +159,7 @@ func CMD_IHAVE(connitem *ConnItem, item *segmentChanItem) (int, string, uint64, * 437 Transfer rejected; do not retry */ wireformat := false // not implemented. read below in: case true - id, err := connitem.srvtp.Cmd("IHAVE <%s>", item.segment.Id) + id, err := connitem.srvtp.Cmd("IHAVE %s", item.segment.Id) if err != nil { dlog(always, "ERROR CMD_IHAVE @ '%s' srvtp.Cmd err='%v'", connitem.c.provider.Name, err) return 0, "", 0, err @@ -880,8 +885,10 @@ readlines: // so we can return the content dlog(cfg.opt.DebugARTICLE, "readDotLines: seg.Id='%s' @ '%s' rxb=%d content=(%d lines)", item.segment.Id, connitem.c.provider.Name, rxb, len(content)) - item.mux.Lock() - defer item.mux.Unlock() + if !proxy && item.mux != nil { + item.mux.Lock() + defer item.mux.Unlock() + } switch what { case cmdARTICLE: diff --git a/Routines.go b/Routines.go index 5388adb..a7f4e4b 100644 --- a/Routines.go +++ b/Routines.go @@ -46,11 +46,11 @@ func (s *SESSION) GoCheckRoutine(wid int, provider *Provider, item *segmentChanI return 0, fmt.Errorf("ERROR in GoCheckRoutine: ConnGet got nil item or conn '%s' connitem='%v' sharedCC='%v' err='%v'", provider.Name, connitem, sharedCC, err) } - code, err = CMD_STAT(connitem, item) + code, msg, err := CMD_STAT(connitem, item) if code == 0 && err != nil { // connection problem, closed? provider.ConnPool.CloseConn(connitem, sharedCC) // close conn on error - dlog(always, "WARN checking seg.Id='%s' failed @ '%s' err='%v'", item.segment.Id, provider.Name, err) + dlog(always, "WARN checking seg.Id='%s' failed @ '%s' code=%d msg='%s' err='%v'", item.segment.Id, provider.Name, code, msg, err) return code, err } @@ -174,7 +174,7 @@ func (s *SESSION) GoDownsRoutine(wid int, provider *Provider, item *segmentChanI dlog(cfg.opt.DebugWorker, "GoDownsRoutine got connitem='%v' sharedCC='%v' --> CMD_ARTICLE seg.Id='%s'", connitem, sharedCC, item.segment.Id) startArticle := time.Now() - code, msg, rxb, err := CMD_ARTICLE(connitem, item) + code, msg, rxb, err := CMD(connitem, item, cmdARTICLE) if err != nil { dlog(always, "ERROR in GoDownsRoutine: CMD_ARTICLE seg.Id='%s' @ '%s'#'%s' err='%v'", item.segment.Id, provider.Name, provider.Group, err) @@ -198,8 +198,8 @@ func (s *SESSION) GoDownsRoutine(wid int, provider *Provider, item *segmentChanI s.counter.Add("TOTAL_RXbytes", uint64(item.size)) // to calulate total download speed of this provider - provider.ConnPool.counter.Add("TMP_RXbytes", uint64(item.size)) - provider.ConnPool.counter.Add("TOTAL_RXbytes", uint64(item.size)) + provider.ConnPool.Counter.Add("TMP_RXbytes", uint64(item.size)) + provider.ConnPool.Counter.Add("TOTAL_RXbytes", uint64(item.size)) // to calulate global total download speed GCounter.Add("TMP_RXbytes", uint64(item.size)) @@ -402,8 +402,8 @@ func (s *SESSION) GoReupsRoutine(wid int, provider *Provider, item *segmentChanI s.counter.Add("TOTAL_TXbytes", uint64(item.size)) // to calulate total upload speed of this provider - provider.ConnPool.counter.Add("TMP_TXbytes", uint64(item.size)) - provider.ConnPool.counter.Add("TOTAL_TXbytes", uint64(item.size)) + provider.ConnPool.Counter.Add("TMP_TXbytes", uint64(item.size)) + provider.ConnPool.Counter.Add("TOTAL_TXbytes", uint64(item.size)) // to calulate global total upload speed GCounter.Add("TMP_TXbytes", uint64(item.size)) diff --git a/Server.go b/Server.go new file mode 100644 index 0000000..e04513d --- /dev/null +++ b/Server.go @@ -0,0 +1,921 @@ +package main + +import ( + "bufio" + "crypto/rand" // Added for random salt generation + "crypto/tls" // Added for TLS support + "strconv" + "time" + + "github.com/Tensai75/nzbparser" + "golang.org/x/crypto/bcrypt" // Added for bcrypt password hashing + + // "encoding/hex" // Uncomment if you want to log/debug hashes as hex strings + "fmt" + "io" + "log" // Added for random delays in password verification (to mitigate timing attacks) + mrand "math/rand" // Added for random delays in password verification (to mitigate timing attacks) + "net" + "net/textproto" // Added for textproto + "os" + "strings" + "sync" +) + +var ( + nntpWelcomeMessage = "ready" // 20x 'NNTP Welcome message' for NNTP clients. x will be set by the server: globalAllowPosting true|false + welcomeCode = 201 // don't change, will be set by server on boot to 200 or 201 depending on globalAllowPosting + // Allow posting by default, can be set to false to disable posting (e.g., for read-only mode) + // this is a global flag. if user/passwd config does not allow posting. user will not be able to post, even if this is true. + globalAllowPosting = true + CID = uint64(0) // Global connection ID counter, can be used for session tracking - currently unused + cidmux = &sync.Mutex{} // cidmux is used to synchronize access to CID + passwdMap = make(map[string]*UserData) // passwdMap holds user credentials (k is username, v is UserData) + ProxySessions = make(map[string]*ProxySession) // ProxySessions map to hold active user sessions (k is username, v is ProxySession) + CountConns = make(map[string]int) // CountConns keeps track of active connections per user (k is username, v is count) + proxyMutex = &sync.RWMutex{} // proxyMutex is used to synchronize access to CID, passwdMap, ProxySessions and CountConns + ProxyParent *SESSION // ProxyParent is the parent session for the proxy, used to link sessions to the main loop +) + +// UserData holds user information. +// When loading from .passwd, Password will be the bcrypt hash string. +// When adding a new user via addUserToPasswdFile, Password should be plaintext to be hashed. +type UserData struct { + Username string + Password string // For loading: bcrypt hash. For adding new user: plaintext. + MaxConns int // Optional: max connections per user, if needed + ExpireAt int64 // Optional: expiration time for the user, if needed (Unix timestamp) + Posting bool // Optional: indicates if the user is allowed to post articles +} + +// ProxySession represents an active user session (currently a placeholder, expand as needed) +type ProxySession struct { + id uint64 // Unique session ID, can be used for tracking + mux sync.RWMutex // Mutex for session data access + Authed bool // Indicates if the user is authenticated + Username string // Username of the authenticated user + Password string // password for the session, can be used for re-authentication + ExpireAt int64 // session expiration time (Unix timestamp) + Conn net.Conn // The user's network connection + Writer *bufio.Writer // bufio writer for the client connection to send articles, headers, bodies, list, xover, xhdr, ... (big data) + tpReader *textproto.Reader // textproto reader for easier command handling + tpWriter *textproto.Writer // textproto writer for easier command handling + CliTp *textproto.Conn // textproto connection for easier command handling + //tmpRXBytes uint64 // proxy has RECEIVED this amount of bytes FROM CLIENT via POST/IHAVE/TAKETHIS in last 60 seconds + RXBytes uint64 // proxy has RECEIVED this amount of bytes FROM CLIENT via POST/IHAVE/TAKETHIS in total this session + tmpTXBytes uint64 // proxy has SENT this amount of bytes TO CLIENT via ARTICLE/HEAD/BODY in last 60 seconds + TXBytes uint64 // proxy has SENT this amount of bytes TO CLIENT via ARTICLE/HEAD/BODY in total this session + ConnectedAt time.Time // Timestamp when the session was created + LastCmd time.Time // Timestamp of the last command received + Group string // current group the user is in (used by GROUP command) + MsgNum int64 // current message number in the group (used by STAT, ARTICLE, etc. commands) + Cron time.Time // last run of periodic tasks, e.g., checking session expiration + // Add other session-specific data here, e.g., current group, article pointer, etc. +} + +// isValidMessageID checks if the provided message ID is valid according to NNTP standards. +func isValidMessageID(ps *ProxySession, messageID string) (isvalid bool, num int64) { + // Placeholder for message ID validation logic + if strings.HasPrefix(messageID, "<") && strings.HasSuffix(messageID, ">") { + // we dont check for @ in the message ID, as it is not required by RFC 5536 + isvalid = true + return + } + number, err := strconv.ParseInt(messageID, 10, 64) + if err == nil && number > 0 { + num = number + // If it's a number, we consider it valid + return + } + log.Printf(" %s | Invalid message ID: %s", ps.Username, messageID) + return +} + +// handleConnection manages a single NNTP client connection. +func handleConnection(conn net.Conn) { + log.Printf("Handling connection from %s", conn.RemoteAddr()) + + // Use textproto.Reader and textproto.Writer + tpReader := textproto.NewReader(bufio.NewReader(conn)) + tpWriter := textproto.NewWriter(bufio.NewWriter(conn)) + + var currentUser string // Stores username after successful AUTHINFO USER + authenticated := false + now := time.Now() // Get the current time for session initialization + var ps = &ProxySession{ + LastCmd: now, // Initialize last command time + ConnectedAt: now, // Set the connection time + } // ProxySession to hold user session data + + // Ensure connection is closed and proxy session cleaned up when done + defer func(s *ProxySession) { + if ps.Authed && s.Username != "" { + proxyMutex.Lock() + if CountConns[s.Username] > 0 { + CountConns[s.Username]-- + log.Printf("Decremented connection count for user '%s'. Active connections: %d", currentUser, CountConns[currentUser]) + } else { + log.Printf("Connection count for user '%s' was already 0 or less, not decrementing. This might indicate an issue.", currentUser) + } + // ProxySessions is used to track this specific session, remove it here. + delete(ProxySessions, s.Username) + proxyMutex.Unlock() + } + //log.Printf("Closed connection for user '%s' from %s", s.Username, conn.RemoteAddr()) + if ps.CliTp != nil { + ps.CliTp.Close() // Close the textproto connection + } else if s.Conn != nil { + ps.Conn.Close() + } + ps.Username = "" // Clear username to avoid dangling pointer + ps.ConnectedAt = time.Time{} // Clear timestamp to avoid dangling pointer + ps.LastCmd = time.Time{} // Clear last command time to avoid dangling pointer + ps = nil // Clear session to avoid dangling pointer + }(ps) + + // Send initial welcome message (RFC 3977: 200 or 201) + // 200 service available, posting allowed + // 201 service available, posting prohibited + time.Sleep(time.Duration(mrand.Intn(128)) * time.Millisecond) // Random delay to simulate server startup + tpWriter.PrintfLine("%d %s", welcomeCode, nntpWelcomeMessage) + // incoming client connection is captured in this for loop until QUIT command or error occurs + tpWriter.W.Flush() // Ensure the welcome message is sent immediately + +forever: + for { + // Read commands from the client in a loop + line, err := tpReader.ReadLine() // Use ReadLine from textproto + if err != nil { + if err != io.EOF { + // Check for common textproto errors, like malformed lines or read errors + if perr, ok := err.(*textproto.Error); ok { + log.Printf("Textproto error from client %s: %v (Code: %d)", conn.RemoteAddr(), perr.Msg, perr.Code) + // You might want to send a specific NNTP error code back to the client here + // For example, if it's a syntax error related to line endings or length. + // tpWriter.PrintfLine("501 Syntax error or line too long") + } else { + log.Printf("Error reading from client %s: %v", conn.RemoteAddr(), err) + } + } else { + log.Printf("Client %s disconnected (EOF).", conn.RemoteAddr()) + } + return + } + + line = strings.TrimSpace(line) // TrimSpace is still useful + parts := strings.Fields(line) + if len(parts) == 0 { + continue + } + var args []string + command := strings.ToUpper(parts[0]) + if len(parts) >= 2 { + args = parts[1:] // Get all parts after the command as arguments + } + + dlog(always, "Client %s command: %s args='%v'", conn.RemoteAddr(), line, args) + ps.LastCmd = time.Now() // Update last command timestamp for unauthenticated users + + if authenticated { + if err := ps.handleRequest(command, args); err != nil { + log.Printf("Error handling command '%s' for user '%s': %v", command, ps.Username, err) + break + } + continue forever // Continue to handle further commands after handling the request + } + + switch command { + + case "CAPABILITIES": + printCapabilities(tpWriter) + + case "AUTHINFO": + if authenticated { + tpWriter.PrintfLine("502 Already authenticated") + break forever + } + if len(parts) < 2 { + tpWriter.PrintfLine("501 Syntax error in AUTHINFO command") + break forever + } + authCmd := strings.ToUpper(parts[1]) + switch authCmd { + + case "USER": + + if len(parts) < 3 { + tpWriter.PrintfLine("501 Syntax error: AUTHINFO USER ") + break + } + currentUser = parts[2] + // RFC 3977 suggests 381 if user is valid, otherwise 481/502 or proceed and fail at PASS. + // To avoid user enumeration, some servers always respond 381. + time.Sleep(time.Duration(mrand.Intn(128)) * time.Millisecond) // Random delay + tpWriter.PrintfLine("381 Password required") + + case "PASS": + + if currentUser == "" { + tpWriter.PrintfLine("482 Authentication commands out of sequence (AUTHINFO USER first)") + break + } + if len(parts) < 3 { + tpWriter.PrintfLine("501 Syntax error: AUTHINFO PASS ") + break + } + time.Sleep(time.Duration(mrand.Intn(128)) * time.Millisecond) // Random delay + passwordToVerify := parts[2] + if verifyPassword(currentUser, passwordToVerify) { + + proxyMutex.Lock() // Lock before checking and updating CountConns and user data + + userData, userExists := passwdMap[currentUser] + if !userExists { + // This case should ideally not be reached if verifyPassword relies on passwdMap + proxyMutex.Unlock() + time.Sleep(time.Duration(mrand.Intn(1000)) * time.Millisecond) // Random delay + tpWriter.PrintfLine("481 Authentication failed (user data inconsistency)") + log.Printf("User data not found for '%s' in passwdMap after successful verifyPassword. Potential data inconsistency.", currentUser) + break + } + + // Check if account is expired + if userData.ExpireAt > 0 && time.Now().Unix() > userData.ExpireAt { + proxyMutex.Unlock() + time.Sleep(time.Duration(mrand.Intn(1000)) * time.Millisecond) // Random delay + tpWriter.PrintfLine("481 Authentication failed (account expired)") + log.Printf("Authentication failed for user '%s' from %s: account expired (ExpireAt: %d, Current: %d)", currentUser, conn.RemoteAddr(), userData.ExpireAt, time.Now().Unix()) + break + } + + // Check connection limit + if userData.MaxConns > 0 && CountConns[currentUser] >= userData.MaxConns { + proxyMutex.Unlock() + time.Sleep(time.Duration(mrand.Intn(1000)) * time.Millisecond) // Random delay + tpWriter.PrintfLine("452 Too many connections for this user. Please try again later.") + log.Printf("Connection denied for user '%s' from %s: too many connections (current: %d, max: %d)", currentUser, conn.RemoteAddr(), CountConns[currentUser], userData.MaxConns) + // Do not set authenticated to true, connection is rejected before full authentication. + break + } + + // Connection allowed, increment count and flag authenticate + CountConns[currentUser]++ + authenticated = true + // Create a new ProxySession for the authenticated user + cidmux.Lock() + CID++ // Increment global connection ID counter + ps.id = CID // Assign a unique session ID + cidmux.Unlock() + ps.Authed = true + ps.Username = currentUser + ps.Password = passwordToVerify // Store the hashed password in the session so we can check every now and then if password has changed and close the session + ps.ExpireAt = userData.ExpireAt // Set session expiration time from user data + ps.Conn = conn // Store the connection in the session + ps.CliTp = textproto.NewConn(conn) // Create a textproto connection for easier command handling + ps.Writer = bufio.NewWriter(conn) // Create a bufio writer for the client connection + ps.tpReader = tpReader // Store the textproto reader in the session + ps.tpWriter = tpWriter // Store the textproto writer in the session + ps.Cron = ps.ConnectedAt // Initialize cron time for periodic tasks + + // Store the session in the global ProxySessions map + ProxySessions[currentUser] = ps + + tpWriter.PrintfLine("281 Welcome to NZBreX Proxy! Your conns: %d/%d. Exp: '%v'", + CountConns[currentUser], userData.MaxConns, time.Unix(userData.ExpireAt, 0).Format(time.RFC1123Z)) + + proxyMutex.Unlock() // Unlock after updating CountConns and user data + + log.Printf("User '%s' authenticated from %s. Active connections for user: %d/%d. ExpireAt: %s", + currentUser, conn.RemoteAddr(), CountConns[currentUser], userData.MaxConns, + time.Unix(userData.ExpireAt, 0).Format(time.RFC1123Z)) // Log expiration time + continue forever // Continue to handle further commands after successful authentication + + } else { + tpWriter.PrintfLine("481 Authentication failed") + log.Printf("Failed authentication attempt for user '%s' from %s", currentUser, conn.RemoteAddr()) + continue forever // Continue to handle further commands after failed authentication + } + + default: + tpWriter.PrintfLine("501 Unknown AUTHINFO subcommand: %s", authCmd) + continue forever // Continue to handle further commands after failed authentication + } + + case "MODE": + if len(parts) < 2 { + tpWriter.PrintfLine("501 Syntax error in MODE command") + break + } + switch strings.ToUpper(parts[1]) { + case "READER": + tpWriter.PrintfLine("201 Posting prohibited") + case "STREAM": + if !authenticated { + tpWriter.PrintfLine("480 Authentication required for MODE STREAM") + break + } + tpWriter.PrintfLine("200 Switching to STREAM mode") + default: + tpWriter.PrintfLine("501 Unknown mode or mode not supported: %s", parts[1]) + } + continue forever // Continue to handle further commands after MODE command + + case "QUIT": + + tpWriter.PrintfLine("205 Closing connection - goodbye.") + log.Printf("Client %s issued QUIT.", conn.RemoteAddr()) + return + + default: + if authenticated { + // Handle other NNTP commands for authenticated users here + // e.g., GROUP, ARTICLE, LIST, NEXT, LAST, DATE, etc. + tpWriter.PrintfLine("500 Unknown command: %s (authenticated)", command) + } else { + // RFC 3977: "480 Authentication required" for most commands if not authenticated + tpWriter.PrintfLine("480 Authentication required") + } + } // end switch command + } // end for forever +} // end func handleConnection + +func (ps *ProxySession) handleRequest(command string, args []string) error { + // Placeholder for handling specific NNTP commands in the session + // This function can be expanded to handle commands like GROUP, ARTICLE, etc. + // For now, we just log the command received. + // returning any error (e.g.: via fmt.Errorf) will disconnect the user + var retry error = nil // retry is used to indicate that the command should be retried + if !ps.Authed || (ps.Username == "") || (ps.CliTp == nil) || ps.ExpireAt < time.Now().Unix() { + ps.CliTp.PrintfLine("480 Authentication required for %s command", command) + return fmt.Errorf("authentication required") + } + log.Printf("Handling command '%s' for user '%s' in session.", command, ps.Username) + // Handle these commands for authenticated users + pass := false + var item *segmentChanItem // segmentChanItem to hold the message ID or number + switch command { //switch command1 + + case "STAT", "ARTICLE", "BODY", "HEAD": + if len(args) == 0 { + ps.CliTp.PrintfLine("501 Syntax error: %s requires a message ID or number", command) + return fmt.Errorf("syntax error: %s requires a message ID or number", command) + } + isvalid, num := isValidMessageID(ps, args[0]) + if !isvalid && num <= 0 { + // protocol error, message ID is not valid + ps.CliTp.PrintfLine("501 Syntax error: command %s requires a valid message ID", command) + return fmt.Errorf("syntax error: command %s requires a valid message ID", command) + } else if num > 0 && ps.Group == "" { + // TODO + ps.tpWriter.PrintfLine("412 No newsgroup selected to read messageid: %d", num) + return retry + + } else if isvalid { + item = &segmentChanItem{ + segment: &nzbparser.NzbSegment{ + Id: args[0], + }, + } + pass = true // we have a valid message ID, so we can pass it to the provider + } + + case "CAPABILITIES": + printCapabilities(ps.tpWriter) + return nil // No error, capabilities printed + case "DATE": + ps.tpWriter.PrintfLine("111 %s", time.Now().Format(time.RFC1123Z)) + return nil // No error, date printed + case "LIST", "XOVER", "XHDR", "GROUP", "NEXT", "LAST": + ps.tpWriter.PrintfLine("500 cmd: %s (not implemented)", command) + return fmt.Errorf("unknown command: %s (not implemented)", command) + case "QUIT": + ps.tpWriter.PrintfLine("205 Closing connection - goodbye. uploaded=%d downloaded=%d connected='%v'", ps.RXBytes, ps.TXBytes, time.Since(ps.ConnectedAt)) + log.Printf("Client %s issued QUIT.", ps.Conn.RemoteAddr()) + time.Sleep(time.Millisecond) // Sleep to ensure the message is sent before closing the connection + return fmt.Errorf("client %s issued QUIT", ps.Conn.RemoteAddr()) + default: + ps.tpWriter.PrintfLine("502 Unknown command") + return fmt.Errorf("unknown command: %s", command) + } // end switch command1 + + if !pass { + ps.tpWriter.PrintfLine("501 Syntax error: command %s requires a valid message ID", command) + return fmt.Errorf("syntax error: command %s requires a valid message ID", command) + } + // Now we have a valid command and item (if applicable), proceed to handle the request + var response string // response to be sent to the client after loopProvider has completed + checkedProviderGroups := make(map[string]bool) +loopProvider: + for _, provider := range ProxyParent.providerList { + if checkedProviderGroups[provider.Group] || provider.NoDownload { + continue + } + // TODO add some caching here for 430 article not found? + + connitem, err := provider.ConnPool.GetConn() // providerconn / proxyconn + if err != nil { + dlog(always, "ERROR GetConn for provider %s: %v", provider.Name, err) + continue loopProvider // Skip this provider if connection fails + } + + dlog(always, " %s | provider %s: %s got pc='%v'", ps.Username, provider.Name, command, connitem) + + // got connection to a provider + switch command { // switch command2 + case "ARTICLE", "BODY", "HEAD": + // Handle commands ARTICLE, BODY, HEAD + code, msg, rxb, err := CMD(connitem, item, command) + if rxb > 0 { + // to calulate total download speed of this provider + // Update provider's RXBytes + provider.ConnPool.Counter.Add("TMP_RXbytes", rxb) + provider.ConnPool.Counter.Add("TOTAL_RXbytes", rxb) + } + if err != nil { + provider.ConnPool.CloseConn(connitem, nil) // Close the connection on error + dlog(always, "ERROR CMD_ARTICLE for provider %s: code=%d msg='%s' err='%v'", provider.Name, code, msg, err) + continue loopProvider + } + switch code { + case 220, 221, 222: // Valid response codes for ARTICLE, HEAD, BODY + // send data to client + txb, err := LinesWriter(ps.Writer, ps.Conn, code, item) // Send data to client + if txb > 0 { + // Update bytes sent to client + ps.tmpTXBytes += txb + ps.TXBytes += txb + } + if err != nil { + provider.ConnPool.CloseConn(connitem, nil) // Close the connection on error + return fmt.Errorf("error writing data command %s to client: %v", command, err) + } + response = "0" // full response already sent, set response to 0 to indicate success + provider.ConnPool.ParkConn(0, connitem, "proxy") // Park the connection for reuse + break loopProvider // Break out of the provider loop after handling the command + + case 423, 430, 451: // messageid not found + checkedProviderGroups[provider.Group] = true + response = fmt.Sprintf("%d %s", code, msg) + provider.ConnPool.ParkConn(0, connitem, "proxy") + continue loopProvider + default: + dlog(always, "ERROR in CMD for provider %s: cmd=%s code=%d msg='%s'", provider.Name, command, code, msg) + response = fmt.Sprintf("502 Unknown Response: %d %s", code, msg) + // Handle other error codes + provider.ConnPool.CloseConn(connitem, nil) + continue loopProvider + } + case "STAT": + // Handle STAT command + code, msg, err := CMD_STAT(connitem, item) + if err != nil { + provider.ConnPool.CloseConn(connitem, nil) // Close the connection on error + dlog(always, "ERROR CMD_STAT for provider %s: err='%v'", provider.Name, err) + continue loopProvider + } + switch code { + case 223: // Article found + response = fmt.Sprintf("%d 0 %s", code, item.segment.Id) + provider.ConnPool.ParkConn(0, connitem, "proxy") // Park the connection for reuse + break loopProvider // Break out of the provider loop after handling the command + + case 423, 430, 451: // messageid not found + checkedProviderGroups[provider.Group] = true + response = fmt.Sprintf("%d nf", code) + provider.ConnPool.ParkConn(0, connitem, "proxy") + continue loopProvider + + default: + response = fmt.Sprintf("%d %s", code, msg) + provider.ConnPool.CloseConn(connitem, nil) + // Handle other error codes + dlog(always, "ERROR CMD_STAT for provider %s: code=%d msg='%s'", provider.Name, code, msg) + continue loopProvider // Continue to the next provider + } + } // end switch command2 + } // end for loopProvider + if response != "0" && response != "" { + ps.tpWriter.PrintfLine("%s", response) + } + return nil // Return nil to indicate the command was handled successfully. an error will disconnect the user +} // end func handleRequest + +func LinesWriter(cliwriter *bufio.Writer, conn net.Conn, code int, item *segmentChanItem) (txb uint64, err error) { + defer func() { + if cliwriter != nil { + err := cliwriter.Flush() // Ensure all buffered data is written to the client + if err != nil { + log.Printf("Error in DotWriter: %v", err) + } + } + }() + var lines *[]string // lines to be sent to the client + switch code { + case 220: + lines = &item.article + case 221: + lines = &item.head + case 222: // Valid response codes for ARTICLE, HEAD, BODY + lines = &item.body + // These codes indicate successful retrieval of article, header, or body + } + // code Num + n, err := io.WriteString(cliwriter, fmt.Sprintf("%d 0 %s", code, item.segment.Id)+CRLF) + txb += uint64(n) + if err != nil { + return txb, fmt.Errorf("error DotWriter WriteString writer @ '%s' err='%v'", conn.RemoteAddr(), err) + } + // Write lines to the client connection using our own buffered dot writer + for _, line := range *lines { + // dot-stuffing when sending to client + if strings.HasPrefix(line, ".") { + line = "." + line // prepend a dot to the line + } + n, err := io.WriteString(cliwriter, line+CRLF) + txb += uint64(n) + if err != nil { + return txb, fmt.Errorf("error DotWriter WriteString writer @ '%s' err='%v'", conn.RemoteAddr(), err) + } + + } + // final sequence + n, err = io.WriteString(cliwriter, DOT+CRLF) + txb += uint64(n) + if err != nil { + return txb, fmt.Errorf("error DotWriter WriteString writer @ '%s' err='%v'", conn.RemoteAddr(), err) + } + return txb, nil +} // end func DataWriter + +// StartNNTPServer initializes and starts the NNTP server. +// addr is the listen address (e.g., ":1119"). +// passwdFilePath is the path to the .passwd file. +// certFile is the path to the TLS certificate file (optional). +// keyFile is the path to the TLS private key file (optional). +func StartNNTPServer(addr string, passwdFilePath string, certFile string, keyFile string) { + time.Sleep(time.Duration(mrand.Intn(128)) * time.Millisecond) // Random delay to simulate server startup + err := loadPasswdFile(passwdFilePath) + if err != nil { + // Decide if server should start if passwd file is missing/corrupt. + // For this example, it logs a warning and continues (no users will be able to auth). + dlog(always, "WARNING: Could not find passwd file '%s': %v. Server starting without users.", passwdFilePath, err) + dlog(always, "To manually add users, use the '-proxyadduser' command or craft a new .passwd file manually") + dlog(always, "Example -proxyadduser \"HelloWorld|NotAsecurePassword|5|42d\" creates a user with 5 conns and 42 days to expiration") + + } else if len(passwdMap) == 0 { + dlog(always, "Warning: Passwd file '%s' loaded, but no users found or all entries were invalid.", passwdFilePath) + } + + var listener net.Listener + + if certFile != "" && keyFile != "" { + dlog(always, "Attempting to start TLS NNTP server on %s", addr) + cer, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + log.Fatalf("Failed to load TLS key pair: %v. Falling back to non-TLS or specify valid cert/key.", err) + // As a fallback, could attempt non-TLS, but for now, we exit if TLS is configured but fails to load. + // To fallback, you might set a flag and then proceed to the non-TLS listener block. + return // Or handle fallback more gracefully + } + + config := &tls.Config{Certificates: []tls.Certificate{cer}} + listener, err = tls.Listen("tcp", addr, config) + if err != nil { + log.Fatalf("Failed to start TLS NNTP server on %s: %v", addr, err) + } + dlog(always, "TLS NNTP server listening on %s", addr) + } else { + dlog(always, "Starting non-TLS NNTP server on %s", addr) + listener, err = net.Listen("tcp", addr) + if err != nil { + log.Fatalf("Failed to start non-TLS NNTP server on %s: %v", addr, err) + } + dlog(always, "Non-TLS NNTP server listening on %s", addr) + } + + defer listener.Close() + + for { + conn, err := listener.Accept() + if err != nil { + // Check if the error is due to the listener being closed. + if opError, ok := err.(*net.OpError); ok && opError.Err.Error() == "use of closed network connection" { + dlog(always, "Listener closed, shutting down accept loop.") + break // Exit loop if listener is closed + } + dlog(always, "Error accepting connection: %v", err) + continue // Continue to try accepting other connections + } + globalmux.RLock() // Lock the global mutex to ensure thread-safe access to ProxyParent + if ProxyParent != nil { + go handleConnection(conn) // Handle each client in a new goroutine + } else { + log.Printf("No ProxyParent available / not booted ... closing connection from %s", conn.RemoteAddr()) + conn.Close() // If ProxyParent is nil, we cannot handle this connection, so close it + } + globalmux.RUnlock() // Unlock the global mutex after checking ProxyParent + } // end for listener.Accept() + dlog(always, "NNTP server on %s stopped.", addr) +} // end func StartNNTPServer + +// StartProxyServers launches the NNTP server on configured TCP and/or TLS ports. +// It uses the global cfg variable for configuration parameters. +func StartProxyServers(appOpt *CFG) { + if appOpt == nil { + dlog(always, "CRITICAL: Application configuration (appOpt) is nil. Cannot start proxy servers.") + return + } + if globalAllowPosting { + welcomeCode = 200 // If posting is allowed, use 200 + } + started := false + + if appOpt.ProxyTCP > 0 { + tcpAddr := fmt.Sprintf(":%d", appOpt.ProxyTCP) + dlog(always, "Attempting to start non-TLS NNTP proxy on port %d", appOpt.ProxyTCP) + go StartNNTPServer(tcpAddr, appOpt.ProxyPasswdFile, "", "") + started = true + } else { + dlog(always, "Non-TLS NNTP proxy (ProxyTCP) not configured or port is 0, skipping.") + } + + if appOpt.ProxyTLS > 0 { + if appOpt.TLSCertPem == "" || appOpt.TLSPrivKey == "" { + dlog(always, "TLS NNTP proxy (ProxyTLS) configured for port %d, but TLSCertPem or TLSPrivKey is missing. Skipping TLS proxy.", appOpt.ProxyTLS) + } else { + tlsAddr := fmt.Sprintf(":%d", appOpt.ProxyTLS) + dlog(always, "Attempting to start TLS NNTP proxy (NNTPS) on port %d", appOpt.ProxyTLS) + go StartNNTPServer(tlsAddr, appOpt.ProxyPasswdFile, appOpt.TLSCertPem, appOpt.TLSPrivKey) + started = true + } + } else { + dlog(always, "TLS NNTP proxy (ProxyTLS) not configured or port is 0, skipping.") + } + + if !started { + dlog(always, "No NNTP proxy servers were started (neither ProxyTCP nor ProxyTLS were configured with valid ports/settings).") + } +} // end func StartProxyServers + +// Passwords in the file are expected to be bcrypt hash strings. +// File format: username:bcrypt_hash:maxconns:ExpireAt_unix_timestamp +func loadPasswdFile(filename string) error { + // Before loading, merge any .new.* files into the main passwd file + base := filename + if idx := strings.LastIndex(filename, "/"); idx != -1 { + base = filename[idx+1:] + } + dir := "." + if idx := strings.LastIndex(filename, "/"); idx != -1 { + dir = filename[:idx] + } + pattern := fmt.Sprintf("%s.new.", base) + entries, err := os.ReadDir(dir) + if err == nil { + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + if strings.HasPrefix(name, pattern) { + newfile := dir + "/" + name + // Append contents to main passwd file + func() { + mainf, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640) + if err != nil { + log.Printf("Could not open %s for appending: %v", filename, err) + return + } + defer mainf.Close() + f, err := os.Open(newfile) + if err != nil { + log.Printf("Could not open %s: %v", newfile, err) + return + } + defer f.Close() + if _, err := io.Copy(mainf, f); err != nil { + log.Printf("Failed to import %s into %s: %v", newfile, filename, err) + } else { + log.Printf("Imported new user(s) from %s into %s", newfile, filename) + os.Remove(newfile) + } + }() + } + } + } + + file, err := os.Open(filename) + if err != nil { + return fmt.Errorf("failed to open passwd file %s: %w", filename, err) + } + defer file.Close() + + proxyMutex.Lock() + defer proxyMutex.Unlock() + + // Clear existing map before loading to support reloading + for k := range passwdMap { + delete(passwdMap, k) + } + + scanner := bufio.NewScanner(file) + lineNumber := 0 + usersLoaded := 0 + newpasswdMap := make(map[string]*UserData) // Temporary map to hold new users + for scanner.Scan() { + lineNumber++ + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { // Skip empty lines and comments + continue + } + parts := strings.SplitN(line, "|", 5) // username|bcrypt_hash|maxconns|ExpireAt|(no)post + if len(parts) != 5 { + log.Printf("Skipping malformed line in passwd file (%d) (expected 5 parts, got %d): %s", lineNumber, len(parts), line) + continue + } + // check the splitted line parts + username := strings.TrimSpace(parts[0]) + bcryptHashFromFile := parts[1] + maxconnsStr := parts[2] + ExpireAtStr := parts[3] + posting := strings.HasPrefix(strings.ToLower(parts[4]), "post") + + maxconns, err := strconv.ParseInt(maxconnsStr, 10, 64) + if err != nil { + log.Printf("Skipping line in passwd file (%d) for user '%s' due to invalid maxconns ('%s'): %v", lineNumber, username, maxconnsStr, err) + continue + } + ExpireAt, err := strconv.ParseInt(ExpireAtStr, 10, 64) + if err != nil { + log.Printf("Skipping line in passwd file (%d) for user '%s' due to invalid ExpireAt ('%s'): %v", lineNumber, username, ExpireAtStr, err) + continue + } + if ExpireAt < time.Now().Unix() { + log.Printf("Skipping line in passwd file (%d) for user '%s' due to expired account (ExpireAt: %d, Current: %d)", lineNumber, username, ExpireAt, time.Now().Unix()) + continue + } + + if len(username) < 10 { + log.Printf("Skipping line in passwd file (%d) due to short username: %s", lineNumber, line) + continue + } + if len(bcryptHashFromFile) < 60 { + // Bcrypt hashes are typically 60 characters long, so we check for that. + // If the hash is empty or too short, we skip this user. + log.Printf("Skipping user '%s' on line in passwd file (%d) due to empty/short password hash.", username, lineNumber) + continue + } + + // check if the username already exists in the map + if _, exists := newpasswdMap[username]; exists { + delete(newpasswdMap, username) // Remove existing entry to avoid duplicates + log.Printf("WARNING: Skipping duplicate user '%s' on line %d in passwd file.", username, lineNumber) + continue // Skip duplicate usernames + } + newpasswdMap[username] = &UserData{ + Username: username, + Password: bcryptHashFromFile, // Store the bcrypt hash directly + MaxConns: int(maxconns), + ExpireAt: ExpireAt, + Posting: posting, + } + usersLoaded++ + } + + if err := scanner.Err(); err != nil { + return fmt.Errorf("error reading passwd file %s: %w", filename, err) + } + + if usersLoaded == 0 { + log.Printf("Warning: No users loaded from passwd file '%s'.", filename) + } else { + passwdMap = newpasswdMap + log.Printf("Successfully loaded %d users from '%s'.", usersLoaded, filename) + } + return nil +} + +// verifyPassword checks if the provided plaintext password matches the stored bcrypt hash for the user. +func verifyPassword(user string, plainPassword string) bool { + proxyMutex.RLock() + userData, ok := passwdMap[user] + proxyMutex.RUnlock() + + if !ok { + return false // User not found + } + time.Sleep(time.Duration(mrand.Intn(128)) * time.Millisecond) // small delay + // userData.Password holds the full bcrypt hash string from the .passwd file + err := bcrypt.CompareHashAndPassword([]byte(userData.Password), []byte(plainPassword)) + return err == nil // If err is nil, the password matches +} + +// addUserToPasswdFile adds a new user to the passwd file with a bcrypt hashed password. +// The UserData.Password field should contain the plaintext password for the new user. +func addUserToProxyPasswdFile(userData *UserData, filename string) error { + if userData.Password == "" { + return fmt.Errorf("cannot add user '%s': plaintext password is empty", userData.Username) + } + + // Generate bcrypt hash from the plaintext password + // bcrypt.DefaultCost is 10. You can increase this for more security (e.g., 12-14), + // but it will also be slower to hash and verify. + hashedPasswordBytes, err := bcrypt.GenerateFromPassword([]byte(userData.Password), bcrypt.DefaultCost) + if err != nil { + return fmt.Errorf("failed to generate bcrypt hash for user '%s': %w", userData.Username, err) + } + bcrpytHashString := string(hashedPasswordBytes) + + // Format the line to be appended to the file + // username:bcrypt_hash_string:maxconns:ExpireAt + postACL := "nopost" // Default ACL for new users, can be changed if needed + if userData.Posting { + postACL = "post" // Set ACL to post if userData.Posting is true + } + newUserLine := fmt.Sprintf("%s|%s|%d|%d|%s\n", userData.Username, bcrpytHashString, userData.MaxConns, userData.ExpireAt, postACL) + + // Open the file in append mode, create if it doesn't exist + file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640) + if err != nil { + return fmt.Errorf("failed to open passwd file '%s' for appending: %w", filename, err) + } + defer file.Close() + + if _, err := file.WriteString(newUserLine); err != nil { + return fmt.Errorf("failed to write new user '%s' to passwd file '%s': %w", userData.Username, filename, err) + } + + log.Printf("Successfully added user '%s' to '%s' with a bcrypt hashed password.", userData.Username, filename) + // Optionally, reload passwdMap or add the new user directly to the in-memory map + // For simplicity here, we assume a restart or separate reload mechanism if immediate in-memory update is needed. + // To update in-memory map immediately: + /* TODO + proxyMutex.Lock() + passwdMap[userData.Username] = &UserData{ + Username: userData.Username, + Password: bcrpytHashString, // Store the newly generated hash + MaxConns: userData.MaxConns, + ExpireAt: userData.ExpireAt, + } + proxyMutex.Unlock() + log.Printf("User '%s' also updated in the in-memory passwdMap.", userData.Username) + */ + return nil +} + +// generateRandomHexCredentials generates a random username and password, each 10 bytes, output as hex strings. +func generateRandomHexCredentials() (username string, password string, err error) { + const length = 8 // 8 bytes will give us 16 hex characters, which is a common length for usernames/passwords + // Use crypto/rand for cryptographically secure random bytes + // If you want longer usernames/passwords, adjust the length accordingly. + // For example, 10 bytes would give 20 hex characters. + buf := make([]byte, length) + if _, err := rand.Read(buf); err != nil { + return "", "", fmt.Errorf("failed to generate random username: %w", err) + } + username = fmt.Sprintf("%x", buf) + if _, err := rand.Read(buf); err != nil { + return "", "", fmt.Errorf("failed to generate random password: %w", err) + } + password = fmt.Sprintf("%x", buf) + return username, password, nil +} + +func printCapabilities(tpWriter *textproto.Writer) { + // Respond with server capabilities (RFC 3977 Section 5.3) + tpWriter.PrintfLine("101 Capability list:") + dw := tpWriter.DotWriter() + fmt.Fprintln(dw, "VERSION 2") // Indicates RFC 3977 support + fmt.Fprintln(dw, "READER") // Indicates MODE READER support + fmt.Fprintln(dw, "AUTHINFO USER PASS") // Supports AUTHINFO USER and AUTHINFO PASS + // Add other capabilities like LIST, IHAVE, POST, etc., as implemented + dw.Close() +} + +func (ps *ProxySession) Close() { + ps.mux.Lock() + defer ps.mux.Unlock() + if ps.CliTp != nil { + ps.CliTp.Close() // Close the textproto connection + ps.CliTp = nil // Clear textproto connection to avoid dangling pointer + } + if ps.Conn != nil { + ps.Conn.Close() // Close the network connection + ps.Conn = nil // Clear the connection to avoid dangling pointer + } + ps.Authed = false // Mark session as unauthenticated + ps.Username = "" // Clear username to avoid dangling pointer + ps.Password = "" // Clear username to avoid dangling pointer + ps.ExpireAt = 0 // Clear expiration time + log.Printf("Closed session for user '%s'", ps.Username) +} + +func (ps *ProxySession) IsExpired() (isExpired bool) { + ps.mux.RLock() + isExpired = ps.ExpireAt > 0 && time.Now().Unix() > ps.ExpireAt + ps.mux.RUnlock() + if isExpired { + ps.Close() // Close the session if it has expired + } + return +} // end func IsExpired diff --git a/Session.go b/Session.go index c41ed33..bb1889e 100644 --- a/Session.go +++ b/Session.go @@ -82,9 +82,10 @@ type SESSION struct { WorkDividerChan chan *WrappedItem // channel to send items to the work divider checkFeedDone bool // checkDone is set to true when the segment feeder has finished feeding to channel, check may still be activly running! segcheckdone bool // segcheckdone is set to true when the segment check is done + proxy bool // flag to signal proxy is used in LaunchSession } // end type SESSION struct -func (p *PROCESSOR) NewProcessor() error { +func (p *PROCESSOR) NewProcessor(proxy bool) error { p.mux = &loggedrwmutex.LoggedSyncRWMutex{Name: "PROCESSOR"} // use a logged sync mutex to log locking and unlocking p.mux.Lock() defer p.mux.Unlock() @@ -107,14 +108,14 @@ func (p *PROCESSOR) NewProcessor() error { } //p.refresh = time.Duration(0 * time.Second) // default. call processor.SetDirRefresh after NewProcessor p.nzbDir = cfg.opt.NzbDir - p.seenFiles = make(map[string]bool, 128) + p.seenFiles = make(map[string]bool) go p.watchDirThread() } - p.sessMap = make(map[uint64]*SESSION, 128) + p.sessMap = make(map[uint64]*SESSION) p.stopChan = make(chan struct{}, 1) go p.processorThread() p.IsRunning = true - dlog(always, "NewProcessor: nzbDir='%s' refresh=%d", p.nzbDir, p.refresh) + dlog(always, "NewProcessor: nzbDir='%s' refresh=%d proxy=%t", p.nzbDir, p.refresh, proxy) return nil } // end func NewProcessor @@ -130,35 +131,41 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s defer waitSession.Done() } - // checks for the correct inputs - if nzbfilepath == "" && s == nil { - // we have no nzbfilepath and no session - // this is a bug in the code, we can't handle both at the same time! - return fmt.Errorf("error LaunchSession: need nzbfilepath or session") - - } else if nzbfilepath != "" && s != nil { - // we have a nzbfilepath and a session - // this is a bug in the code, we can't handle both at the same time! - return fmt.Errorf("error LaunchSession: nzbfilepath and session supplied! can only take one") - - } else if nzbfilepath != "" && s == nil { - // we have a nzbfilepath but no session - // no session supplied, create one! - if sessId, newsession := p.newSession(nzbfilepath); sessId <= 0 || newsession == nil { - return fmt.Errorf("error LaunchSession: sessId <= 0 ?! nzbfilepath='%s' newsession='%#v' err='%v'", nzbfilepath, newsession, err) + if !proxy { + // not booting proxy, no session supplied, create a new session + // checks for the correct inputs + if nzbfilepath == "" && s == nil { + // we have no nzbfilepath and no session + // this is a bug in the code, we can't handle both at the same time! + return fmt.Errorf("error LaunchSession: need nzbfilepath or session") + + } else if nzbfilepath != "" && s != nil { + // we have a nzbfilepath and a session + // this is a bug in the code, we can't handle both at the same time! + return fmt.Errorf("error LaunchSession: nzbfilepath and session supplied! can only take one") + + } else if nzbfilepath != "" && s == nil { + // we have a nzbfilepath but no session + // no session supplied, create one! + if sessId, newsession := p.newSession(nzbfilepath); sessId <= 0 || newsession == nil { + return fmt.Errorf("error LaunchSession: sessId <= 0 ?! nzbfilepath='%s' newsession='%#v' err='%v'", nzbfilepath, newsession, err) + } else { + // we have a new session + dlog(cfg.opt.Verbose, "LaunchSession: created new session with sessId=%d nzbfilepath='%s'", sessId, nzbfilepath) + s = newsession + } + + } else if nzbfilepath == "" && s != nil { + // we have no nzbfilepath but a session + // pass: we have "s" as session! } else { - // we have a new session - dlog(cfg.opt.Verbose, "LaunchSession: created new session with sessId=%d nzbfilepath='%s'", sessId, nzbfilepath) - s = newsession + return fmt.Errorf("error LaunchSession: uncatched bug in launch checks: nzbfilepath='%s' s='%#v'", nzbfilepath, s) } + } // end if !proxy - } else if nzbfilepath == "" && s != nil { - // we have no nzbfilepath but a session - // pass: we have "s" as session! - } else { - return fmt.Errorf("error LaunchSession: uncatched bug in launch checks: nzbfilepath='%s' s='%#v'", nzbfilepath, s) + if s == nil { + return fmt.Errorf("error LaunchSession: session is nil, can't start") } - s.mux.Lock() if s.preBoot || s.active { // we are already booting or active @@ -189,82 +196,90 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s defer SetLogToTerminal() // reset log output to stdout after the session is done } - dlog(cfg.opt.Verbose, "LaunchSession Settings: '%#v'", *cfg.opt) // - s.preparationStartTime = time.Now() - dlog(cfg.opt.Verbose, "LaunchSession Loading NZB: '%s'", s.nzbPath) + if !s.proxy { + dlog(cfg.opt.Verbose, "LaunchSession Settings: '%#v'", *cfg.opt) // + s.preparationStartTime = time.Now() + dlog(cfg.opt.Verbose, "LaunchSession Loading NZB: '%s'", s.nzbPath) - if s.nzbFile != nil { - // nzbfile is still open and loaded, pass + if s.nzbFile != nil { + // nzbfile is still open and loaded, pass - } else if s.nzbFile == nil && nzbfilepath != "" { - // we have a nzbfilepath and no nzbFile loaded yet - // nzbfile has not been opened and parsed before (or has been closed) - nzbfile, err := loadNzbFile(s.nzbPath) - if err != nil || nzbfile == nil { - return fmt.Errorf("error unable to load file s.nzbPath='%s' err=%v'", s.nzbPath, err) - } - s.nzbFile = nzbfile + } else if s.nzbFile == nil && nzbfilepath != "" { + // we have a nzbfilepath and no nzbFile loaded yet + // nzbfile has not been opened and parsed before (or has been closed) + nzbfile, err := loadNzbFile(s.nzbPath) + if err != nil || nzbfile == nil { + return fmt.Errorf("error unable to load file s.nzbPath='%s' err=%v'", s.nzbPath, err) + } + s.nzbFile = nzbfile - dlog(cfg.opt.Debug, "nzbfile='%#v'", s.nzbFile) + dlog(cfg.opt.Debug, "nzbfile='%#v'", s.nzbFile) - // loop through all file tags within the NZB file - for _, file := range nzbfile.Files { - if cfg.opt.Debug { - fmt.Printf(">> nzbfile file='%#v'\n\n", file) - } - s.fileStatLock.Lock() - s.fileStat[file.Filename] = new(fileStatistic) - s.fileStat[file.Filename].available = make(providerStatistic) - s.fileStat[file.Filename].totalSegments = uint64(file.TotalSegments) - s.fileStatLock.Unlock() - // loop through all segment tags within each file tag - if cfg.opt.Debug { - for _, agroup := range file.Groups { - if agroup != "" && !slices.Contains(s.nzbGroups, agroup) { - s.nzbGroups = append(s.nzbGroups, agroup) + // loop through all file tags within the NZB file + for _, file := range nzbfile.Files { + if cfg.opt.Debug { + fmt.Printf(">> nzbfile file='%#v'\n\n", file) + } + s.fileStatLock.Lock() + s.fileStat[file.Filename] = new(fileStatistic) + s.fileStat[file.Filename].available = make(providerStatistic) + s.fileStat[file.Filename].totalSegments = uint64(file.TotalSegments) + s.fileStatLock.Unlock() + // loop through all segment tags within each file tag + if cfg.opt.Debug { + for _, agroup := range file.Groups { + if agroup != "" && !slices.Contains(s.nzbGroups, agroup) { + s.nzbGroups = append(s.nzbGroups, agroup) + } } + dlog(always, "NewsGroups: %v", s.nzbGroups) + } + // filling s.segmentList + for _, segment := range file.Segments { + dlog(cfg.opt.BUG, "append nzb to segmentList: Id='%s' file='%s'", segment.Id, file.Filename) + // If you add more fields to the 'segmentChanItem' struct, the compiler will catch missing initializations here and crash on compilation. + // mux := new(sync.RWMutex) + segmux := &loggedrwmutex.LoggedSyncRWMutex{Name: "segment-" + segment.Id} // use a logged sync mutex to log locking and unlocking + // create a new segmentChanItem for each segment + item := &segmentChanItem{ + segmux, s, &segment, &file, // sync.RWMutex, *SESSION, *nzbparser.Segment, *nzbparser.File + "hashfieldfilllater", // string fields + &s.nzbHashedname, // *string fields + make(chan int, 1), make(chan bool, 1), // chan fields + // map fields for segment status + make(map[int]bool, len(s.providerList)), + make(map[int]bool, len(s.providerList)), + make(map[int]bool, len(s.providerList)), + make(map[int]bool, len(s.providerList)), + make(map[int]bool, len(s.providerList)), + make(map[int]bool, len(s.providerList)), + make(map[int]bool, len(s.providerList)), + make(map[int]bool, len(s.providerList)), + make(map[int]bool, len(s.providerList)), + []string{}, []string{}, []string{}, // []string fields + false, false, false, false, false, false, false, false, false, // bool fields + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} // int fields + if len(segment.Id) < 3 { // a@a + dlog(always, "ERROR LaunchSession: segment.Id='%s' is too short! file='%s'", segment.Id, file.Filename) + continue + } + if !strings.HasPrefix(segment.Id, "<") && !strings.HasSuffix(segment.Id, ">") { + item.segment.Id = "<" + segment.Id + ">" // ensure segment.Id is always in the format "" for consistency + } + item.hashedId = SHA256str(item.segment.Id) + s.segmentList = append(s.segmentList, item) } - dlog(always, "NewsGroups: %v", s.nzbGroups) - } - // filling s.segmentList - for _, segment := range file.Segments { - dlog(cfg.opt.BUG, "append nzb to segmentList: Id='%s' file='%s'", segment.Id, file.Filename) - // If you add more fields to the 'segmentChanItem' struct, the compiler will catch missing initializations here and crash on compilation. - // mux := new(sync.RWMutex) - segmux := &loggedrwmutex.LoggedSyncRWMutex{Name: "segment-" + segment.Id} // use a logged sync mutex to log locking and unlocking - // create a new segmentChanItem for each segment - item := &segmentChanItem{ - segmux, s, &segment, &file, // sync.RWMutex, *SESSION, *nzbparser.Segment, *nzbparser.File - SHA256str("<" + segment.Id + ">"), // string fields - &s.nzbHashedname, // *string fields - make(chan int, 1), make(chan bool, 1), // chan fields - // map fields for segment status - make(map[int]bool, len(s.providerList)), - make(map[int]bool, len(s.providerList)), - make(map[int]bool, len(s.providerList)), - make(map[int]bool, len(s.providerList)), - make(map[int]bool, len(s.providerList)), - make(map[int]bool, len(s.providerList)), - make(map[int]bool, len(s.providerList)), - make(map[int]bool, len(s.providerList)), - make(map[int]bool, len(s.providerList)), - []string{}, []string{}, []string{}, // []string fields - false, false, false, false, false, false, false, false, false, // bool fields - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} // int fields - s.segmentList = append(s.segmentList, item) } - } - - mibsize := float64(nzbfile.Bytes) / 1024 / 1024 - artsize := mibsize / float64(len(s.segmentList)) * 1024 - s.nzbSize = nzbfile.Bytes // store the size of the nzb file in bytes - dlog(always, "%s [%s] loaded NZB: '%s' [%d/%d] ( %.02f MiB | ~%.0f KiB/segment )", appName, appVersion, s.nzbName, len(s.segmentList), nzbfile.TotalSegments, mibsize, artsize) - } // end if s.nzbFile - - // left-padding for log output - s.digStr = fmt.Sprintf("%d", len(s.segmentList)) - s.D = fmt.Sprintf("%d", len(s.digStr)) + mibsize := float64(nzbfile.Bytes) / 1024 / 1024 + artsize := mibsize / float64(len(s.segmentList)) * 1024 + s.nzbSize = nzbfile.Bytes // store the size of the nzb file in bytes + dlog(always, "%s [%s] loaded NZB: '%s' [%d/%d] ( %.02f MiB | ~%.0f KiB/segment )", appName, appVersion, s.nzbName, len(s.segmentList), nzbfile.TotalSegments, mibsize, artsize) + // left-padding for log output + s.digStr = fmt.Sprintf("%d", len(s.segmentList)) + s.D = fmt.Sprintf("%d", len(s.digStr)) + } // end if s.nzbFile + } // end if !s.proxy // re-load the provider list var workerWGconnReady sync.WaitGroup // workerWGconnReady is used to wait for all connections to be established before starting the work @@ -317,7 +332,14 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s } } globalmux.Unlock() - + if s.proxy { + // infinite block here for proxy server. if a wg is supplied it will never be released + _, ok := <-s.stopChan + if !ok { + dlog(always, "LaunchSession: proxy server stopChan closed, exiting LaunchSession") + return + } + } dlog(cfg.opt.Debug, "Loaded s.providerList: %d ... preparation took '%v' | cfg.opt.MemMax=%d totalMaxConns=%d", len(s.providerList), time.Since(s.preparationStartTime).Milliseconds(), cfg.opt.MemMax, totalMaxConns) // setup wait groups diff --git a/Utils.go b/Utils.go index 0ee854f..10fc34c 100644 --- a/Utils.go +++ b/Utils.go @@ -243,12 +243,7 @@ func AppendFileBytes(nullbytes int, dstPath string) error { if err != nil { return err } - defer func() { - cerr := dstFile.Close() - if err == nil && cerr != nil { - err = cerr - } - }() + defer dstFile.Close() nul := make([]byte, nullbytes) for i := 0; i < nullbytes; i++ { nul = append(nul, 0x00) @@ -267,33 +262,26 @@ func AppendFileBytes(nullbytes int, dstPath string) error { // If the source file does not exist, it returns an error. // If the destination file does not exist, it creates a new file. // If the source file is empty, it does nothing. -func AppendFile(srcPath string, dstPath string, delsrc bool) (err error) { +func AppendFile(srcPath string, dstPath string, delsrc bool) error { if srcPath == "" || dstPath == "" { return fmt.Errorf("error AppendFile srcPath='%s' or dstPath='%s' empty", srcPath, dstPath) } + // Open source file for reading srcFile, err := os.Open(srcPath) if err != nil { return err } - defer func() { - cerr := srcFile.Close() - if err == nil && cerr != nil { - err = cerr - } - }() + defer srcFile.Close() + // Open destination file in append mode, create if not exists dstFile, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) if err != nil { return err } - defer func() { - cerr := dstFile.Close() - if err == nil && cerr != nil { - err = cerr - } - }() + defer dstFile.Close() + // Create a buffer and copy in chunks buf := make([]byte, DefaultYencWriteBuffer) for { n, readErr := srcFile.Read(buf) @@ -314,8 +302,8 @@ func AppendFile(srcPath string, dstPath string, delsrc bool) (err error) { return fmt.Errorf("error Yenc AppendFile Remove err='%v'", err) } } - return -} // end func AppendFile (written by AI! GPT-4o, complaint and changed by GPT-4.1!) + return nil +} // end func AppendFile (written by AI! GPT-4o) func SHA256SumFile(path string) (string, error) { // Open the file for reading @@ -348,11 +336,6 @@ func (s *SESSION) writeCsvFile() (err error) { if err != nil { return fmt.Errorf("unable to open csv file: %v", err) } - defer func() { - if cerr := f.Close(); cerr != nil { - log.Printf("ERROR closing file %s: %v", csvFileName, cerr) - } - }() log.Println("writing csv file...") fmt.Print("Writing csv file... ") csvWriter := csv.NewWriter(f) @@ -396,6 +379,7 @@ func (s *SESSION) writeCsvFile() (err error) { if err := csvWriter.Error(); err != nil { return fmt.Errorf("unable to write to the csv file: %v", err) } + f.Close() dlog(cfg.opt.Csv, "writeCsv: done") return } // end func writeCsv @@ -639,6 +623,10 @@ func AdjustMicroSleep(microsleep int64, pushed, todo uint64, lastRunTook time.Du return newSleep } +func fatal() bool { + return true +} + // GetImpossibleCloseCaseVariablesToString returns a string representation of the impossible close case variables. // This is used for debugging purposes to understand the state of the system when an impossible close case occurs. func GetImpossibleCloseCaseVariablesToString(segm, allOk, done, dead, isdl, indl, inup, isup, checked, dmca, nodl, noup, cached, inretry, inyenc, isyenc, dlQ, upQ, yeQ uint64) string { diff --git a/go.mod b/go.mod index da33cc5..e635711 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/crypto v0.39.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) @@ -19,5 +20,5 @@ require ( github.com/gorilla/mux v1.8.1 // indirect github.com/stretchr/testify v1.10.0 golang.org/x/net v0.40.0 // indirect - golang.org/x/text v0.25.0 + golang.org/x/text v0.26.0 ) diff --git a/go.sum b/go.sum index 76db6a2..fad6773 100644 --- a/go.sum +++ b/go.sum @@ -14,10 +14,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/main.go b/main.go index b63d0a8..898659b 100644 --- a/main.go +++ b/main.go @@ -55,6 +55,7 @@ var ( nzbfile string // flag testmode bool // flag: used to test compilation testrapidyenc bool // flag: used to test rapidyenc decoder + proxy bool // flag: used to enable proxy server ) func init() { @@ -75,9 +76,8 @@ func main() { wg := new(sync.WaitGroup) thisProcessor := &PROCESSOR{} go GoMutexStatus() - - if err := thisProcessor.NewProcessor(); err != nil { - dlog(always, "ERROR NewProcessor: err='%v'", err) + if err := thisProcessor.NewProcessor(proxy); err != nil { + dlog(always, "ERROR NewProcessor: proxy err='%v'", err) os.Exit(1) } else { if cfg.opt.NzbDir != "" { @@ -86,8 +86,31 @@ func main() { select {} // infinite wait! } } - - if cfg.opt.NzbDir == "" && nzbfile != "" { + if proxy { + // boot proxy server + wg.Add(1) // waitSession + go func(wg *sync.WaitGroup) { + globalmux.Lock() + // set up a session as like for nzb files + // but only join launch the session to load the providerlist + // to establish connections to the providers and have pools running + dlog(cfg.opt.Debug, "pre:thisProcessor.LaunchSession: proxy server") + s := &SESSION{ + proc: thisProcessor, + mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "PROXYSESSION"}, + proxy: true, + } + ProxyParent = s // set global proxy session so proxy has access to provider list and pools + globalmux.Unlock() + go StartProxyServers(cfg.opt) + if err := thisProcessor.LaunchSession(s, "", wg); err != nil { + dlog(always, "ERROR NewProcessor Boot Proxy Server: err='%v'", err) + os.Exit(1) + } + }(wg) + dlog(cfg.opt.Debug, "main: wg.Wait()") + wg.Wait() + } else if cfg.opt.NzbDir == "" && nzbfile != "" { wg.Add(1) // waitSession go func(nzbfile string, wg *sync.WaitGroup) { dlog(cfg.opt.Debug, "pre:thisProcessor.LaunchSession: nzbfile='%s'", nzbfile) From 60a886db6c019638ddcb83bc3cd873d8e943ff29 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 19:55:32 +0100 Subject: [PATCH 003/152] typo --- Flags.go | 17 ++++++++++------- Server.go | 23 +++++++++++++---------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/Flags.go b/Flags.go index c327d6a..3758c51 100644 --- a/Flags.go +++ b/Flags.go @@ -128,39 +128,42 @@ func ParseFlags() { if len(password) < 10 { log.Fatalf("-proxyadduser: password must be at least 10 characters long, got %d characters", len(password)) } + if Ausername == "auto" && Apassword == "auto" { + dlog(always, "-proxyadduser: auto generated credentials: username='%s' password='%s'", username, password) + } maxconns, err1 := strconv.Atoi(parts[2]) expiresStr := strings.TrimSpace(parts[3]) var expires int64 if strings.HasSuffix(expiresStr, "d") { days, err := strconv.ParseInt(strings.TrimSuffix(expiresStr, "d"), 10, 64) if err != nil { - log.Fatalf("-adduser: invalid expires (days): %s", expiresStr) + log.Fatalf("-proxyadduser: invalid expires (days): %s", expiresStr) } expires = time.Now().Add(time.Duration(days) * 24 * time.Hour).Unix() } else if strings.HasSuffix(expiresStr, "h") { hours, err := strconv.ParseInt(strings.TrimSuffix(expiresStr, "h"), 10, 64) if err != nil { - log.Fatalf("-adduser: invalid expires (hours): %s", expiresStr) + log.Fatalf("-proxyadduser: invalid expires (hours): %s", expiresStr) } expires = time.Now().Add(time.Duration(hours) * time.Hour).Unix() } else if strings.HasSuffix(expiresStr, "m") { minutes, err := strconv.ParseInt(strings.TrimSuffix(expiresStr, "m"), 10, 64) if err != nil { - log.Fatalf("-adduser: invalid expires (minutes): %s", expiresStr) + log.Fatalf("-proxyadduser: invalid expires (minutes): %s", expiresStr) } expires = time.Now().Add(time.Duration(minutes) * time.Minute).Unix() } else { expiresInt, err2 := strconv.ParseInt(parts[3], 10, 64) if err1 != nil || err2 != nil { - log.Fatalf("-adduser: invalid maxconns or expires: maxconns='%s' expires='%s'", parts[2], parts[3]) + log.Fatalf("-proxyadduser: invalid maxconns or expires: maxconns='%s' expires='%s'", parts[2], parts[3]) } expires = expiresInt } if username == "" || password == "" { - log.Fatalf("-adduser: username and password must not be empty") + log.Fatalf("-proxyadduser: username and password must not be empty") } if expires < time.Now().Unix() { - log.Fatalf("-adduser: expires must be a future timestamp, got %d", expires) + log.Fatalf("-proxyadduser: expires must be a future timestamp, got %d", expires) } // Compose new passwd file name newPasswdFile := fmt.Sprintf("%s.new.%d.%d", cfg.opt.ProxyPasswdFile, time.Now().Unix(), mrand.Intn(65535)) @@ -175,7 +178,7 @@ func ParseFlags() { log.Fatalf("Failed to add user to %s: %v", newPasswdFile, err) os.Exit(1) } - log.Printf("User '%s' added to new passwd file: '%s'. Next reload in <60s", username, newPasswdFile) + log.Printf("User '%s':\n %#v \n added to new passwd file: '%s'. Next reload in <60s", username, userData, newPasswdFile) os.Exit(0) } // test rapidyenc decoder diff --git a/Server.go b/Server.go index e04513d..0d902ff 100644 --- a/Server.go +++ b/Server.go @@ -28,13 +28,16 @@ var ( // Allow posting by default, can be set to false to disable posting (e.g., for read-only mode) // this is a global flag. if user/passwd config does not allow posting. user will not be able to post, even if this is true. globalAllowPosting = true - CID = uint64(0) // Global connection ID counter, can be used for session tracking - currently unused - cidmux = &sync.Mutex{} // cidmux is used to synchronize access to CID - passwdMap = make(map[string]*UserData) // passwdMap holds user credentials (k is username, v is UserData) - ProxySessions = make(map[string]*ProxySession) // ProxySessions map to hold active user sessions (k is username, v is ProxySession) - CountConns = make(map[string]int) // CountConns keeps track of active connections per user (k is username, v is count) - proxyMutex = &sync.RWMutex{} // proxyMutex is used to synchronize access to CID, passwdMap, ProxySessions and CountConns - ProxyParent *SESSION // ProxyParent is the parent session for the proxy, used to link sessions to the main loop + + cidmux = &sync.Mutex{} // cidmux is used to synchronize access to CID1 + CID = uint64(0) // Global connection ID counter, can be used for session tracking - currently unused + + proxyMutex = &sync.RWMutex{} // proxyMutex is used to synchronize access to passwdMap, ProxySessions and CountConns + proxyCron *time.Time // MainCron is the main cron job ticker + passwdMap = make(map[string]*UserData) // passwdMap holds user credentials (k is username, v is UserData) + ProxySessions = make(map[string]*ProxySession) // ProxySessions map to hold active user sessions (k is username, v is ProxySession) + CountConns = make(map[string]int) // CountConns keeps track of active connections per user (k is username, v is count) + ProxyParent *SESSION // ProxyParent is the parent session for the proxy, used to link sessions to the main loop ) // UserData holds user information. @@ -828,11 +831,11 @@ func addUserToProxyPasswdFile(userData *UserData, filename string) error { // Format the line to be appended to the file // username:bcrypt_hash_string:maxconns:ExpireAt - postACL := "nopost" // Default ACL for new users, can be changed if needed + posting := "nopost" // Default ACL for new users, can be changed if needed if userData.Posting { - postACL = "post" // Set ACL to post if userData.Posting is true + posting = "post" // Set ACL to post if userData.Posting is true } - newUserLine := fmt.Sprintf("%s|%s|%d|%d|%s\n", userData.Username, bcrpytHashString, userData.MaxConns, userData.ExpireAt, postACL) + newUserLine := fmt.Sprintf("%s|%s|%d|%d|%s\n", userData.Username, bcrpytHashString, userData.MaxConns, userData.ExpireAt, posting) // Open the file in append mode, create if it doesn't exist file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640) From 562e4852eb145081a1378dc268c34880822a543e Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 20:48:09 +0100 Subject: [PATCH 004/152] Update Server.go --- Server.go | 164 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 129 insertions(+), 35 deletions(-) diff --git a/Server.go b/Server.go index 0d902ff..9dab6a5 100644 --- a/Server.go +++ b/Server.go @@ -33,7 +33,7 @@ var ( CID = uint64(0) // Global connection ID counter, can be used for session tracking - currently unused proxyMutex = &sync.RWMutex{} // proxyMutex is used to synchronize access to passwdMap, ProxySessions and CountConns - proxyCron *time.Time // MainCron is the main cron job ticker + proxyCron = time.Now() // reload passwdMap every minute passwdMap = make(map[string]*UserData) // passwdMap holds user credentials (k is username, v is UserData) ProxySessions = make(map[string]*ProxySession) // ProxySessions map to hold active user sessions (k is username, v is ProxySession) CountConns = make(map[string]int) // CountConns keeps track of active connections per user (k is username, v is count) @@ -96,7 +96,7 @@ func isValidMessageID(ps *ProxySession, messageID string) (isvalid bool, num int // handleConnection manages a single NNTP client connection. func handleConnection(conn net.Conn) { - log.Printf("Handling connection from %s", conn.RemoteAddr()) + //log.Printf("Handling connection from %s", conn.RemoteAddr()) // Use textproto.Reader and textproto.Writer tpReader := textproto.NewReader(bufio.NewReader(conn)) @@ -106,8 +106,9 @@ func handleConnection(conn net.Conn) { authenticated := false now := time.Now() // Get the current time for session initialization var ps = &ProxySession{ - LastCmd: now, // Initialize last command time - ConnectedAt: now, // Set the connection time + Conn: conn, // Store the connection in the session + LastCmd: now, // Initialize last command time + ConnectedAt: now, // Set the connection time } // ProxySession to hold user session data // Ensure connection is closed and proxy session cleaned up when done @@ -124,12 +125,15 @@ func handleConnection(conn net.Conn) { delete(ProxySessions, s.Username) proxyMutex.Unlock() } - //log.Printf("Closed connection for user '%s' from %s", s.Username, conn.RemoteAddr()) + if ps.Authed { + dlog(always, "Closed connection for user '%s'", s.Username) + } if ps.CliTp != nil { ps.CliTp.Close() // Close the textproto connection } else if s.Conn != nil { ps.Conn.Close() } + ps.Authed = false // Clear authentication status ps.Username = "" // Clear username to avoid dangling pointer ps.ConnectedAt = time.Time{} // Clear timestamp to avoid dangling pointer ps.LastCmd = time.Time{} // Clear last command time to avoid dangling pointer @@ -152,23 +156,29 @@ forever: if err != io.EOF { // Check for common textproto errors, like malformed lines or read errors if perr, ok := err.(*textproto.Error); ok { - log.Printf("Textproto error from client %s: %v (Code: %d)", conn.RemoteAddr(), perr.Msg, perr.Code) + dlog(always, "Textproto error from client err='%v'", perr) // You might want to send a specific NNTP error code back to the client here // For example, if it's a syntax error related to line endings or length. // tpWriter.PrintfLine("501 Syntax error or line too long") } else { - log.Printf("Error reading from client %s: %v", conn.RemoteAddr(), err) + //DEBUG log.Printf("Error reading from client %s: %v", conn.RemoteAddr(), err) } } else { - log.Printf("Client %s disconnected (EOF).", conn.RemoteAddr()) + //DEBUG log.Printf("Client %s disconnected (EOF).", conn.RemoteAddr()) } return } - + if len(line) > 128 { // only command lines are captured here + // reading a line longer than 128 characters is not allowed by RFC 3977 + // Line is too long, send error response + tpWriter.PrintfLine("501 Syntax error: cmd line too long") + return + } line = strings.TrimSpace(line) // TrimSpace is still useful parts := strings.Fields(line) if len(parts) == 0 { - continue + // just close on an empty line + return } var args []string command := strings.ToUpper(parts[0]) @@ -176,7 +186,8 @@ forever: args = parts[1:] // Get all parts after the command as arguments } - dlog(always, "Client %s command: %s args='%v'", conn.RemoteAddr(), line, args) + //dlog(cfg.opt.Bug, "Client %s command: %s args='%v'", conn.RemoteAddr(), line, args) + ps.LastCmd = time.Now() // Update last command timestamp for unauthenticated users if authenticated { @@ -195,11 +206,11 @@ forever: case "AUTHINFO": if authenticated { tpWriter.PrintfLine("502 Already authenticated") - break forever + return } if len(parts) < 2 { tpWriter.PrintfLine("501 Syntax error in AUTHINFO command") - break forever + return } authCmd := strings.ToUpper(parts[1]) switch authCmd { @@ -208,7 +219,7 @@ forever: if len(parts) < 3 { tpWriter.PrintfLine("501 Syntax error: AUTHINFO USER ") - break + return } currentUser = parts[2] // RFC 3977 suggests 381 if user is valid, otherwise 481/502 or proceed and fail at PASS. @@ -220,18 +231,28 @@ forever: if currentUser == "" { tpWriter.PrintfLine("482 Authentication commands out of sequence (AUTHINFO USER first)") - break + return } if len(parts) < 3 { tpWriter.PrintfLine("501 Syntax error: AUTHINFO PASS ") - break + return } time.Sleep(time.Duration(mrand.Intn(128)) * time.Millisecond) // Random delay passwordToVerify := parts[2] if verifyPassword(currentUser, passwordToVerify) { proxyMutex.Lock() // Lock before checking and updating CountConns and user data - + // reload passwd file + if time.Since(proxyCron) > time.Minute { + proxyCron = time.Now() + if err := loadPasswdFile(cfg.opt.ProxyPasswdFile); err != nil { + proxyMutex.Unlock() // Unlock before sleeping + time.Sleep(time.Duration(mrand.Intn(1000)) * time.Millisecond) // Random delay + log.Printf("Failed to reload passwd file: %v", err) + tpWriter.PrintfLine("481 Authentication failed (passwd file reload error)") + return + } + } userData, userExists := passwdMap[currentUser] if !userExists { // This case should ideally not be reached if verifyPassword relies on passwdMap @@ -239,7 +260,7 @@ forever: time.Sleep(time.Duration(mrand.Intn(1000)) * time.Millisecond) // Random delay tpWriter.PrintfLine("481 Authentication failed (user data inconsistency)") log.Printf("User data not found for '%s' in passwdMap after successful verifyPassword. Potential data inconsistency.", currentUser) - break + return } // Check if account is expired @@ -248,7 +269,7 @@ forever: time.Sleep(time.Duration(mrand.Intn(1000)) * time.Millisecond) // Random delay tpWriter.PrintfLine("481 Authentication failed (account expired)") log.Printf("Authentication failed for user '%s' from %s: account expired (ExpireAt: %d, Current: %d)", currentUser, conn.RemoteAddr(), userData.ExpireAt, time.Now().Unix()) - break + return } // Check connection limit @@ -258,17 +279,19 @@ forever: tpWriter.PrintfLine("452 Too many connections for this user. Please try again later.") log.Printf("Connection denied for user '%s' from %s: too many connections (current: %d, max: %d)", currentUser, conn.RemoteAddr(), CountConns[currentUser], userData.MaxConns) // Do not set authenticated to true, connection is rejected before full authentication. - break + return } // Connection allowed, increment count and flag authenticate CountConns[currentUser]++ authenticated = true + // Create a new ProxySession for the authenticated user cidmux.Lock() CID++ // Increment global connection ID counter ps.id = CID // Assign a unique session ID cidmux.Unlock() + ps.Authed = true ps.Username = currentUser ps.Password = passwordToVerify // Store the hashed password in the session so we can check every now and then if password has changed and close the session @@ -296,30 +319,32 @@ forever: } else { tpWriter.PrintfLine("481 Authentication failed") log.Printf("Failed authentication attempt for user '%s' from %s", currentUser, conn.RemoteAddr()) - continue forever // Continue to handle further commands after failed authentication + return } default: tpWriter.PrintfLine("501 Unknown AUTHINFO subcommand: %s", authCmd) - continue forever // Continue to handle further commands after failed authentication + return } case "MODE": if len(parts) < 2 { tpWriter.PrintfLine("501 Syntax error in MODE command") - break + return } switch strings.ToUpper(parts[1]) { case "READER": tpWriter.PrintfLine("201 Posting prohibited") + case "STREAM": if !authenticated { tpWriter.PrintfLine("480 Authentication required for MODE STREAM") - break + return } tpWriter.PrintfLine("200 Switching to STREAM mode") default: - tpWriter.PrintfLine("501 Unknown mode or mode not supported: %s", parts[1]) + tpWriter.PrintfLine("501 Unknown mode") + return } continue forever // Continue to handle further commands after MODE command @@ -342,6 +367,65 @@ forever: } // end for forever } // end func handleConnection +var articleNotFound = &ArticleNotFound{Map: make(map[string]map[string]*A430)} // Global variable to track articles not found by provider + +// ArticleNotFound is a map to track articles not found by provider (k is provider group, v is map of message IDs) +type ArticleNotFound struct { + mux sync.RWMutex // Mutex to protect access to the map + Map map[string]map[string]*A430 // Map of provider groups to message IDs not found +} + +type A430 struct { + expires time.Time // Expiration time for the A430 article not found +} + +func IsArticleNotFoundAtProviderGroup(messageId string, providerGroup string) bool { + // Check if the article is not found at the provider group + articleNotFound.mux.RLock() + defer articleNotFound.mux.RUnlock() + if providerGroupMap, exists := articleNotFound.Map[providerGroup]; exists { + if a430, found := providerGroupMap[messageId]; found { + if time.Now().Before(a430.expires) { + log.Printf("cache: a430 isflag messageId '%s' not found at provider group '%s'", messageId, providerGroup) + return true // messageId flaged as not found and entry not expired + } + log.Printf("cache: a430 expired messageId '%s' at provider group '%s'", messageId, providerGroup) + go ClearArticleNotFoundAtProviderGroup(messageId, providerGroup) // Clear expired article not found entry + } + } + return false // not cached +} + +func ClearArticleNotFoundAtProviderGroup(messageId string, providerGroup string) { + // Clear the article not found at the provider group + articleNotFound.mux.Lock() + defer articleNotFound.mux.Unlock() + if providerGroupMap, exists := articleNotFound.Map[providerGroup]; exists { + if _, found := providerGroupMap[messageId]; found { + delete(providerGroupMap, messageId) // Remove the article not found entry + log.Printf("Cleared article not found for message ID '%s' at provider group '%s'", messageId, providerGroup) + } else { + log.Printf("Article '%s' not found at provider group '%s' (nothing to clear)", messageId, providerGroup) + } + } else { + log.Printf("Provider group '%s' does not exist in article not found map", providerGroup) + } +} + +func SetArticleNotFoundAtProviderGroup(messageId string, providerGroup string) { + // Set the article not found at the provider group + articleNotFound.mux.Lock() + defer articleNotFound.mux.Unlock() + if _, exists := articleNotFound.Map[providerGroup]; !exists { + articleNotFound.Map[providerGroup] = make(map[string]*A430) + } + articleNotFound.Map[providerGroup][messageId] = &A430{ + expires: time.Now().Add(1 * time.Minute), // Set expiration to 1 minute from now + } + log.Printf("Set article not found for message ID '%s' at provider group '%s'", messageId, providerGroup) +} + +// handleRequest processes NNTP commands for an authenticated user session. func (ps *ProxySession) handleRequest(command string, args []string) error { // Placeholder for handling specific NNTP commands in the session // This function can be expanded to handle commands like GROUP, ARTICLE, etc. @@ -379,7 +463,8 @@ func (ps *ProxySession) handleRequest(command string, args []string) error { Id: args[0], }, } - pass = true // we have a valid message ID, so we can pass it to the provider + pass = true // we have a valid message ID, so we can pass it to a provider + // TODO: add disk caching here? } case "CAPABILITIES": @@ -410,11 +495,13 @@ func (ps *ProxySession) handleRequest(command string, args []string) error { checkedProviderGroups := make(map[string]bool) loopProvider: for _, provider := range ProxyParent.providerList { - if checkedProviderGroups[provider.Group] || provider.NoDownload { - continue + if provider.NoDownload || + checkedProviderGroups[provider.Group] || + IsArticleNotFoundAtProviderGroup(item.segment.Id, provider.Group) { + response = "430 NO" + // Skip this provider if it has already been checked or is not available for download + continue loopProvider } - // TODO add some caching here for 430 article not found? - connitem, err := provider.ConnPool.GetConn() // providerconn / proxyconn if err != nil { dlog(always, "ERROR GetConn for provider %s: %v", provider.Name, err) @@ -460,6 +547,7 @@ loopProvider: checkedProviderGroups[provider.Group] = true response = fmt.Sprintf("%d %s", code, msg) provider.ConnPool.ParkConn(0, connitem, "proxy") + SetArticleNotFoundAtProviderGroup(item.segment.Id, provider.Group) // Set article not found at provider group continue loopProvider default: dlog(always, "ERROR in CMD for provider %s: cmd=%s code=%d msg='%s'", provider.Name, command, code, msg) @@ -472,8 +560,8 @@ loopProvider: // Handle STAT command code, msg, err := CMD_STAT(connitem, item) if err != nil { - provider.ConnPool.CloseConn(connitem, nil) // Close the connection on error dlog(always, "ERROR CMD_STAT for provider %s: err='%v'", provider.Name, err) + provider.ConnPool.CloseConn(connitem, nil) // Close the connection on error continue loopProvider } switch code { @@ -486,6 +574,7 @@ loopProvider: checkedProviderGroups[provider.Group] = true response = fmt.Sprintf("%d nf", code) provider.ConnPool.ParkConn(0, connitem, "proxy") + SetArticleNotFoundAtProviderGroup(item.segment.Id, provider.Group) // Set article not found at provider group continue loopProvider default: @@ -497,8 +586,13 @@ loopProvider: } } // end switch command2 } // end for loopProvider - if response != "0" && response != "" { - ps.tpWriter.PrintfLine("%s", response) + if response != "0" { + if response != "" { + ps.tpWriter.PrintfLine("%s", response) + } else { + dlog(always, " %s | ERROR response to client empty", ps.Username) + ps.tpWriter.PrintfLine("500 Unknown error occurred while processing command %s", command) + } } return nil // Return nil to indicate the command was handled successfully. an error will disconnect the user } // end func handleRequest @@ -537,7 +631,7 @@ func LinesWriter(cliwriter *bufio.Writer, conn net.Conn, code int, item *segment n, err := io.WriteString(cliwriter, line+CRLF) txb += uint64(n) if err != nil { - return txb, fmt.Errorf("error DotWriter WriteString writer @ '%s' err='%v'", conn.RemoteAddr(), err) + return txb, fmt.Errorf("error DotWriter WriteString writer err='%v'", err) } } @@ -545,7 +639,7 @@ func LinesWriter(cliwriter *bufio.Writer, conn net.Conn, code int, item *segment n, err = io.WriteString(cliwriter, DOT+CRLF) txb += uint64(n) if err != nil { - return txb, fmt.Errorf("error DotWriter WriteString writer @ '%s' err='%v'", conn.RemoteAddr(), err) + return txb, fmt.Errorf("error DotWriter WriteString writer err='%v'", err) } return txb, nil } // end func DataWriter From 23926c5f200100f52d457957598829b422c88ee2 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 21:18:11 +0100 Subject: [PATCH 005/152] Update Server.go --- Server.go | 151 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 79 insertions(+), 72 deletions(-) diff --git a/Server.go b/Server.go index 9dab6a5..fbc1ab1 100644 --- a/Server.go +++ b/Server.go @@ -226,6 +226,7 @@ forever: // To avoid user enumeration, some servers always respond 381. time.Sleep(time.Duration(mrand.Intn(128)) * time.Millisecond) // Random delay tpWriter.PrintfLine("381 Password required") + continue forever // Continue to handle further commands after AUTHINFO USER case "PASS": @@ -239,88 +240,94 @@ forever: } time.Sleep(time.Duration(mrand.Intn(128)) * time.Millisecond) // Random delay passwordToVerify := parts[2] - if verifyPassword(currentUser, passwordToVerify) { - - proxyMutex.Lock() // Lock before checking and updating CountConns and user data - // reload passwd file - if time.Since(proxyCron) > time.Minute { - proxyCron = time.Now() - if err := loadPasswdFile(cfg.opt.ProxyPasswdFile); err != nil { - proxyMutex.Unlock() // Unlock before sleeping - time.Sleep(time.Duration(mrand.Intn(1000)) * time.Millisecond) // Random delay - log.Printf("Failed to reload passwd file: %v", err) - tpWriter.PrintfLine("481 Authentication failed (passwd file reload error)") - return - } - } - userData, userExists := passwdMap[currentUser] - if !userExists { - // This case should ideally not be reached if verifyPassword relies on passwdMap - proxyMutex.Unlock() - time.Sleep(time.Duration(mrand.Intn(1000)) * time.Millisecond) // Random delay - tpWriter.PrintfLine("481 Authentication failed (user data inconsistency)") - log.Printf("User data not found for '%s' in passwdMap after successful verifyPassword. Potential data inconsistency.", currentUser) - return - } - // Check if account is expired - if userData.ExpireAt > 0 && time.Now().Unix() > userData.ExpireAt { - proxyMutex.Unlock() - time.Sleep(time.Duration(mrand.Intn(1000)) * time.Millisecond) // Random delay - tpWriter.PrintfLine("481 Authentication failed (account expired)") - log.Printf("Authentication failed for user '%s' from %s: account expired (ExpireAt: %d, Current: %d)", currentUser, conn.RemoteAddr(), userData.ExpireAt, time.Now().Unix()) - return - } + if !verifyPassword(currentUser, passwordToVerify) { + tpWriter.PrintfLine("481 Authentication failed") + log.Printf("Failed authentication attempt for user '%s' from %s", currentUser, conn.RemoteAddr()) + return + } + + proxyMutex.RLock() // Lock before checking and updating CountConns and user data + reloadCron := time.Since(proxyCron) > time.Minute + proxyMutex.RUnlock() - // Check connection limit - if userData.MaxConns > 0 && CountConns[currentUser] >= userData.MaxConns { - proxyMutex.Unlock() + if reloadCron { + proxyMutex.Lock() + proxyCron = time.Now() + proxyMutex.Unlock() + if err := loadPasswdFile(cfg.opt.ProxyPasswdFile); err != nil { + // Unlock before sleeping time.Sleep(time.Duration(mrand.Intn(1000)) * time.Millisecond) // Random delay - tpWriter.PrintfLine("452 Too many connections for this user. Please try again later.") - log.Printf("Connection denied for user '%s' from %s: too many connections (current: %d, max: %d)", currentUser, conn.RemoteAddr(), CountConns[currentUser], userData.MaxConns) - // Do not set authenticated to true, connection is rejected before full authentication. + log.Printf("Failed to reload passwd file: %v", err) + tpWriter.PrintfLine("481 Authentication failed (passwd file reload error)") return } + } - // Connection allowed, increment count and flag authenticate - CountConns[currentUser]++ - authenticated = true - - // Create a new ProxySession for the authenticated user - cidmux.Lock() - CID++ // Increment global connection ID counter - ps.id = CID // Assign a unique session ID - cidmux.Unlock() - - ps.Authed = true - ps.Username = currentUser - ps.Password = passwordToVerify // Store the hashed password in the session so we can check every now and then if password has changed and close the session - ps.ExpireAt = userData.ExpireAt // Set session expiration time from user data - ps.Conn = conn // Store the connection in the session - ps.CliTp = textproto.NewConn(conn) // Create a textproto connection for easier command handling - ps.Writer = bufio.NewWriter(conn) // Create a bufio writer for the client connection - ps.tpReader = tpReader // Store the textproto reader in the session - ps.tpWriter = tpWriter // Store the textproto writer in the session - ps.Cron = ps.ConnectedAt // Initialize cron time for periodic tasks - - // Store the session in the global ProxySessions map - ProxySessions[currentUser] = ps - - tpWriter.PrintfLine("281 Welcome to NZBreX Proxy! Your conns: %d/%d. Exp: '%v'", - CountConns[currentUser], userData.MaxConns, time.Unix(userData.ExpireAt, 0).Format(time.RFC1123Z)) - - proxyMutex.Unlock() // Unlock after updating CountConns and user data + proxyMutex.RLock() - log.Printf("User '%s' authenticated from %s. Active connections for user: %d/%d. ExpireAt: %s", - currentUser, conn.RemoteAddr(), CountConns[currentUser], userData.MaxConns, - time.Unix(userData.ExpireAt, 0).Format(time.RFC1123Z)) // Log expiration time - continue forever // Continue to handle further commands after successful authentication + userData, userExists := passwdMap[currentUser] + if !userExists { + // This case should ideally not be reached if verifyPassword relies on passwdMap + proxyMutex.RUnlock() + time.Sleep(time.Duration(mrand.Intn(1000)) * time.Millisecond) // Random delay + tpWriter.PrintfLine("481 Authentication failed (user data inconsistency)") + log.Printf("User data not found for '%s' in passwdMap after successful verifyPassword. Potential data inconsistency.", currentUser) + return + } - } else { - tpWriter.PrintfLine("481 Authentication failed") - log.Printf("Failed authentication attempt for user '%s' from %s", currentUser, conn.RemoteAddr()) + // Check if account is expired + if userData.ExpireAt > 0 && time.Now().Unix() > userData.ExpireAt { + proxyMutex.RUnlock() + time.Sleep(time.Duration(mrand.Intn(1000)) * time.Millisecond) // Random delay + tpWriter.PrintfLine("481 Authentication failed (account expired)") + log.Printf("Authentication failed for user '%s' from %s: account expired (ExpireAt: %d, Current: %d)", currentUser, conn.RemoteAddr(), userData.ExpireAt, time.Now().Unix()) + return + } + proxyMutex.RUnlock() + + proxyMutex.Lock() + // Check connection limit + if userData.MaxConns > 0 && CountConns[currentUser] >= userData.MaxConns { + proxyMutex.Unlock() + time.Sleep(time.Duration(mrand.Intn(1000)) * time.Millisecond) // Random delay + tpWriter.PrintfLine("452 Too many connections for this user. Please try again later.") + log.Printf("Connection denied for user '%s' from %s: too many connections (current: %d, max: %d)", currentUser, conn.RemoteAddr(), CountConns[currentUser], userData.MaxConns) + // Do not set authenticated to true, connection is rejected before full authentication. return } + // Connection allowed, increment count and flag authenticate + CountConns[currentUser]++ + authenticated = true + + // Create a new ProxySession for the authenticated user + + CID++ // Increment global connection ID counter + ps.id = CID // Assign a unique session ID + ps.Authed = true + ps.Username = currentUser + ps.Password = passwordToVerify // Store the hashed password in the session so we can check every now and then if password has changed and close the session + ps.ExpireAt = userData.ExpireAt // Set session expiration time from user data + ps.Conn = conn // Store the connection in the session + ps.CliTp = textproto.NewConn(conn) // Create a textproto connection for easier command handling + ps.Writer = bufio.NewWriter(conn) // Create a bufio writer for the client connection + ps.tpReader = tpReader // Store the textproto reader in the session + ps.tpWriter = tpWriter // Store the textproto writer in the session + ps.Cron = ps.ConnectedAt // Initialize cron time for periodic tasks + + // Store the session in the global ProxySessions map + ProxySessions[currentUser] = ps + + tpWriter.PrintfLine("281 Welcome to NZBreX Proxy! Your conns: %d/%d. Exp: '%v'", + CountConns[currentUser], userData.MaxConns, time.Unix(userData.ExpireAt, 0).Format(time.RFC1123Z)) + + log.Printf("User '%s' authenticated from %s. Active connections for user: %d/%d. ExpireAt: %s", + currentUser, conn.RemoteAddr(), CountConns[currentUser], userData.MaxConns, + time.Unix(userData.ExpireAt, 0).Format(time.RFC1123Z)) // Log expiration time + + proxyMutex.Unlock() // Unlock after updating CountConns and user data + + continue forever // Continue to handle further commands after successful authentication default: tpWriter.PrintfLine("501 Unknown AUTHINFO subcommand: %s", authCmd) From b7fe69050c4778e2e1db4bed23d195aa3f8ad7ed Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 21:22:11 +0100 Subject: [PATCH 006/152] Update Server.go --- Server.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Server.go b/Server.go index fbc1ab1..0a4b2c8 100644 --- a/Server.go +++ b/Server.go @@ -321,9 +321,8 @@ forever: tpWriter.PrintfLine("281 Welcome to NZBreX Proxy! Your conns: %d/%d. Exp: '%v'", CountConns[currentUser], userData.MaxConns, time.Unix(userData.ExpireAt, 0).Format(time.RFC1123Z)) - log.Printf("User '%s' authenticated from %s. Active connections for user: %d/%d. ExpireAt: %s", - currentUser, conn.RemoteAddr(), CountConns[currentUser], userData.MaxConns, - time.Unix(userData.ExpireAt, 0).Format(time.RFC1123Z)) // Log expiration time + log.Printf("User '%s' authenticated from %s. Active connections for user: %d/%d", + currentUser, conn.RemoteAddr(), CountConns[currentUser], userData.MaxConns) proxyMutex.Unlock() // Unlock after updating CountConns and user data @@ -443,7 +442,7 @@ func (ps *ProxySession) handleRequest(command string, args []string) error { ps.CliTp.PrintfLine("480 Authentication required for %s command", command) return fmt.Errorf("authentication required") } - log.Printf("Handling command '%s' for user '%s' in session.", command, ps.Username) + //dlog(always, "Handling command '%s' for user '%s' in session.", command, ps.Username) // Handle these commands for authenticated users pass := false var item *segmentChanItem // segmentChanItem to hold the message ID or number @@ -515,7 +514,7 @@ loopProvider: continue loopProvider // Skip this provider if connection fails } - dlog(always, " %s | provider %s: %s got pc='%v'", ps.Username, provider.Name, command, connitem) + dlog(cfg.opt.BUG, " %s | provider %s: %s got pc='%v'", ps.Username, provider.Name, command, connitem) // got connection to a provider switch command { // switch command2 From f4b7bb0cd36f2dcf3c6d58ed6a743aeae5b32d90 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 21:33:46 +0100 Subject: [PATCH 007/152] Update NetConn.go --- NetConn.go | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/NetConn.go b/NetConn.go index bedd201..d15cbec 100644 --- a/NetConn.go +++ b/NetConn.go @@ -487,19 +487,23 @@ readlines: return 0, rxb, nil, err } - // found final dot in line, break here - if len(line) == 1 && line == "." { - break - } - - if parseHeader && len(line) == 0 { + if (parseHeader && len(line) == 0) || (len(line) == 1 && line == ".") { // reading header ends here parseHeader = false + if !proxy { + // add new headers for ignored ones + now := time.Now().Format(time.RFC1123Z) + datestr := fmt.Sprintf("Date: %s", now) + content = append(content, datestr) + } else { + //content = append(content, "X-NZBreX: "+appVersion) + } + content = append(content, "Path: not-for-mail") + } - // add new headers for ignored ones - now := time.Now().Format(time.RFC1123Z) - datestr := fmt.Sprintf("Date: %s", now) - content = append(content, datestr) + // found final dot in line, break here + if len(line) == 1 && line == "." { + break } if parseHeader { @@ -516,6 +520,11 @@ readlines: if cfg.opt.CleanHeaders { // ignore headers from cleanHeader slice for _, key := range cleanHeader { + if proxy && key == "Date:" { + // if we are a proxy, we do not want to clean the Date header + // because we want to keep the original Date header from the article + continue + } if strings.HasPrefix(line, key) { ignoreNextContinuedLine = true dlog(cfg.opt.DebugARTICLE, "cleanHeader: seg.ID='%s' ignore key='%s'", item.segment.Id, key) From 862e08129228b677b4d0e4e99dd5af31fd55ba5d Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 21:36:12 +0100 Subject: [PATCH 008/152] Update NetConn.go --- NetConn.go | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/NetConn.go b/NetConn.go index d15cbec..3a0c157 100644 --- a/NetConn.go +++ b/NetConn.go @@ -332,26 +332,26 @@ func CMD_POST(connitem *ConnItem, item *segmentChanItem) (int, string, uint64, e // The function supports parsing headers, handling continued lines, and cleaning headers based on the configuration. // It also supports yenc decoding if enabled in the configuration. // WARNING: TODO! Be careful when reading X/OVER or X/HDR on large groups because you may run out of memory! -func readDotLines(connitem *ConnItem, item *segmentChanItem, what string) (code int, rxb int, content []string, err error) { +func readDotLines(connitem *ConnItem, item *segmentChanItem, command string) (code int, rxb int, content []string, err error) { if connitem.conn == nil || connitem.srvtp == nil { connitem.c.CloseConn(connitem, nil) return 0, 0, nil, fmt.Errorf("error readArticleDotLines: conn or srvtp nil @ '%s'", connitem.c.provider.Name) } - if !IsItemCommand(what) && !IsOtherCommand(what) { + if !IsItemCommand(command) && !IsOtherCommand(command) { // if not an item command or other command! this is a bug! // we just die here because returning an error will f***up the connection as it is already receiving lines from remote - log.Printf("error readArticleDotLines: invalid command '%s'", what) + log.Printf("error readArticleDotLines: invalid command '%s'", command) os.Exit(1) } - if IsOtherCommand(what) && item != nil { + if IsOtherCommand(command) && item != nil { // do not submit an item for other commands! this is a bug! // we just die here because returning an error will f***up the connection as it is already receiving lines from remote - log.Printf("error readArticleDotLines: do not submit an item for command '%s'", what) + log.Printf("error readArticleDotLines: do not submit an item for command '%s'", command) os.Exit(1) } var decoder *yenc.Decoder var parseHeader, ignoreNextContinuedLine, gotYencHeader, gotMultipart, brokenYenc bool // = false, false, false, false - if what == cmdARTICLE || what == cmdHEAD { + if command == cmdARTICLE || command == cmdHEAD { parseHeader = true } async, sentLinesToDecoder := false, 0 // initialize async and sentLinesToDecoder, will be set depending on -yenctest=1,2,3,4 @@ -478,16 +478,16 @@ readlines: // see every line thats coming in //dlog( "readArticleDotLines: seg.Id='%s' line='%s'", segment.Id, line) rxb += len(line) - if IsItemCommand(what) && rxb > cfg.opt.MaxArtSize { + if IsItemCommand(command) && rxb > cfg.opt.MaxArtSize { // max article size reached, stop reading // this is a DoS protection, so we do not read more than maxartsize - err = fmt.Errorf("error readDotLines: maxartsize=%d > rxb=%d seg.Id='%s' what='%s'", cfg.opt.MaxArtSize, rxb, item.segment.Id, what) + err = fmt.Errorf("error readDotLines: maxartsize=%d > rxb=%d seg.Id='%s' what='%s'", cfg.opt.MaxArtSize, rxb, item.segment.Id, command) log.Print(err) connitem.c.CloseConn(connitem, nil) return 0, rxb, nil, err } - if (parseHeader && len(line) == 0) || (len(line) == 1 && line == ".") { + if (parseHeader && len(line) == 0) || (command == cmdHEAD && len(line) == 1 && line == ".") { // reading header ends here parseHeader = false if !proxy { @@ -536,9 +536,9 @@ readlines: content = append(content, line) } // end parseHeader - if !parseHeader && what != cmdHEAD { + if !parseHeader && command != cmdHEAD { i++ // counts body lines - if what == cmdARTICLE || what == cmdBODY { + if command == cmdARTICLE || command == cmdBODY { // dot-stuffing on received lines /* Receiver Side: How to Handle Dot-Stuffing @@ -554,12 +554,12 @@ readlines: } } - if what == cmdARTICLE || what == cmdBODY || IsOtherCommand(what) { + if command == cmdARTICLE || command == cmdBODY || IsOtherCommand(command) { // if we are in ARTICLE or BODY or any other multiline command, we store the line content = append(content, line) } - if cfg.opt.YencCRC && !brokenYenc && (what == cmdARTICLE || what == cmdBODY) { + if cfg.opt.YencCRC && !brokenYenc && (command == cmdARTICLE || command == cmdBODY) { switch cfg.opt.YencTest { case 1: // case 1 needs double the memory @@ -682,9 +682,9 @@ readlines: } else if async && decodeBodyChan != nil { // case 3: close(decodeBodyChan) // close the channel to signal we are done with reading lines } - dlog(cfg.opt.Debug, "readDotLines: seg.Id='%s' rxb=%d content=(%d lines) took=(%d µs) what='%s'", item.segment.Id, rxb, len(content), time.Since(startReadLines).Microseconds(), what) + dlog(cfg.opt.Debug, "readDotLines: seg.Id='%s' rxb=%d content=(%d lines) took=(%d µs) what='%s'", item.segment.Id, rxb, len(content), time.Since(startReadLines).Microseconds(), command) - if cfg.opt.YencCRC && !brokenYenc && (what == cmdARTICLE || what == cmdBODY) { + if cfg.opt.YencCRC && !brokenYenc && (command == cmdARTICLE || command == cmdBODY) { yencstart := time.Now() var startReadSignals time.Time var isBadCrc bool @@ -899,7 +899,7 @@ readlines: defer item.mux.Unlock() } - switch what { + switch command { case cmdARTICLE: item.article = content item.size = rxb @@ -912,9 +912,9 @@ readlines: item.headsize = rxb default: // handle multi-line dot-terminated command - if !IsOtherCommand(what) { + if !IsOtherCommand(command) { // error unknown command - return 0, rxb, nil, fmt.Errorf("error readDotLines parsed unknown command '%s' @ '%s'", what, connitem.c.provider.Name) + return 0, rxb, nil, fmt.Errorf("error readDotLines parsed unknown command '%s' @ '%s'", command, connitem.c.provider.Name) } // these commands are not stored in item, but will be returned as content // pass @@ -922,7 +922,7 @@ readlines: // reaching here means we have read the article or body or head // or any other command that is not stored in the item - if !IsOtherCommand(what) { + if !IsOtherCommand(command) { // clears content on head, body or article content = nil // clear content if not an other command } // else pass From 6cb50f4043b635937c7d69944c3b6eb47ba00ab5 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 21:37:16 +0100 Subject: [PATCH 009/152] Update Server.go --- Server.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Server.go b/Server.go index 0a4b2c8..2733e58 100644 --- a/Server.go +++ b/Server.go @@ -29,8 +29,7 @@ var ( // this is a global flag. if user/passwd config does not allow posting. user will not be able to post, even if this is true. globalAllowPosting = true - cidmux = &sync.Mutex{} // cidmux is used to synchronize access to CID1 - CID = uint64(0) // Global connection ID counter, can be used for session tracking - currently unused + CID = uint64(0) // Global connection ID counter, can be used for session tracking - currently unused proxyMutex = &sync.RWMutex{} // proxyMutex is used to synchronize access to passwdMap, ProxySessions and CountConns proxyCron = time.Now() // reload passwdMap every minute From 071e916129adf546d698fe9416f0507f1f8d731e Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 21:44:34 +0100 Subject: [PATCH 010/152] Update Server.go --- Server.go | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/Server.go b/Server.go index 2733e58..605e48b 100644 --- a/Server.go +++ b/Server.go @@ -533,6 +533,7 @@ loopProvider: } switch code { case 220, 221, 222: // Valid response codes for ARTICLE, HEAD, BODY + provider.ConnPool.ParkConn(0, connitem, "proxy") // Park the connection for reuse // send data to client txb, err := LinesWriter(ps.Writer, ps.Conn, code, item) // Send data to client if txb > 0 { @@ -541,24 +542,21 @@ loopProvider: ps.TXBytes += txb } if err != nil { - provider.ConnPool.CloseConn(connitem, nil) // Close the connection on error return fmt.Errorf("error writing data command %s to client: %v", command, err) } - response = "0" // full response already sent, set response to 0 to indicate success - provider.ConnPool.ParkConn(0, connitem, "proxy") // Park the connection for reuse - break loopProvider // Break out of the provider loop after handling the command + response = "0" // full response already sent, set response to 0 to indicate success + break loopProvider // Break out of the provider loop after handling the command case 423, 430, 451: // messageid not found + provider.ConnPool.ParkConn(0, connitem, "proxy") checkedProviderGroups[provider.Group] = true response = fmt.Sprintf("%d %s", code, msg) - provider.ConnPool.ParkConn(0, connitem, "proxy") SetArticleNotFoundAtProviderGroup(item.segment.Id, provider.Group) // Set article not found at provider group continue loopProvider default: + provider.ConnPool.CloseConn(connitem, nil) dlog(always, "ERROR in CMD for provider %s: cmd=%s code=%d msg='%s'", provider.Name, command, code, msg) response = fmt.Sprintf("502 Unknown Response: %d %s", code, msg) - // Handle other error codes - provider.ConnPool.CloseConn(connitem, nil) continue loopProvider } case "STAT": @@ -571,20 +569,20 @@ loopProvider: } switch code { case 223: // Article found - response = fmt.Sprintf("%d 0 %s", code, item.segment.Id) provider.ConnPool.ParkConn(0, connitem, "proxy") // Park the connection for reuse - break loopProvider // Break out of the provider loop after handling the command + response = fmt.Sprintf("%d 0 %s", code, item.segment.Id) + break loopProvider // Break out of the provider loop after handling the command case 423, 430, 451: // messageid not found + provider.ConnPool.ParkConn(0, connitem, "proxy") checkedProviderGroups[provider.Group] = true response = fmt.Sprintf("%d nf", code) - provider.ConnPool.ParkConn(0, connitem, "proxy") SetArticleNotFoundAtProviderGroup(item.segment.Id, provider.Group) // Set article not found at provider group continue loopProvider default: - response = fmt.Sprintf("%d %s", code, msg) provider.ConnPool.CloseConn(connitem, nil) + response = fmt.Sprintf("%d %s", code, msg) // Handle other error codes dlog(always, "ERROR CMD_STAT for provider %s: code=%d msg='%s'", provider.Name, code, msg) continue loopProvider // Continue to the next provider From 49158dae965820468ca23faf0a3ee74ec3bd048f Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 21:48:15 +0100 Subject: [PATCH 011/152] Update Server.go --- Server.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Server.go b/Server.go index 605e48b..f33d051 100644 --- a/Server.go +++ b/Server.go @@ -500,10 +500,17 @@ func (ps *ProxySession) handleRequest(command string, args []string) error { checkedProviderGroups := make(map[string]bool) loopProvider: for _, provider := range ProxyParent.providerList { - if provider.NoDownload || - checkedProviderGroups[provider.Group] || + switch command { + case "ARTICLE", "BODY", "HEAD", "STAT": + if provider.NoDownload { + // yes, doing stat on a provider that does not allow downloading articles does not make sense + response = "430 NODL" // 430 No Download, provider does not allow downloading articles + continue loopProvider // Skip this provider if it does not allow downloading articles + } + } + if checkedProviderGroups[provider.Group] || IsArticleNotFoundAtProviderGroup(item.segment.Id, provider.Group) { - response = "430 NO" + response = "430 NOPG" // 430 cached Not Found in ProviderGroup // Skip this provider if it has already been checked or is not available for download continue loopProvider } From 85abe14d95a9f500ca06fd6eeb0cb27b67ee7381 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 21:52:39 +0100 Subject: [PATCH 012/152] Update Server.go --- Server.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Server.go b/Server.go index f33d051..99a5a5c 100644 --- a/Server.go +++ b/Server.go @@ -447,6 +447,7 @@ func (ps *ProxySession) handleRequest(command string, args []string) error { var item *segmentChanItem // segmentChanItem to hold the message ID or number switch command { //switch command1 + // extract messageId from command case "STAT", "ARTICLE", "BODY", "HEAD": if len(args) == 0 { ps.CliTp.PrintfLine("501 Syntax error: %s requires a message ID or number", command) @@ -457,8 +458,9 @@ func (ps *ProxySession) handleRequest(command string, args []string) error { // protocol error, message ID is not valid ps.CliTp.PrintfLine("501 Syntax error: command %s requires a valid message ID", command) return fmt.Errorf("syntax error: command %s requires a valid message ID", command) + } else if num > 0 && ps.Group == "" { - // TODO + // TODO select newsgroup ps.tpWriter.PrintfLine("412 No newsgroup selected to read messageid: %d", num) return retry @@ -475,17 +477,21 @@ func (ps *ProxySession) handleRequest(command string, args []string) error { case "CAPABILITIES": printCapabilities(ps.tpWriter) return nil // No error, capabilities printed + case "DATE": ps.tpWriter.PrintfLine("111 %s", time.Now().Format(time.RFC1123Z)) return nil // No error, date printed + case "LIST", "XOVER", "XHDR", "GROUP", "NEXT", "LAST": ps.tpWriter.PrintfLine("500 cmd: %s (not implemented)", command) return fmt.Errorf("unknown command: %s (not implemented)", command) + case "QUIT": ps.tpWriter.PrintfLine("205 Closing connection - goodbye. uploaded=%d downloaded=%d connected='%v'", ps.RXBytes, ps.TXBytes, time.Since(ps.ConnectedAt)) log.Printf("Client %s issued QUIT.", ps.Conn.RemoteAddr()) time.Sleep(time.Millisecond) // Sleep to ensure the message is sent before closing the connection return fmt.Errorf("client %s issued QUIT", ps.Conn.RemoteAddr()) + default: ps.tpWriter.PrintfLine("502 Unknown command") return fmt.Errorf("unknown command: %s", command) @@ -495,7 +501,7 @@ func (ps *ProxySession) handleRequest(command string, args []string) error { ps.tpWriter.PrintfLine("501 Syntax error: command %s requires a valid message ID", command) return fmt.Errorf("syntax error: command %s requires a valid message ID", command) } - // Now we have a valid command and item (if applicable), proceed to handle the request + // Now we have a valid command and messageId (if applicable), proceed to handle the request var response string // response to be sent to the client after loopProvider has completed checkedProviderGroups := make(map[string]bool) loopProvider: From 4ce9b4213349df24bb223eb75eab257cdebe2c9b Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 22:36:15 +0100 Subject: [PATCH 013/152] Update Server.go --- Server.go | 140 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 114 insertions(+), 26 deletions(-) diff --git a/Server.go b/Server.go index 99a5a5c..4340185 100644 --- a/Server.go +++ b/Server.go @@ -31,14 +31,24 @@ var ( CID = uint64(0) // Global connection ID counter, can be used for session tracking - currently unused - proxyMutex = &sync.RWMutex{} // proxyMutex is used to synchronize access to passwdMap, ProxySessions and CountConns - proxyCron = time.Now() // reload passwdMap every minute - passwdMap = make(map[string]*UserData) // passwdMap holds user credentials (k is username, v is UserData) - ProxySessions = make(map[string]*ProxySession) // ProxySessions map to hold active user sessions (k is username, v is ProxySession) - CountConns = make(map[string]int) // CountConns keeps track of active connections per user (k is username, v is count) - ProxyParent *SESSION // ProxyParent is the parent session for the proxy, used to link sessions to the main loop + proxyMutex = &sync.RWMutex{} // proxyMutex is used to synchronize access to passwdMap, ProxySessions and CountConns + proxyCron = time.Now() // reload passwdMap every minute + passwdMap = make(map[string]*UserData) // passwdMap holds user credentials (k is username, v is UserData) + ProxySessions = make(map[string]*ProxySession) // ProxySessions map to hold active user sessions (k is username, v is ProxySession) + CountConns = make(map[string]int) // CountConns keeps track of active connections per user (k is username, v is count) + ProxyParent *SESSION // ProxyParent is the parent session for the proxy, used to link sessions to the main loop + CliRxTxCounter = make(map[string]*Counter_uint64) // RxTxCounter is a global counter for received and sent bytes (used for statistics) + CliRxTxMux = &sync.RWMutex{} // CliRxTxMux is used to synchronize access to CliRxTxCounter + statsChan = make(chan *statsItem, 1000) // statsChan is used to send segment items for statistics processing ) +type statsItem struct { + username string // Username of the client + rxbytes uint64 // Received bytes + txbytes uint64 // Sent bytes + clear error // Clear is used to indicate that the stats item should be cleared +} + // UserData holds user information. // When loading from .passwd, Password will be the bcrypt hash string. // When adding a new user via addUserToPasswdFile, Password should be plaintext to be hashed. @@ -52,26 +62,26 @@ type UserData struct { // ProxySession represents an active user session (currently a placeholder, expand as needed) type ProxySession struct { - id uint64 // Unique session ID, can be used for tracking - mux sync.RWMutex // Mutex for session data access - Authed bool // Indicates if the user is authenticated - Username string // Username of the authenticated user - Password string // password for the session, can be used for re-authentication - ExpireAt int64 // session expiration time (Unix timestamp) - Conn net.Conn // The user's network connection - Writer *bufio.Writer // bufio writer for the client connection to send articles, headers, bodies, list, xover, xhdr, ... (big data) - tpReader *textproto.Reader // textproto reader for easier command handling - tpWriter *textproto.Writer // textproto writer for easier command handling - CliTp *textproto.Conn // textproto connection for easier command handling - //tmpRXBytes uint64 // proxy has RECEIVED this amount of bytes FROM CLIENT via POST/IHAVE/TAKETHIS in last 60 seconds - RXBytes uint64 // proxy has RECEIVED this amount of bytes FROM CLIENT via POST/IHAVE/TAKETHIS in total this session - tmpTXBytes uint64 // proxy has SENT this amount of bytes TO CLIENT via ARTICLE/HEAD/BODY in last 60 seconds - TXBytes uint64 // proxy has SENT this amount of bytes TO CLIENT via ARTICLE/HEAD/BODY in total this session - ConnectedAt time.Time // Timestamp when the session was created - LastCmd time.Time // Timestamp of the last command received - Group string // current group the user is in (used by GROUP command) - MsgNum int64 // current message number in the group (used by STAT, ARTICLE, etc. commands) - Cron time.Time // last run of periodic tasks, e.g., checking session expiration + id uint64 // Unique session ID, can be used for tracking + mux sync.RWMutex // Mutex for session data access + Authed bool // Indicates if the user is authenticated + Username string // Username of the authenticated user + Password string // password for the session, can be used for re-authentication + ExpireAt int64 // session expiration time (Unix timestamp) + Conn net.Conn // The user's network connection + Writer *bufio.Writer // bufio writer for the client connection to send articles, headers, bodies, list, xover, xhdr, ... (big data) + tpReader *textproto.Reader // textproto reader for easier command handling + tpWriter *textproto.Writer // textproto writer for easier command handling + CliTp *textproto.Conn // textproto connection for easier command handling + tmpRXBytes uint64 // proxy has RECEIVED this amount of bytes FROM CLIENT via POST/IHAVE/TAKETHIS in last 60 seconds + RXBytes uint64 // proxy has RECEIVED this amount of bytes FROM CLIENT via POST/IHAVE/TAKETHIS in total this session + tmpTXBytes uint64 // proxy has SENT this amount of bytes TO CLIENT via ARTICLE/HEAD/BODY in last 60 seconds + TXBytes uint64 // proxy has SENT this amount of bytes TO CLIENT via ARTICLE/HEAD/BODY in total this session + ConnectedAt time.Time // Timestamp when the session was created + LastCmd time.Time // Timestamp of the last command received + Group string // current group the user is in (used by GROUP command) + MsgNum int64 // current message number in the group (used by STAT, ARTICLE, etc. commands) + Cron time.Time // last run of periodic tasks, e.g., checking session expiration // Add other session-specific data here, e.g., current group, article pointer, etc. } @@ -122,6 +132,10 @@ func handleConnection(conn net.Conn) { } // ProxySessions is used to track this specific session, remove it here. delete(ProxySessions, s.Username) + statsChan <- &statsItem{ + username: s.Username, + clear: fmt.Errorf("1"), + } proxyMutex.Unlock() } if ps.Authed { @@ -443,6 +457,22 @@ func (ps *ProxySession) handleRequest(command string, args []string) error { } //dlog(always, "Handling command '%s' for user '%s' in session.", command, ps.Username) // Handle these commands for authenticated users + if time.Since(ps.Cron) > time.Second*15 { // every 15s + // calulate download speed of this user session + txspeedinKB := float64(ps.tmpTXBytes) / 1024 / float64(time.Since(ps.Cron)) // speed in KB + rxspeedinKB := float64(ps.tmpRXBytes) / 1024 / float64(time.Since(ps.Cron)) // speed in KB + log.Printf(" %s | session DL speed: %.0f KB/s | session UL speed: %.0f KB/s", ps.Username, rxspeedinKB, txspeedinKB) + if ps.tmpTXBytes > 0 || ps.tmpRXBytes > 0 { + statsChan <- &statsItem{ + username: ps.Username, + rxbytes: ps.tmpRXBytes, + txbytes: ps.tmpTXBytes, + } + } + ps.Cron = time.Now() // Reset cron time for the next speed calculation + ps.tmpTXBytes = 0 + ps.tmpRXBytes = 0 + } pass := false var item *segmentChanItem // segmentChanItem to hold the message ID or number switch command { //switch command1 @@ -1032,3 +1062,61 @@ func (ps *ProxySession) IsExpired() (isExpired bool) { } return } // end func IsExpired + +func GoCliRxTxCounter() { + // GoCliRxTxCounter starts a goroutine to periodically log received and sent bytes per user + go func() { + for { + si := <-statsChan // Wait for stats items from the channel + CliRxTxMux.Lock() // Lock the global mutex to ensure thread-safe access to CliRxTxCounter + if si.clear != nil { + delete(CliRxTxCounter, si.username) + CliRxTxMux.Unlock() + continue + } + if _, ok := CliRxTxCounter[si.username]; !ok { + CliRxTxCounter[si.username] = NewCounter(4) // Initialize counter for the user if it doesn't exist + } + if si.rxbytes > 0 { + CliRxTxCounter[si.username].Add("tmpRXBytes", si.rxbytes) + } + if si.txbytes > 0 { + CliRxTxCounter[si.username].Add("tmpTXBytes", si.txbytes) + } + CliRxTxMux.Unlock() // Unlock the global mutex + } + }() + go func() { + for { + time.Sleep(time.Minute) + CliRxTxMux.Lock() + for username, counter := range CliRxTxCounter { + if counter == nil { + continue + } + rxbytes := counter.GetReset("tmpRXBytes") // Get and reset temporary RX bytes + txbytes := counter.GetReset("tmpTXBytes") // Get and reset temporary TX bytes + var RXspeedInKB, TXspeedInKB float64 + if rxbytes > 0 { + counter.Add("RXBytes", rxbytes) + RXspeedInKB = float64(rxbytes) / 1024 / 60 // Calculate speed in KB/s + } + if txbytes > 0 { + counter.Add("TXBytes", txbytes) + TXspeedInKB = float64(txbytes) / 1024 / 60 // Calculate speed in KB/s + } + var rxMiB, txMiB float64 + trx := counter.GetValue("RXBytes") + ttx := counter.GetValue("TXBytes") + if trx > 1024*1024 { + rxMiB = float64(trx) / 1024 / 1024 + } + if ttx > 1024*1024 { + txMiB = float64(ttx) / 1024 / 1024 + } + log.Printf(" %s | total DL: %.0f KiB/s (%.2f MiB) [%d bytes] | total UP: %.0f KiB/s (%.2f MiB) [%d bytes]", username, TXspeedInKB, txMiB, ttx, RXspeedInKB, rxMiB, trx) + } + CliRxTxMux.Unlock() + } + }() +} From 962c3e6c2d5218710ce9c586ca1e641f8af98aef Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 22:36:19 +0100 Subject: [PATCH 014/152] Update main.go --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 898659b..156370c 100644 --- a/main.go +++ b/main.go @@ -101,6 +101,7 @@ func main() { proxy: true, } ProxyParent = s // set global proxy session so proxy has access to provider list and pools + go GoCliRxTxCounter() globalmux.Unlock() go StartProxyServers(cfg.opt) if err := thisProcessor.LaunchSession(s, "", wg); err != nil { From cb7ce6a7412d518126efe2fefdb344d26e18ad8d Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 22:38:11 +0100 Subject: [PATCH 015/152] Update Server.go --- Server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Server.go b/Server.go index 4340185..d8812d0 100644 --- a/Server.go +++ b/Server.go @@ -1099,11 +1099,11 @@ func GoCliRxTxCounter() { var RXspeedInKB, TXspeedInKB float64 if rxbytes > 0 { counter.Add("RXBytes", rxbytes) - RXspeedInKB = float64(rxbytes) / 1024 / 60 // Calculate speed in KB/s + RXspeedInKB = float64(rxbytes) / 1024 / 60 // Calculate users upload speed in KB/s (proxy received) } if txbytes > 0 { counter.Add("TXBytes", txbytes) - TXspeedInKB = float64(txbytes) / 1024 / 60 // Calculate speed in KB/s + TXspeedInKB = float64(txbytes) / 1024 / 60 // Calculate users download speed in KB/s (proxy transceived) } var rxMiB, txMiB float64 trx := counter.GetValue("RXBytes") From 92c9510c8d466eb7d9bb0f2389cc76430fc4f87e Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 22:39:36 +0100 Subject: [PATCH 016/152] Update Server.go --- Server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Server.go b/Server.go index d8812d0..d8a04ef 100644 --- a/Server.go +++ b/Server.go @@ -1108,10 +1108,10 @@ func GoCliRxTxCounter() { var rxMiB, txMiB float64 trx := counter.GetValue("RXBytes") ttx := counter.GetValue("TXBytes") - if trx > 1024*1024 { + if trx > 1024 { rxMiB = float64(trx) / 1024 / 1024 } - if ttx > 1024*1024 { + if ttx > 1024 { txMiB = float64(ttx) / 1024 / 1024 } log.Printf(" %s | total DL: %.0f KiB/s (%.2f MiB) [%d bytes] | total UP: %.0f KiB/s (%.2f MiB) [%d bytes]", username, TXspeedInKB, txMiB, ttx, RXspeedInKB, rxMiB, trx) From 1bf9787a651c2a55f07c9a309a5c15dd21c1bb65 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 22:42:40 +0100 Subject: [PATCH 017/152] Update Server.go --- Server.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Server.go b/Server.go index d8a04ef..ebceab6 100644 --- a/Server.go +++ b/Server.go @@ -1108,13 +1108,18 @@ func GoCliRxTxCounter() { var rxMiB, txMiB float64 trx := counter.GetValue("RXBytes") ttx := counter.GetValue("TXBytes") - if trx > 1024 { + if trx > 1024*1024 { rxMiB = float64(trx) / 1024 / 1024 } - if ttx > 1024 { + if ttx > 1024*1024 { txMiB = float64(ttx) / 1024 / 1024 } - log.Printf(" %s | total DL: %.0f KiB/s (%.2f MiB) [%d bytes] | total UP: %.0f KiB/s (%.2f MiB) [%d bytes]", username, TXspeedInKB, txMiB, ttx, RXspeedInKB, rxMiB, trx) + if RXspeedInKB > 0 || TXspeedInKB > 0 { + log.Printf(" %s | session DL speed: %.0f KiB/s (%.0f MiB) [%d bytes] | session UL speed: %.0f KiB/s (%.0f MiB) [%d bytes]", username, TXspeedInKB, txMiB, ttx, RXspeedInKB, rxMiB, trx) + } else { + log.Printf(" %s | idle, no data transfer in last minute", username) + // TODO close session if no data transfer in last minute? + } } CliRxTxMux.Unlock() } From 2993d43a502601c02f46d717c0490f4e74ecee87 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 23:11:12 +0100 Subject: [PATCH 018/152] Update Flags.go --- Flags.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Flags.go b/Flags.go index 3758c51..2675092 100644 --- a/Flags.go +++ b/Flags.go @@ -87,6 +87,8 @@ func ParseFlags() { flag.BoolVar(&testrapidyenc, "testrapidyenc", false, "[true|false] will test rapidyenc testfiles on boot and exit (default: false)") flag.IntVar(&cfg.opt.ProxyTCP, "proxytcp", 0, "if set: use this port (e.g.: 1119) for TCP proxying (default: 0 = no proxy)") flag.IntVar(&cfg.opt.ProxyTLS, "proxytls", 0, "if set: use this port (e.g.: 1563) for TLS proxying (default: 0 = no proxy)") + flag.StringVar(&cfg.opt.TLSCertPem, "tlscrt", "fullchain.pem", "path to TLS certificate file (default: fullchain.pem in current dir)") + flag.StringVar(&cfg.opt.TLSPrivKey, "tlskey", "privkey.pem", "path to TLS private key file (default: privkey.pem in current dir)") flag.StringVar(&AddUserDataToProxy, "proxyadduser", "", "if set: adds this user to proxy ( \ne.g.: -proxyadduser='username1234|password4567|maxconns|expires'\n ) \nPassword entered MUST be cleartext, hashing is done when writing to .passwd file!\n Expiration: you can use a 'number of days' with 'd' at the end (e.g. 7d, 30d, 365d) or 'h' for hours (1h, 12h, ...) or 'm' minutes (1m, 30m, ...), it will be converted to a valid unixtimestamp expiring in the future from now on! (default: empty = no user added)\nExample -proxyadduser \"HelloWorld|NotAsecurePassword|5|42d\" creates a user with 5 conns and 42 days to expiration") flag.StringVar(&cfg.opt.ProxyPasswdFile, "proxypasswdfile", ".proxypasswd", "if set: use this file for proxy (default: .proxypasswd in current dir)") // cosmetics: segmentBar needs fixing: only when everything else works! From f8444277b89489fdfce4b76d7a8b831e6fb06c93 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 23:13:08 +0100 Subject: [PATCH 019/152] Update Flags.go --- Flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Flags.go b/Flags.go index 2675092..65a5515 100644 --- a/Flags.go +++ b/Flags.go @@ -110,7 +110,7 @@ func ParseFlags() { } if AddUserDataToProxy != "" { // Format: 'user|password|maxconns|expires|(no)post' (expires is unix timestamp) - parts := strings.SplitN(AddUserDataToProxy, "|", 4) + parts := strings.SplitN(AddUserDataToProxy, "|", 5) if len(parts) != 5 { log.Fatalf("-proxyadduser expects format: user|password|maxconns|expires|(no)post (got: %q)", AddUserDataToProxy) } From 1d5b88258a1c9c16beb9676f7a2967c3aa244a03 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 10 Jun 2025 23:37:31 +0100 Subject: [PATCH 020/152] Update Flags.go --- Flags.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Flags.go b/Flags.go index 65a5515..95e1130 100644 --- a/Flags.go +++ b/Flags.go @@ -87,9 +87,9 @@ func ParseFlags() { flag.BoolVar(&testrapidyenc, "testrapidyenc", false, "[true|false] will test rapidyenc testfiles on boot and exit (default: false)") flag.IntVar(&cfg.opt.ProxyTCP, "proxytcp", 0, "if set: use this port (e.g.: 1119) for TCP proxying (default: 0 = no proxy)") flag.IntVar(&cfg.opt.ProxyTLS, "proxytls", 0, "if set: use this port (e.g.: 1563) for TLS proxying (default: 0 = no proxy)") - flag.StringVar(&cfg.opt.TLSCertPem, "tlscrt", "fullchain.pem", "path to TLS certificate file (default: fullchain.pem in current dir)") - flag.StringVar(&cfg.opt.TLSPrivKey, "tlskey", "privkey.pem", "path to TLS private key file (default: privkey.pem in current dir)") - flag.StringVar(&AddUserDataToProxy, "proxyadduser", "", "if set: adds this user to proxy ( \ne.g.: -proxyadduser='username1234|password4567|maxconns|expires'\n ) \nPassword entered MUST be cleartext, hashing is done when writing to .passwd file!\n Expiration: you can use a 'number of days' with 'd' at the end (e.g. 7d, 30d, 365d) or 'h' for hours (1h, 12h, ...) or 'm' minutes (1m, 30m, ...), it will be converted to a valid unixtimestamp expiring in the future from now on! (default: empty = no user added)\nExample -proxyadduser \"HelloWorld|NotAsecurePassword|5|42d\" creates a user with 5 conns and 42 days to expiration") + flag.StringVar(&cfg.opt.TLSCertPem, "tlscrt", "fullchain.pem", "path to TLS certificate file for proxy (default: fullchain.pem in current dir)") + flag.StringVar(&cfg.opt.TLSPrivKey, "tlskey", "privkey.pem", "path to TLS private key file for proxy (default: privkey.pem in current dir)") + flag.StringVar(&AddUserDataToProxy, "proxyadduser", "", "if set: adds this user to proxy ( \ne.g.: -proxyadduser='username1234|password4567|maxconns|expires|(no)post'\n ) \nPassword entered MUST be cleartext, hashing is done when writing to .passwd file!\n Expiration: you can use a 'number of days' with 'd' at the end (e.g. 7d, 30d, 365d) or 'h' for hours (1h, 12h, ...) or 'm' minutes (1m, 30m, ...), it will be converted to a valid unixtimestamp expiring in the future from now on! (default: empty = no user added)\nExample -proxyadduser \"HelloWorld|NotAsecurePassword|5|42d|nopost\" creates a user with 5 conns and 42 days to expiration") flag.StringVar(&cfg.opt.ProxyPasswdFile, "proxypasswdfile", ".proxypasswd", "if set: use this file for proxy (default: .proxypasswd in current dir)") // cosmetics: segmentBar needs fixing: only when everything else works! //flag.BoolVar(&cfg.opt.Bar, "bar", false, "show progress bars") // FIXME TODO From 5b97e709c1caec44755917191ebc93188980e36e Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 11 Jun 2025 00:03:59 +0100 Subject: [PATCH 021/152] moving funcs to files --- Proxy.go | 197 +++++++++++++++ ProxyHelper.go | 455 ++++++++++++++++++++++++++++++++++ Server.go | 649 +------------------------------------------------ 3 files changed, 665 insertions(+), 636 deletions(-) create mode 100644 Proxy.go create mode 100644 ProxyHelper.go diff --git a/Proxy.go b/Proxy.go new file mode 100644 index 0000000..3c4edc8 --- /dev/null +++ b/Proxy.go @@ -0,0 +1,197 @@ +package main + +import ( + "bufio" + "crypto/tls" + "fmt" + "log" + mrand "math/rand" + "net" + "net/textproto" + "sync" + "time" +) + +var ( + nntpWelcomeMessage = "ready" // 20x 'NNTP Welcome message' for NNTP clients. x will be set by the server: globalAllowPosting true|false + welcomeCode = 201 // don't change, will be set by server on boot to 200 or 201 depending on globalAllowPosting + // Allow posting by default, can be set to false to disable posting (e.g., for read-only mode) + // this is a global flag. if user/passwd config does not allow posting. user will not be able to post, even if this is true. + globalAllowPosting = true + + CID = uint64(0) // Global connection ID counter, can be used for session tracking - currently unused + + proxyMutex = &sync.RWMutex{} // proxyMutex is used to synchronize access to passwdMap, ProxySessions and CountConns + proxyCron = time.Now() // reload passwdMap every minute + passwdMap = make(map[string]*UserData) // passwdMap holds user credentials (k is username, v is UserData) + ProxySessions = make(map[string]*ProxySession) // ProxySessions map to hold active user sessions (k is username, v is ProxySession) + CountConns = make(map[string]int) // CountConns keeps track of active connections per user (k is username, v is count) + ProxyParent *SESSION // ProxyParent is the parent session for the proxy, used to link sessions to the main loop + CliRxTxCounter = make(map[string]*Counter_uint64) // RxTxCounter is a global counter for received and sent bytes (used for statistics) + CliRxTxMux = &sync.RWMutex{} // CliRxTxMux is used to synchronize access to CliRxTxCounter + statsChan = make(chan *statsItem, 1000) // statsChan is used to send segment items for statistics processing + articleNotFound = &ArticleNotFound{Map: make(map[string]map[string]*A430)} // Global variable to track articles not found by provider +) + +// ArticleNotFound is a map to track articles not found by provider (k is provider group, v is map of message IDs) +type ArticleNotFound struct { + mux sync.RWMutex // Mutex to protect access to the map + Map map[string]map[string]*A430 // Map of provider groups to message IDs not found +} + +type A430 struct { + expires time.Time // Expiration time for the A430 article not found +} +type statsItem struct { + username string // Username of the client + rxbytes uint64 // Received bytes + txbytes uint64 // Sent bytes + clear error // Clear is used to indicate that the stats item should be cleared +} + +// UserData holds user information. +// When loading from .passwd, Password will be the bcrypt hash string. +// When adding a new user via addUserToPasswdFile, Password should be plaintext to be hashed. +type UserData struct { + Username string + Password string // For loading: bcrypt hash. For adding new user: plaintext. + MaxConns int // Optional: max connections per user, if needed + ExpireAt int64 // Optional: expiration time for the user, if needed (Unix timestamp) + Posting bool // Optional: indicates if the user is allowed to post articles +} + +// ProxySession represents an active user session (currently a placeholder, expand as needed) +type ProxySession struct { + id uint64 // Unique session ID, can be used for tracking + mux sync.RWMutex // Mutex for session data access + cmdmux sync.RWMutex // Mutex for command handling + Authed bool // Indicates if the user is authenticated + Username string // Username of the authenticated user + Password string // password for the session, can be used for re-authentication + ExpireAt int64 // session expiration time (Unix timestamp) + Conn net.Conn // The user's network connection + Writer *bufio.Writer // bufio writer for the client connection to send articles, headers, bodies, list, xover, xhdr, ... (big data) + tpReader *textproto.Reader // textproto reader for easier command handling + tpWriter *textproto.Writer // textproto writer for easier command handling + CliTp *textproto.Conn // textproto connection for easier command handling + tmpRXBytes uint64 // proxy has RECEIVED this amount of bytes FROM CLIENT via POST/IHAVE/TAKETHIS in last 60 seconds + RXBytes uint64 // proxy has RECEIVED this amount of bytes FROM CLIENT via POST/IHAVE/TAKETHIS in total this session + tmpTXBytes uint64 // proxy has SENT this amount of bytes TO CLIENT via ARTICLE/HEAD/BODY in last 60 seconds + TXBytes uint64 // proxy has SENT this amount of bytes TO CLIENT via ARTICLE/HEAD/BODY in total this session + ConnectedAt time.Time // Timestamp when the session was created + LastCmd time.Time // Timestamp of the last command received + Group string // current group the user is in (used by GROUP command) + MsgNum int64 // current message number in the group (used by STAT, ARTICLE, etc. commands) + Cron time.Time // last run of periodic tasks, e.g., checking session expiration + // Add other session-specific data here, e.g., current group, article pointer, etc. +} + +// StartNNTPServer initializes and starts the NNTP server. +// addr is the listen address (e.g., ":1119"). +// passwdFilePath is the path to the .passwd file. +// certFile is the path to the TLS certificate file (optional). +// keyFile is the path to the TLS private key file (optional). +func StartNNTPServer(addr string, passwdFilePath string, certFile string, keyFile string) { + time.Sleep(time.Duration(mrand.Intn(128)) * time.Millisecond) // Random delay to simulate server startup + err := loadPasswdFile(passwdFilePath) + if err != nil { + // Decide if server should start if passwd file is missing/corrupt. + // For this example, it logs a warning and continues (no users will be able to auth). + dlog(always, "WARNING: Could not find passwd file '%s': %v. Server starting without users.", passwdFilePath, err) + dlog(always, "To manually add users, use the '-proxyadduser' command or craft a new .passwd file manually") + dlog(always, "Example -proxyadduser \"HelloWorld|NotAsecurePassword|5|42d\" creates a user with 5 conns and 42 days to expiration") + + } else if len(passwdMap) == 0 { + dlog(always, "Warning: Passwd file '%s' loaded, but no users found or all entries were invalid.", passwdFilePath) + } + + var listener net.Listener + + if certFile != "" && keyFile != "" { + dlog(always, "Attempting to start TLS NNTP server on %s", addr) + cer, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + log.Fatalf("Failed to load TLS key pair: %v. Falling back to non-TLS or specify valid cert/key.", err) + // As a fallback, could attempt non-TLS, but for now, we exit if TLS is configured but fails to load. + // To fallback, you might set a flag and then proceed to the non-TLS listener block. + return // Or handle fallback more gracefully + } + + config := &tls.Config{Certificates: []tls.Certificate{cer}} + listener, err = tls.Listen("tcp", addr, config) + if err != nil { + log.Fatalf("Failed to start TLS NNTP server on %s: %v", addr, err) + } + dlog(always, "TLS NNTP server listening on %s", addr) + } else { + dlog(always, "Starting non-TLS NNTP server on %s", addr) + listener, err = net.Listen("tcp", addr) + if err != nil { + log.Fatalf("Failed to start non-TLS NNTP server on %s: %v", addr, err) + } + dlog(always, "Non-TLS NNTP server listening on %s", addr) + } + + defer listener.Close() + + for { + conn, err := listener.Accept() + if err != nil { + // Check if the error is due to the listener being closed. + if opError, ok := err.(*net.OpError); ok && opError.Err.Error() == "use of closed network connection" { + dlog(always, "Listener closed, shutting down accept loop.") + break // Exit loop if listener is closed + } + dlog(always, "Error accepting connection: %v", err) + continue // Continue to try accepting other connections + } + globalmux.RLock() // Lock the global mutex to ensure thread-safe access to ProxyParent + if ProxyParent != nil { + go handleConnection(conn) // Handle each client in a new goroutine + } else { + log.Printf("No ProxyParent available / not booted ... closing connection from %s", conn.RemoteAddr()) + conn.Close() // If ProxyParent is nil, we cannot handle this connection, so close it + } + globalmux.RUnlock() // Unlock the global mutex after checking ProxyParent + } // end for listener.Accept() + dlog(always, "NNTP server on %s stopped.", addr) +} // end func StartNNTPServer + +// StartProxyServers launches the NNTP server on configured TCP and/or TLS ports. +// It uses the global cfg variable for configuration parameters. +func StartProxyServers(appOpt *CFG) { + if appOpt == nil { + dlog(always, "CRITICAL: Application configuration (appOpt) is nil. Cannot start proxy servers.") + return + } + if globalAllowPosting { + welcomeCode = 200 // If posting is allowed, use 200 + } + started := false + + if appOpt.ProxyTCP > 0 { + tcpAddr := fmt.Sprintf(":%d", appOpt.ProxyTCP) + dlog(always, "Attempting to start non-TLS NNTP proxy on port %d", appOpt.ProxyTCP) + go StartNNTPServer(tcpAddr, appOpt.ProxyPasswdFile, "", "") + started = true + } else { + dlog(always, "Non-TLS NNTP proxy (ProxyTCP) not configured or port is 0, skipping.") + } + + if appOpt.ProxyTLS > 0 { + if appOpt.TLSCertPem == "" || appOpt.TLSPrivKey == "" { + dlog(always, "TLS NNTP proxy (ProxyTLS) configured for port %d, but TLSCertPem or TLSPrivKey is missing. Skipping TLS proxy.", appOpt.ProxyTLS) + } else { + tlsAddr := fmt.Sprintf(":%d", appOpt.ProxyTLS) + dlog(always, "Attempting to start TLS NNTP proxy (NNTPS) on port %d", appOpt.ProxyTLS) + go StartNNTPServer(tlsAddr, appOpt.ProxyPasswdFile, appOpt.TLSCertPem, appOpt.TLSPrivKey) + started = true + } + } else { + dlog(always, "TLS NNTP proxy (ProxyTLS) not configured or port is 0, skipping.") + } + + if !started { + dlog(always, "No NNTP proxy servers were started (neither ProxyTCP nor ProxyTLS were configured with valid ports/settings).") + } +} // end func StartProxyServers diff --git a/ProxyHelper.go b/ProxyHelper.go new file mode 100644 index 0000000..fd52707 --- /dev/null +++ b/ProxyHelper.go @@ -0,0 +1,455 @@ +package main + +import ( + "bufio" + "crypto/rand" + "fmt" + "io" + "log" + mrand "math/rand" + "net" + "net/textproto" + "os" + "strconv" + "strings" + "time" + + "golang.org/x/crypto/bcrypt" +) + +func LinesWriter(cliwriter *bufio.Writer, conn net.Conn, code int, item *segmentChanItem) (txb uint64, err error) { + defer func() { + if cliwriter != nil { + err := cliwriter.Flush() // Ensure all buffered data is written to the client + if err != nil { + log.Printf("Error in DotWriter: %v", err) + } + } + }() + var lines *[]string // lines to be sent to the client + switch code { + case 220: + lines = &item.article + case 221: + lines = &item.head + case 222: // Valid response codes for ARTICLE, HEAD, BODY + lines = &item.body + // These codes indicate successful retrieval of article, header, or body + } + // code Num + n, err := io.WriteString(cliwriter, fmt.Sprintf("%d 0 %s", code, item.segment.Id)+CRLF) + txb += uint64(n) + if err != nil { + return txb, fmt.Errorf("error DotWriter WriteString writer @ '%s' err='%v'", conn.RemoteAddr(), err) + } + // Write lines to the client connection using our own buffered dot writer + for _, line := range *lines { + // dot-stuffing when sending to client + if strings.HasPrefix(line, ".") { + line = "." + line // prepend a dot to the line + } + n, err := io.WriteString(cliwriter, line+CRLF) + txb += uint64(n) + if err != nil { + return txb, fmt.Errorf("error DotWriter WriteString writer err='%v'", err) + } + + } + // final sequence + n, err = io.WriteString(cliwriter, DOT+CRLF) + txb += uint64(n) + if err != nil { + return txb, fmt.Errorf("error DotWriter WriteString writer err='%v'", err) + } + return txb, nil +} // end func LinesWriter + +func printCapabilities(tpWriter *textproto.Writer) { + // Respond with server capabilities (RFC 3977 Section 5.3) + tpWriter.PrintfLine("101 Capability list:") + dw := tpWriter.DotWriter() + fmt.Fprintln(dw, "VERSION 2") // Indicates RFC 3977 support + fmt.Fprintln(dw, "READER") // Indicates MODE READER support + fmt.Fprintln(dw, "AUTHINFO USER PASS") // Supports AUTHINFO USER and AUTHINFO PASS + // Add other capabilities like LIST, IHAVE, POST, etc., as implemented + dw.Close() +} + +// Passwords in the file are expected to be bcrypt hash strings. +// File format: username:bcrypt_hash:maxconns:ExpireAt_unix_timestamp +func loadPasswdFile(filename string) error { + // Before loading, merge any .new.* files into the main passwd file + base := filename + if idx := strings.LastIndex(filename, "/"); idx != -1 { + base = filename[idx+1:] + } + dir := "." + if idx := strings.LastIndex(filename, "/"); idx != -1 { + dir = filename[:idx] + } + pattern := fmt.Sprintf("%s.new.", base) + entries, err := os.ReadDir(dir) + if err == nil { + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + if strings.HasPrefix(name, pattern) { + newfile := dir + "/" + name + // Append contents to main passwd file + func() { + mainf, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640) + if err != nil { + log.Printf("Could not open %s for appending: %v", filename, err) + return + } + defer mainf.Close() + f, err := os.Open(newfile) + if err != nil { + log.Printf("Could not open %s: %v", newfile, err) + return + } + defer f.Close() + if _, err := io.Copy(mainf, f); err != nil { + log.Printf("Failed to import %s into %s: %v", newfile, filename, err) + } else { + log.Printf("Imported new user(s) from %s into %s", newfile, filename) + os.Remove(newfile) + } + }() + } + } + } + + file, err := os.Open(filename) + if err != nil { + return fmt.Errorf("failed to open passwd file %s: %w", filename, err) + } + defer file.Close() + + proxyMutex.Lock() + defer proxyMutex.Unlock() + + // Clear existing map before loading to support reloading + for k := range passwdMap { + delete(passwdMap, k) + } + + scanner := bufio.NewScanner(file) + lineNumber := 0 + usersLoaded := 0 + newpasswdMap := make(map[string]*UserData) // Temporary map to hold new users + for scanner.Scan() { + lineNumber++ + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { // Skip empty lines and comments + continue + } + parts := strings.SplitN(line, "|", 5) // username|bcrypt_hash|maxconns|ExpireAt|(no)post + if len(parts) != 5 { + log.Printf("Skipping malformed line in passwd file (%d) (expected 5 parts, got %d): %s", lineNumber, len(parts), line) + continue + } + // check the splitted line parts + username := strings.TrimSpace(parts[0]) + bcryptHashFromFile := parts[1] + maxconnsStr := parts[2] + ExpireAtStr := parts[3] + posting := strings.HasPrefix(strings.ToLower(parts[4]), "post") + + maxconns, err := strconv.ParseInt(maxconnsStr, 10, 64) + if err != nil { + log.Printf("Skipping line in passwd file (%d) for user '%s' due to invalid maxconns ('%s'): %v", lineNumber, username, maxconnsStr, err) + continue + } + ExpireAt, err := strconv.ParseInt(ExpireAtStr, 10, 64) + if err != nil { + log.Printf("Skipping line in passwd file (%d) for user '%s' due to invalid ExpireAt ('%s'): %v", lineNumber, username, ExpireAtStr, err) + continue + } + if ExpireAt < time.Now().Unix() { + log.Printf("Skipping line in passwd file (%d) for user '%s' due to expired account (ExpireAt: %d, Current: %d)", lineNumber, username, ExpireAt, time.Now().Unix()) + continue + } + + if len(username) < 10 { + log.Printf("Skipping line in passwd file (%d) due to short username: %s", lineNumber, line) + continue + } + if len(bcryptHashFromFile) < 60 { + // Bcrypt hashes are typically 60 characters long, so we check for that. + // If the hash is empty or too short, we skip this user. + log.Printf("Skipping user '%s' on line in passwd file (%d) due to empty/short password hash.", username, lineNumber) + continue + } + + // check if the username already exists in the map + if _, exists := newpasswdMap[username]; exists { + delete(newpasswdMap, username) // Remove existing entry to avoid duplicates + log.Printf("WARNING: Skipping duplicate user '%s' on line %d in passwd file.", username, lineNumber) + continue // Skip duplicate usernames + } + newpasswdMap[username] = &UserData{ + Username: username, + Password: bcryptHashFromFile, // Store the bcrypt hash directly + MaxConns: int(maxconns), + ExpireAt: ExpireAt, + Posting: posting, + } + usersLoaded++ + } + + if err := scanner.Err(); err != nil { + return fmt.Errorf("error reading passwd file %s: %w", filename, err) + } + + if usersLoaded == 0 { + log.Printf("Warning: No users loaded from passwd file '%s'.", filename) + } else { + passwdMap = newpasswdMap + log.Printf("Successfully loaded %d users from '%s'.", usersLoaded, filename) + } + return nil +} + +// verifyPassword checks if the provided plaintext password matches the stored bcrypt hash for the user. +func verifyPassword(user string, plainPassword string) bool { + proxyMutex.RLock() + userData, ok := passwdMap[user] + proxyMutex.RUnlock() + + if !ok { + return false // User not found + } + time.Sleep(time.Duration(mrand.Intn(128)) * time.Millisecond) // small delay + // userData.Password holds the full bcrypt hash string from the .passwd file + err := bcrypt.CompareHashAndPassword([]byte(userData.Password), []byte(plainPassword)) + return err == nil // If err is nil, the password matches +} + +// addUserToPasswdFile adds a new user to the passwd file with a bcrypt hashed password. +// The UserData.Password field should contain the plaintext password for the new user. +func addUserToProxyPasswdFile(userData *UserData, filename string) error { + if userData.Password == "" { + return fmt.Errorf("cannot add user '%s': plaintext password is empty", userData.Username) + } + + // Generate bcrypt hash from the plaintext password + // bcrypt.DefaultCost is 10. You can increase this for more security (e.g., 12-14), + // but it will also be slower to hash and verify. + hashedPasswordBytes, err := bcrypt.GenerateFromPassword([]byte(userData.Password), bcrypt.DefaultCost) + if err != nil { + return fmt.Errorf("failed to generate bcrypt hash for user '%s': %w", userData.Username, err) + } + bcrpytHashString := string(hashedPasswordBytes) + + // Format the line to be appended to the file + // username:bcrypt_hash_string:maxconns:ExpireAt + posting := "nopost" // Default ACL for new users, can be changed if needed + if userData.Posting { + posting = "post" // Set ACL to post if userData.Posting is true + } + newUserLine := fmt.Sprintf("%s|%s|%d|%d|%s\n", userData.Username, bcrpytHashString, userData.MaxConns, userData.ExpireAt, posting) + + // Open the file in append mode, create if it doesn't exist + file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640) + if err != nil { + return fmt.Errorf("failed to open passwd file '%s' for appending: %w", filename, err) + } + defer file.Close() + + if _, err := file.WriteString(newUserLine); err != nil { + return fmt.Errorf("failed to write new user '%s' to passwd file '%s': %w", userData.Username, filename, err) + } + + log.Printf("Successfully added user '%s' to '%s' with a bcrypt hashed password.", userData.Username, filename) + // Optionally, reload passwdMap or add the new user directly to the in-memory map + // For simplicity here, we assume a restart or separate reload mechanism if immediate in-memory update is needed. + // To update in-memory map immediately: + /* TODO + proxyMutex.Lock() + passwdMap[userData.Username] = &UserData{ + Username: userData.Username, + Password: bcrpytHashString, // Store the newly generated hash + MaxConns: userData.MaxConns, + ExpireAt: userData.ExpireAt, + } + proxyMutex.Unlock() + log.Printf("User '%s' also updated in the in-memory passwdMap.", userData.Username) + */ + return nil +} + +// generateRandomHexCredentials generates a random username and password, each 10 bytes, output as hex strings. +func generateRandomHexCredentials() (username string, password string, err error) { + const length = 8 // 8 bytes will give us 16 hex characters, which is a common length for usernames/passwords + // Use crypto/rand for cryptographically secure random bytes + // If you want longer usernames/passwords, adjust the length accordingly. + // For example, 10 bytes would give 20 hex characters. + buf := make([]byte, length) + if _, err := rand.Read(buf); err != nil { + return "", "", fmt.Errorf("failed to generate random username: %w", err) + } + username = fmt.Sprintf("%x", buf) + if _, err := rand.Read(buf); err != nil { + return "", "", fmt.Errorf("failed to generate random password: %w", err) + } + password = fmt.Sprintf("%x", buf) + return username, password, nil +} + +func (ps *ProxySession) Close() { + ps.mux.Lock() + defer ps.mux.Unlock() + if ps.CliTp != nil { + ps.CliTp.Close() // Close the textproto connection + ps.CliTp = nil // Clear textproto connection to avoid dangling pointer + } + if ps.Conn != nil { + ps.Conn.Close() // Close the network connection + ps.Conn = nil // Clear the connection to avoid dangling pointer + } + ps.Authed = false // Mark session as unauthenticated + ps.Username = "" // Clear username to avoid dangling pointer + ps.Password = "" // Clear username to avoid dangling pointer + ps.ExpireAt = 0 // Clear expiration time + log.Printf("Closed session for user '%s'", ps.Username) +} + +func (ps *ProxySession) IsExpired() (isExpired bool) { + ps.mux.RLock() + isExpired = ps.ExpireAt > 0 && time.Now().Unix() > ps.ExpireAt + ps.mux.RUnlock() + if isExpired { + ps.Close() // Close the session if it has expired + } + return +} // end func IsExpired + +func GoCliRxTxCounter() { + // GoCliRxTxCounter starts a goroutine to periodically log received and sent bytes per user + go func() { + for { + si := <-statsChan // Wait for stats items from the channel + CliRxTxMux.Lock() // Lock the global mutex to ensure thread-safe access to CliRxTxCounter + if si.clear != nil { + delete(CliRxTxCounter, si.username) + CliRxTxMux.Unlock() + continue + } + if _, ok := CliRxTxCounter[si.username]; !ok { + CliRxTxCounter[si.username] = NewCounter(4) // Initialize counter for the user if it doesn't exist + } + if si.rxbytes > 0 { + CliRxTxCounter[si.username].Add("tmpRXBytes", si.rxbytes) + } + if si.txbytes > 0 { + CliRxTxCounter[si.username].Add("tmpTXBytes", si.txbytes) + } + CliRxTxMux.Unlock() // Unlock the global mutex + } + }() + go func() { + for { + time.Sleep(time.Minute) + CliRxTxMux.Lock() + for username, counter := range CliRxTxCounter { + if counter == nil { + continue + } + rxbytes := counter.GetReset("tmpRXBytes") // Get and reset temporary RX bytes + txbytes := counter.GetReset("tmpTXBytes") // Get and reset temporary TX bytes + var RXspeedInKB, TXspeedInKB float64 + if rxbytes > 0 { + counter.Add("RXBytes", rxbytes) + RXspeedInKB = float64(rxbytes) / 1024 / 60 // Calculate users upload speed in KB/s (proxy received) + } + if txbytes > 0 { + counter.Add("TXBytes", txbytes) + TXspeedInKB = float64(txbytes) / 1024 / 60 // Calculate users download speed in KB/s (proxy transceived) + } + var rxMiB, txMiB float64 + trx := counter.GetValue("RXBytes") + ttx := counter.GetValue("TXBytes") + if trx > 1024*1024 { + rxMiB = float64(trx) / 1024 / 1024 + } + if ttx > 1024*1024 { + txMiB = float64(ttx) / 1024 / 1024 + } + if RXspeedInKB > 0 || TXspeedInKB > 0 { + log.Printf(" %s | session DL speed: %.0f KiB/s (%.0f MiB) [%d bytes] | session UL speed: %.0f KiB/s (%.0f MiB) [%d bytes]", username, TXspeedInKB, txMiB, ttx, RXspeedInKB, rxMiB, trx) + } else { + log.Printf(" %s | idle, no data transfer in last minute", username) + // TODO close session if no data transfer in last minute? + } + } + CliRxTxMux.Unlock() + } + }() +} + +func IsArticleNotFoundAtProviderGroup(messageId string, providerGroup string) bool { + // Check if the article is not found at the provider group + articleNotFound.mux.RLock() + defer articleNotFound.mux.RUnlock() + if providerGroupMap, exists := articleNotFound.Map[providerGroup]; exists { + if a430, found := providerGroupMap[messageId]; found { + if time.Now().Before(a430.expires) { + log.Printf("cache: a430 isflag messageId '%s' not found at provider group '%s'", messageId, providerGroup) + return true // messageId flaged as not found and entry not expired + } + log.Printf("cache: a430 expired messageId '%s' at provider group '%s'", messageId, providerGroup) + go ClearArticleNotFoundAtProviderGroup(messageId, providerGroup) // Clear expired article not found entry + } + } + return false // not cached +} + +func ClearArticleNotFoundAtProviderGroup(messageId string, providerGroup string) { + // Clear the article not found at the provider group + articleNotFound.mux.Lock() + defer articleNotFound.mux.Unlock() + if providerGroupMap, exists := articleNotFound.Map[providerGroup]; exists { + if _, found := providerGroupMap[messageId]; found { + delete(providerGroupMap, messageId) // Remove the article not found entry + log.Printf("Cleared article not found for message ID '%s' at provider group '%s'", messageId, providerGroup) + } else { + log.Printf("Article '%s' not found at provider group '%s' (nothing to clear)", messageId, providerGroup) + } + } else { + log.Printf("Provider group '%s' does not exist in article not found map", providerGroup) + } +} + +func SetArticleNotFoundAtProviderGroup(messageId string, providerGroup string) { + // Set the article not found at the provider group + articleNotFound.mux.Lock() + defer articleNotFound.mux.Unlock() + if _, exists := articleNotFound.Map[providerGroup]; !exists { + articleNotFound.Map[providerGroup] = make(map[string]*A430) + } + articleNotFound.Map[providerGroup][messageId] = &A430{ + expires: time.Now().Add(1 * time.Minute), // Set expiration to 1 minute from now + } + log.Printf("Set article not found for message ID '%s' at provider group '%s'", messageId, providerGroup) +} + +// isValidMessageID checks if the provided message ID is valid according to NNTP standards. +func isValidMessageID(ps *ProxySession, messageID string) (isvalid bool, num int64) { + // Placeholder for message ID validation logic + if strings.HasPrefix(messageID, "<") && strings.HasSuffix(messageID, ">") { + // we dont check for @ in the message ID, as it is not required by RFC 5536 + isvalid = true + return + } + number, err := strconv.ParseInt(messageID, 10, 64) + if err == nil && number > 0 { + num = number + // If it's a number, we consider it valid + return + } + log.Printf(" %s | Invalid message ID: %s", ps.Username, messageID) + return +} diff --git a/Server.go b/Server.go index ebceab6..6354351 100644 --- a/Server.go +++ b/Server.go @@ -1,15 +1,12 @@ package main import ( - "bufio" - "crypto/rand" // Added for random salt generation - "crypto/tls" // Added for TLS support - "strconv" + "bufio" // Added for random salt generation + // Added for TLS support "time" "github.com/Tensai75/nzbparser" - "golang.org/x/crypto/bcrypt" // Added for bcrypt password hashing - + // Added for bcrypt password hashing // "encoding/hex" // Uncomment if you want to log/debug hashes as hex strings "fmt" "io" @@ -17,93 +14,10 @@ import ( mrand "math/rand" // Added for random delays in password verification (to mitigate timing attacks) "net" "net/textproto" // Added for textproto - "os" "strings" - "sync" -) - -var ( - nntpWelcomeMessage = "ready" // 20x 'NNTP Welcome message' for NNTP clients. x will be set by the server: globalAllowPosting true|false - welcomeCode = 201 // don't change, will be set by server on boot to 200 or 201 depending on globalAllowPosting - // Allow posting by default, can be set to false to disable posting (e.g., for read-only mode) - // this is a global flag. if user/passwd config does not allow posting. user will not be able to post, even if this is true. - globalAllowPosting = true - - CID = uint64(0) // Global connection ID counter, can be used for session tracking - currently unused - - proxyMutex = &sync.RWMutex{} // proxyMutex is used to synchronize access to passwdMap, ProxySessions and CountConns - proxyCron = time.Now() // reload passwdMap every minute - passwdMap = make(map[string]*UserData) // passwdMap holds user credentials (k is username, v is UserData) - ProxySessions = make(map[string]*ProxySession) // ProxySessions map to hold active user sessions (k is username, v is ProxySession) - CountConns = make(map[string]int) // CountConns keeps track of active connections per user (k is username, v is count) - ProxyParent *SESSION // ProxyParent is the parent session for the proxy, used to link sessions to the main loop - CliRxTxCounter = make(map[string]*Counter_uint64) // RxTxCounter is a global counter for received and sent bytes (used for statistics) - CliRxTxMux = &sync.RWMutex{} // CliRxTxMux is used to synchronize access to CliRxTxCounter - statsChan = make(chan *statsItem, 1000) // statsChan is used to send segment items for statistics processing ) -type statsItem struct { - username string // Username of the client - rxbytes uint64 // Received bytes - txbytes uint64 // Sent bytes - clear error // Clear is used to indicate that the stats item should be cleared -} - -// UserData holds user information. -// When loading from .passwd, Password will be the bcrypt hash string. -// When adding a new user via addUserToPasswdFile, Password should be plaintext to be hashed. -type UserData struct { - Username string - Password string // For loading: bcrypt hash. For adding new user: plaintext. - MaxConns int // Optional: max connections per user, if needed - ExpireAt int64 // Optional: expiration time for the user, if needed (Unix timestamp) - Posting bool // Optional: indicates if the user is allowed to post articles -} - -// ProxySession represents an active user session (currently a placeholder, expand as needed) -type ProxySession struct { - id uint64 // Unique session ID, can be used for tracking - mux sync.RWMutex // Mutex for session data access - Authed bool // Indicates if the user is authenticated - Username string // Username of the authenticated user - Password string // password for the session, can be used for re-authentication - ExpireAt int64 // session expiration time (Unix timestamp) - Conn net.Conn // The user's network connection - Writer *bufio.Writer // bufio writer for the client connection to send articles, headers, bodies, list, xover, xhdr, ... (big data) - tpReader *textproto.Reader // textproto reader for easier command handling - tpWriter *textproto.Writer // textproto writer for easier command handling - CliTp *textproto.Conn // textproto connection for easier command handling - tmpRXBytes uint64 // proxy has RECEIVED this amount of bytes FROM CLIENT via POST/IHAVE/TAKETHIS in last 60 seconds - RXBytes uint64 // proxy has RECEIVED this amount of bytes FROM CLIENT via POST/IHAVE/TAKETHIS in total this session - tmpTXBytes uint64 // proxy has SENT this amount of bytes TO CLIENT via ARTICLE/HEAD/BODY in last 60 seconds - TXBytes uint64 // proxy has SENT this amount of bytes TO CLIENT via ARTICLE/HEAD/BODY in total this session - ConnectedAt time.Time // Timestamp when the session was created - LastCmd time.Time // Timestamp of the last command received - Group string // current group the user is in (used by GROUP command) - MsgNum int64 // current message number in the group (used by STAT, ARTICLE, etc. commands) - Cron time.Time // last run of periodic tasks, e.g., checking session expiration - // Add other session-specific data here, e.g., current group, article pointer, etc. -} - -// isValidMessageID checks if the provided message ID is valid according to NNTP standards. -func isValidMessageID(ps *ProxySession, messageID string) (isvalid bool, num int64) { - // Placeholder for message ID validation logic - if strings.HasPrefix(messageID, "<") && strings.HasSuffix(messageID, ">") { - // we dont check for @ in the message ID, as it is not required by RFC 5536 - isvalid = true - return - } - number, err := strconv.ParseInt(messageID, 10, 64) - if err == nil && number > 0 { - num = number - // If it's a number, we consider it valid - return - } - log.Printf(" %s | Invalid message ID: %s", ps.Username, messageID) - return -} - -// handleConnection manages a single NNTP client connection. +// handleConnection manages a single NNTP proxy client connection. func handleConnection(conn net.Conn) { //log.Printf("Handling connection from %s", conn.RemoteAddr()) @@ -204,10 +118,13 @@ forever: ps.LastCmd = time.Now() // Update last command timestamp for unauthenticated users if authenticated { + ps.cmdmux.Lock() if err := ps.handleRequest(command, args); err != nil { log.Printf("Error handling command '%s' for user '%s': %v", command, ps.Username, err) + ps.cmdmux.Unlock() break } + ps.cmdmux.Unlock() continue forever // Continue to handle further commands after handling the request } @@ -386,64 +303,6 @@ forever: } // end for forever } // end func handleConnection -var articleNotFound = &ArticleNotFound{Map: make(map[string]map[string]*A430)} // Global variable to track articles not found by provider - -// ArticleNotFound is a map to track articles not found by provider (k is provider group, v is map of message IDs) -type ArticleNotFound struct { - mux sync.RWMutex // Mutex to protect access to the map - Map map[string]map[string]*A430 // Map of provider groups to message IDs not found -} - -type A430 struct { - expires time.Time // Expiration time for the A430 article not found -} - -func IsArticleNotFoundAtProviderGroup(messageId string, providerGroup string) bool { - // Check if the article is not found at the provider group - articleNotFound.mux.RLock() - defer articleNotFound.mux.RUnlock() - if providerGroupMap, exists := articleNotFound.Map[providerGroup]; exists { - if a430, found := providerGroupMap[messageId]; found { - if time.Now().Before(a430.expires) { - log.Printf("cache: a430 isflag messageId '%s' not found at provider group '%s'", messageId, providerGroup) - return true // messageId flaged as not found and entry not expired - } - log.Printf("cache: a430 expired messageId '%s' at provider group '%s'", messageId, providerGroup) - go ClearArticleNotFoundAtProviderGroup(messageId, providerGroup) // Clear expired article not found entry - } - } - return false // not cached -} - -func ClearArticleNotFoundAtProviderGroup(messageId string, providerGroup string) { - // Clear the article not found at the provider group - articleNotFound.mux.Lock() - defer articleNotFound.mux.Unlock() - if providerGroupMap, exists := articleNotFound.Map[providerGroup]; exists { - if _, found := providerGroupMap[messageId]; found { - delete(providerGroupMap, messageId) // Remove the article not found entry - log.Printf("Cleared article not found for message ID '%s' at provider group '%s'", messageId, providerGroup) - } else { - log.Printf("Article '%s' not found at provider group '%s' (nothing to clear)", messageId, providerGroup) - } - } else { - log.Printf("Provider group '%s' does not exist in article not found map", providerGroup) - } -} - -func SetArticleNotFoundAtProviderGroup(messageId string, providerGroup string) { - // Set the article not found at the provider group - articleNotFound.mux.Lock() - defer articleNotFound.mux.Unlock() - if _, exists := articleNotFound.Map[providerGroup]; !exists { - articleNotFound.Map[providerGroup] = make(map[string]*A430) - } - articleNotFound.Map[providerGroup][messageId] = &A430{ - expires: time.Now().Add(1 * time.Minute), // Set expiration to 1 minute from now - } - log.Printf("Set article not found for message ID '%s' at provider group '%s'", messageId, providerGroup) -} - // handleRequest processes NNTP commands for an authenticated user session. func (ps *ProxySession) handleRequest(command string, args []string) error { // Placeholder for handling specific NNTP commands in the session @@ -459,19 +318,20 @@ func (ps *ProxySession) handleRequest(command string, args []string) error { // Handle these commands for authenticated users if time.Since(ps.Cron) > time.Second*15 { // every 15s // calulate download speed of this user session - txspeedinKB := float64(ps.tmpTXBytes) / 1024 / float64(time.Since(ps.Cron)) // speed in KB - rxspeedinKB := float64(ps.tmpRXBytes) / 1024 / float64(time.Since(ps.Cron)) // speed in KB - log.Printf(" %s | session DL speed: %.0f KB/s | session UL speed: %.0f KB/s", ps.Username, rxspeedinKB, txspeedinKB) + //txspeedinKB := float64(ps.tmpTXBytes) / 1024 / float64(time.Since(ps.Cron)) // speed in KB + //rxspeedinKB := float64(ps.tmpRXBytes) / 1024 / float64(time.Since(ps.Cron)) // speed in KB + //log.Printf(" %s | session DL speed: %.0f KB/s | session UL speed: %.0f KB/s", ps.Username, rxspeedinKB, txspeedinKB) if ps.tmpTXBytes > 0 || ps.tmpRXBytes > 0 { statsChan <- &statsItem{ username: ps.Username, rxbytes: ps.tmpRXBytes, txbytes: ps.tmpTXBytes, } + ps.tmpTXBytes = 0 + ps.tmpRXBytes = 0 } ps.Cron = time.Now() // Reset cron time for the next speed calculation - ps.tmpTXBytes = 0 - ps.tmpRXBytes = 0 + } pass := false var item *segmentChanItem // segmentChanItem to hold the message ID or number @@ -642,486 +502,3 @@ loopProvider: } return nil // Return nil to indicate the command was handled successfully. an error will disconnect the user } // end func handleRequest - -func LinesWriter(cliwriter *bufio.Writer, conn net.Conn, code int, item *segmentChanItem) (txb uint64, err error) { - defer func() { - if cliwriter != nil { - err := cliwriter.Flush() // Ensure all buffered data is written to the client - if err != nil { - log.Printf("Error in DotWriter: %v", err) - } - } - }() - var lines *[]string // lines to be sent to the client - switch code { - case 220: - lines = &item.article - case 221: - lines = &item.head - case 222: // Valid response codes for ARTICLE, HEAD, BODY - lines = &item.body - // These codes indicate successful retrieval of article, header, or body - } - // code Num - n, err := io.WriteString(cliwriter, fmt.Sprintf("%d 0 %s", code, item.segment.Id)+CRLF) - txb += uint64(n) - if err != nil { - return txb, fmt.Errorf("error DotWriter WriteString writer @ '%s' err='%v'", conn.RemoteAddr(), err) - } - // Write lines to the client connection using our own buffered dot writer - for _, line := range *lines { - // dot-stuffing when sending to client - if strings.HasPrefix(line, ".") { - line = "." + line // prepend a dot to the line - } - n, err := io.WriteString(cliwriter, line+CRLF) - txb += uint64(n) - if err != nil { - return txb, fmt.Errorf("error DotWriter WriteString writer err='%v'", err) - } - - } - // final sequence - n, err = io.WriteString(cliwriter, DOT+CRLF) - txb += uint64(n) - if err != nil { - return txb, fmt.Errorf("error DotWriter WriteString writer err='%v'", err) - } - return txb, nil -} // end func DataWriter - -// StartNNTPServer initializes and starts the NNTP server. -// addr is the listen address (e.g., ":1119"). -// passwdFilePath is the path to the .passwd file. -// certFile is the path to the TLS certificate file (optional). -// keyFile is the path to the TLS private key file (optional). -func StartNNTPServer(addr string, passwdFilePath string, certFile string, keyFile string) { - time.Sleep(time.Duration(mrand.Intn(128)) * time.Millisecond) // Random delay to simulate server startup - err := loadPasswdFile(passwdFilePath) - if err != nil { - // Decide if server should start if passwd file is missing/corrupt. - // For this example, it logs a warning and continues (no users will be able to auth). - dlog(always, "WARNING: Could not find passwd file '%s': %v. Server starting without users.", passwdFilePath, err) - dlog(always, "To manually add users, use the '-proxyadduser' command or craft a new .passwd file manually") - dlog(always, "Example -proxyadduser \"HelloWorld|NotAsecurePassword|5|42d\" creates a user with 5 conns and 42 days to expiration") - - } else if len(passwdMap) == 0 { - dlog(always, "Warning: Passwd file '%s' loaded, but no users found or all entries were invalid.", passwdFilePath) - } - - var listener net.Listener - - if certFile != "" && keyFile != "" { - dlog(always, "Attempting to start TLS NNTP server on %s", addr) - cer, err := tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - log.Fatalf("Failed to load TLS key pair: %v. Falling back to non-TLS or specify valid cert/key.", err) - // As a fallback, could attempt non-TLS, but for now, we exit if TLS is configured but fails to load. - // To fallback, you might set a flag and then proceed to the non-TLS listener block. - return // Or handle fallback more gracefully - } - - config := &tls.Config{Certificates: []tls.Certificate{cer}} - listener, err = tls.Listen("tcp", addr, config) - if err != nil { - log.Fatalf("Failed to start TLS NNTP server on %s: %v", addr, err) - } - dlog(always, "TLS NNTP server listening on %s", addr) - } else { - dlog(always, "Starting non-TLS NNTP server on %s", addr) - listener, err = net.Listen("tcp", addr) - if err != nil { - log.Fatalf("Failed to start non-TLS NNTP server on %s: %v", addr, err) - } - dlog(always, "Non-TLS NNTP server listening on %s", addr) - } - - defer listener.Close() - - for { - conn, err := listener.Accept() - if err != nil { - // Check if the error is due to the listener being closed. - if opError, ok := err.(*net.OpError); ok && opError.Err.Error() == "use of closed network connection" { - dlog(always, "Listener closed, shutting down accept loop.") - break // Exit loop if listener is closed - } - dlog(always, "Error accepting connection: %v", err) - continue // Continue to try accepting other connections - } - globalmux.RLock() // Lock the global mutex to ensure thread-safe access to ProxyParent - if ProxyParent != nil { - go handleConnection(conn) // Handle each client in a new goroutine - } else { - log.Printf("No ProxyParent available / not booted ... closing connection from %s", conn.RemoteAddr()) - conn.Close() // If ProxyParent is nil, we cannot handle this connection, so close it - } - globalmux.RUnlock() // Unlock the global mutex after checking ProxyParent - } // end for listener.Accept() - dlog(always, "NNTP server on %s stopped.", addr) -} // end func StartNNTPServer - -// StartProxyServers launches the NNTP server on configured TCP and/or TLS ports. -// It uses the global cfg variable for configuration parameters. -func StartProxyServers(appOpt *CFG) { - if appOpt == nil { - dlog(always, "CRITICAL: Application configuration (appOpt) is nil. Cannot start proxy servers.") - return - } - if globalAllowPosting { - welcomeCode = 200 // If posting is allowed, use 200 - } - started := false - - if appOpt.ProxyTCP > 0 { - tcpAddr := fmt.Sprintf(":%d", appOpt.ProxyTCP) - dlog(always, "Attempting to start non-TLS NNTP proxy on port %d", appOpt.ProxyTCP) - go StartNNTPServer(tcpAddr, appOpt.ProxyPasswdFile, "", "") - started = true - } else { - dlog(always, "Non-TLS NNTP proxy (ProxyTCP) not configured or port is 0, skipping.") - } - - if appOpt.ProxyTLS > 0 { - if appOpt.TLSCertPem == "" || appOpt.TLSPrivKey == "" { - dlog(always, "TLS NNTP proxy (ProxyTLS) configured for port %d, but TLSCertPem or TLSPrivKey is missing. Skipping TLS proxy.", appOpt.ProxyTLS) - } else { - tlsAddr := fmt.Sprintf(":%d", appOpt.ProxyTLS) - dlog(always, "Attempting to start TLS NNTP proxy (NNTPS) on port %d", appOpt.ProxyTLS) - go StartNNTPServer(tlsAddr, appOpt.ProxyPasswdFile, appOpt.TLSCertPem, appOpt.TLSPrivKey) - started = true - } - } else { - dlog(always, "TLS NNTP proxy (ProxyTLS) not configured or port is 0, skipping.") - } - - if !started { - dlog(always, "No NNTP proxy servers were started (neither ProxyTCP nor ProxyTLS were configured with valid ports/settings).") - } -} // end func StartProxyServers - -// Passwords in the file are expected to be bcrypt hash strings. -// File format: username:bcrypt_hash:maxconns:ExpireAt_unix_timestamp -func loadPasswdFile(filename string) error { - // Before loading, merge any .new.* files into the main passwd file - base := filename - if idx := strings.LastIndex(filename, "/"); idx != -1 { - base = filename[idx+1:] - } - dir := "." - if idx := strings.LastIndex(filename, "/"); idx != -1 { - dir = filename[:idx] - } - pattern := fmt.Sprintf("%s.new.", base) - entries, err := os.ReadDir(dir) - if err == nil { - for _, entry := range entries { - if entry.IsDir() { - continue - } - name := entry.Name() - if strings.HasPrefix(name, pattern) { - newfile := dir + "/" + name - // Append contents to main passwd file - func() { - mainf, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640) - if err != nil { - log.Printf("Could not open %s for appending: %v", filename, err) - return - } - defer mainf.Close() - f, err := os.Open(newfile) - if err != nil { - log.Printf("Could not open %s: %v", newfile, err) - return - } - defer f.Close() - if _, err := io.Copy(mainf, f); err != nil { - log.Printf("Failed to import %s into %s: %v", newfile, filename, err) - } else { - log.Printf("Imported new user(s) from %s into %s", newfile, filename) - os.Remove(newfile) - } - }() - } - } - } - - file, err := os.Open(filename) - if err != nil { - return fmt.Errorf("failed to open passwd file %s: %w", filename, err) - } - defer file.Close() - - proxyMutex.Lock() - defer proxyMutex.Unlock() - - // Clear existing map before loading to support reloading - for k := range passwdMap { - delete(passwdMap, k) - } - - scanner := bufio.NewScanner(file) - lineNumber := 0 - usersLoaded := 0 - newpasswdMap := make(map[string]*UserData) // Temporary map to hold new users - for scanner.Scan() { - lineNumber++ - line := strings.TrimSpace(scanner.Text()) - if line == "" || strings.HasPrefix(line, "#") { // Skip empty lines and comments - continue - } - parts := strings.SplitN(line, "|", 5) // username|bcrypt_hash|maxconns|ExpireAt|(no)post - if len(parts) != 5 { - log.Printf("Skipping malformed line in passwd file (%d) (expected 5 parts, got %d): %s", lineNumber, len(parts), line) - continue - } - // check the splitted line parts - username := strings.TrimSpace(parts[0]) - bcryptHashFromFile := parts[1] - maxconnsStr := parts[2] - ExpireAtStr := parts[3] - posting := strings.HasPrefix(strings.ToLower(parts[4]), "post") - - maxconns, err := strconv.ParseInt(maxconnsStr, 10, 64) - if err != nil { - log.Printf("Skipping line in passwd file (%d) for user '%s' due to invalid maxconns ('%s'): %v", lineNumber, username, maxconnsStr, err) - continue - } - ExpireAt, err := strconv.ParseInt(ExpireAtStr, 10, 64) - if err != nil { - log.Printf("Skipping line in passwd file (%d) for user '%s' due to invalid ExpireAt ('%s'): %v", lineNumber, username, ExpireAtStr, err) - continue - } - if ExpireAt < time.Now().Unix() { - log.Printf("Skipping line in passwd file (%d) for user '%s' due to expired account (ExpireAt: %d, Current: %d)", lineNumber, username, ExpireAt, time.Now().Unix()) - continue - } - - if len(username) < 10 { - log.Printf("Skipping line in passwd file (%d) due to short username: %s", lineNumber, line) - continue - } - if len(bcryptHashFromFile) < 60 { - // Bcrypt hashes are typically 60 characters long, so we check for that. - // If the hash is empty or too short, we skip this user. - log.Printf("Skipping user '%s' on line in passwd file (%d) due to empty/short password hash.", username, lineNumber) - continue - } - - // check if the username already exists in the map - if _, exists := newpasswdMap[username]; exists { - delete(newpasswdMap, username) // Remove existing entry to avoid duplicates - log.Printf("WARNING: Skipping duplicate user '%s' on line %d in passwd file.", username, lineNumber) - continue // Skip duplicate usernames - } - newpasswdMap[username] = &UserData{ - Username: username, - Password: bcryptHashFromFile, // Store the bcrypt hash directly - MaxConns: int(maxconns), - ExpireAt: ExpireAt, - Posting: posting, - } - usersLoaded++ - } - - if err := scanner.Err(); err != nil { - return fmt.Errorf("error reading passwd file %s: %w", filename, err) - } - - if usersLoaded == 0 { - log.Printf("Warning: No users loaded from passwd file '%s'.", filename) - } else { - passwdMap = newpasswdMap - log.Printf("Successfully loaded %d users from '%s'.", usersLoaded, filename) - } - return nil -} - -// verifyPassword checks if the provided plaintext password matches the stored bcrypt hash for the user. -func verifyPassword(user string, plainPassword string) bool { - proxyMutex.RLock() - userData, ok := passwdMap[user] - proxyMutex.RUnlock() - - if !ok { - return false // User not found - } - time.Sleep(time.Duration(mrand.Intn(128)) * time.Millisecond) // small delay - // userData.Password holds the full bcrypt hash string from the .passwd file - err := bcrypt.CompareHashAndPassword([]byte(userData.Password), []byte(plainPassword)) - return err == nil // If err is nil, the password matches -} - -// addUserToPasswdFile adds a new user to the passwd file with a bcrypt hashed password. -// The UserData.Password field should contain the plaintext password for the new user. -func addUserToProxyPasswdFile(userData *UserData, filename string) error { - if userData.Password == "" { - return fmt.Errorf("cannot add user '%s': plaintext password is empty", userData.Username) - } - - // Generate bcrypt hash from the plaintext password - // bcrypt.DefaultCost is 10. You can increase this for more security (e.g., 12-14), - // but it will also be slower to hash and verify. - hashedPasswordBytes, err := bcrypt.GenerateFromPassword([]byte(userData.Password), bcrypt.DefaultCost) - if err != nil { - return fmt.Errorf("failed to generate bcrypt hash for user '%s': %w", userData.Username, err) - } - bcrpytHashString := string(hashedPasswordBytes) - - // Format the line to be appended to the file - // username:bcrypt_hash_string:maxconns:ExpireAt - posting := "nopost" // Default ACL for new users, can be changed if needed - if userData.Posting { - posting = "post" // Set ACL to post if userData.Posting is true - } - newUserLine := fmt.Sprintf("%s|%s|%d|%d|%s\n", userData.Username, bcrpytHashString, userData.MaxConns, userData.ExpireAt, posting) - - // Open the file in append mode, create if it doesn't exist - file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640) - if err != nil { - return fmt.Errorf("failed to open passwd file '%s' for appending: %w", filename, err) - } - defer file.Close() - - if _, err := file.WriteString(newUserLine); err != nil { - return fmt.Errorf("failed to write new user '%s' to passwd file '%s': %w", userData.Username, filename, err) - } - - log.Printf("Successfully added user '%s' to '%s' with a bcrypt hashed password.", userData.Username, filename) - // Optionally, reload passwdMap or add the new user directly to the in-memory map - // For simplicity here, we assume a restart or separate reload mechanism if immediate in-memory update is needed. - // To update in-memory map immediately: - /* TODO - proxyMutex.Lock() - passwdMap[userData.Username] = &UserData{ - Username: userData.Username, - Password: bcrpytHashString, // Store the newly generated hash - MaxConns: userData.MaxConns, - ExpireAt: userData.ExpireAt, - } - proxyMutex.Unlock() - log.Printf("User '%s' also updated in the in-memory passwdMap.", userData.Username) - */ - return nil -} - -// generateRandomHexCredentials generates a random username and password, each 10 bytes, output as hex strings. -func generateRandomHexCredentials() (username string, password string, err error) { - const length = 8 // 8 bytes will give us 16 hex characters, which is a common length for usernames/passwords - // Use crypto/rand for cryptographically secure random bytes - // If you want longer usernames/passwords, adjust the length accordingly. - // For example, 10 bytes would give 20 hex characters. - buf := make([]byte, length) - if _, err := rand.Read(buf); err != nil { - return "", "", fmt.Errorf("failed to generate random username: %w", err) - } - username = fmt.Sprintf("%x", buf) - if _, err := rand.Read(buf); err != nil { - return "", "", fmt.Errorf("failed to generate random password: %w", err) - } - password = fmt.Sprintf("%x", buf) - return username, password, nil -} - -func printCapabilities(tpWriter *textproto.Writer) { - // Respond with server capabilities (RFC 3977 Section 5.3) - tpWriter.PrintfLine("101 Capability list:") - dw := tpWriter.DotWriter() - fmt.Fprintln(dw, "VERSION 2") // Indicates RFC 3977 support - fmt.Fprintln(dw, "READER") // Indicates MODE READER support - fmt.Fprintln(dw, "AUTHINFO USER PASS") // Supports AUTHINFO USER and AUTHINFO PASS - // Add other capabilities like LIST, IHAVE, POST, etc., as implemented - dw.Close() -} - -func (ps *ProxySession) Close() { - ps.mux.Lock() - defer ps.mux.Unlock() - if ps.CliTp != nil { - ps.CliTp.Close() // Close the textproto connection - ps.CliTp = nil // Clear textproto connection to avoid dangling pointer - } - if ps.Conn != nil { - ps.Conn.Close() // Close the network connection - ps.Conn = nil // Clear the connection to avoid dangling pointer - } - ps.Authed = false // Mark session as unauthenticated - ps.Username = "" // Clear username to avoid dangling pointer - ps.Password = "" // Clear username to avoid dangling pointer - ps.ExpireAt = 0 // Clear expiration time - log.Printf("Closed session for user '%s'", ps.Username) -} - -func (ps *ProxySession) IsExpired() (isExpired bool) { - ps.mux.RLock() - isExpired = ps.ExpireAt > 0 && time.Now().Unix() > ps.ExpireAt - ps.mux.RUnlock() - if isExpired { - ps.Close() // Close the session if it has expired - } - return -} // end func IsExpired - -func GoCliRxTxCounter() { - // GoCliRxTxCounter starts a goroutine to periodically log received and sent bytes per user - go func() { - for { - si := <-statsChan // Wait for stats items from the channel - CliRxTxMux.Lock() // Lock the global mutex to ensure thread-safe access to CliRxTxCounter - if si.clear != nil { - delete(CliRxTxCounter, si.username) - CliRxTxMux.Unlock() - continue - } - if _, ok := CliRxTxCounter[si.username]; !ok { - CliRxTxCounter[si.username] = NewCounter(4) // Initialize counter for the user if it doesn't exist - } - if si.rxbytes > 0 { - CliRxTxCounter[si.username].Add("tmpRXBytes", si.rxbytes) - } - if si.txbytes > 0 { - CliRxTxCounter[si.username].Add("tmpTXBytes", si.txbytes) - } - CliRxTxMux.Unlock() // Unlock the global mutex - } - }() - go func() { - for { - time.Sleep(time.Minute) - CliRxTxMux.Lock() - for username, counter := range CliRxTxCounter { - if counter == nil { - continue - } - rxbytes := counter.GetReset("tmpRXBytes") // Get and reset temporary RX bytes - txbytes := counter.GetReset("tmpTXBytes") // Get and reset temporary TX bytes - var RXspeedInKB, TXspeedInKB float64 - if rxbytes > 0 { - counter.Add("RXBytes", rxbytes) - RXspeedInKB = float64(rxbytes) / 1024 / 60 // Calculate users upload speed in KB/s (proxy received) - } - if txbytes > 0 { - counter.Add("TXBytes", txbytes) - TXspeedInKB = float64(txbytes) / 1024 / 60 // Calculate users download speed in KB/s (proxy transceived) - } - var rxMiB, txMiB float64 - trx := counter.GetValue("RXBytes") - ttx := counter.GetValue("TXBytes") - if trx > 1024*1024 { - rxMiB = float64(trx) / 1024 / 1024 - } - if ttx > 1024*1024 { - txMiB = float64(ttx) / 1024 / 1024 - } - if RXspeedInKB > 0 || TXspeedInKB > 0 { - log.Printf(" %s | session DL speed: %.0f KiB/s (%.0f MiB) [%d bytes] | session UL speed: %.0f KiB/s (%.0f MiB) [%d bytes]", username, TXspeedInKB, txMiB, ttx, RXspeedInKB, rxMiB, trx) - } else { - log.Printf(" %s | idle, no data transfer in last minute", username) - // TODO close session if no data transfer in last minute? - } - } - CliRxTxMux.Unlock() - } - }() -} From bc95d1feee0c286bedaeae162e738e8a5005150a Mon Sep 17 00:00:00 2001 From: go-while <137838162+go-while@users.noreply.github.com> Date: Wed, 11 Jun 2025 00:05:34 +0100 Subject: [PATCH 022/152] Update Utils.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- Utils.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Utils.go b/Utils.go index 10fc34c..84d44b1 100644 --- a/Utils.go +++ b/Utils.go @@ -379,7 +379,14 @@ func (s *SESSION) writeCsvFile() (err error) { if err := csvWriter.Error(); err != nil { return fmt.Errorf("unable to write to the csv file: %v", err) } - f.Close() + defer func() { + if closeErr := f.Close(); closeErr != nil { + log.Printf("error closing csv file: %v", closeErr) + if err == nil { + err = closeErr + } + } + }() dlog(cfg.opt.Csv, "writeCsv: done") return } // end func writeCsv From e8223a96cc0bb18e092bbb8e188084b43215bfd3 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 11 Jun 2025 00:07:31 +0100 Subject: [PATCH 023/152] Update Session.go --- Session.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Session.go b/Session.go index bb1889e..ef76e19 100644 --- a/Session.go +++ b/Session.go @@ -85,7 +85,7 @@ type SESSION struct { proxy bool // flag to signal proxy is used in LaunchSession } // end type SESSION struct -func (p *PROCESSOR) NewProcessor(proxy bool) error { +func (p *PROCESSOR) NewProcessor() error { p.mux = &loggedrwmutex.LoggedSyncRWMutex{Name: "PROCESSOR"} // use a logged sync mutex to log locking and unlocking p.mux.Lock() defer p.mux.Unlock() From 8344089ab42febcb6ab768b10290150c992051c5 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 11 Jun 2025 00:07:33 +0100 Subject: [PATCH 024/152] Update main.go --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 156370c..c9e042c 100644 --- a/main.go +++ b/main.go @@ -76,7 +76,7 @@ func main() { wg := new(sync.WaitGroup) thisProcessor := &PROCESSOR{} go GoMutexStatus() - if err := thisProcessor.NewProcessor(proxy); err != nil { + if err := thisProcessor.NewProcessor(); err != nil { dlog(always, "ERROR NewProcessor: proxy err='%v'", err) os.Exit(1) } else { From da975c66d3bb925fde00a3046b1b2392646a8806 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 11 Jun 2025 00:10:10 +0100 Subject: [PATCH 025/152] Update Utils.go --- Utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Utils.go b/Utils.go index 10fc34c..97753f0 100644 --- a/Utils.go +++ b/Utils.go @@ -244,7 +244,7 @@ func AppendFileBytes(nullbytes int, dstPath string) error { return err } defer dstFile.Close() - nul := make([]byte, nullbytes) + nul := make([]byte, 0, nullbytes) for i := 0; i < nullbytes; i++ { nul = append(nul, 0x00) } From 82ca4665e56f18724fcd80375f62a0693e751125 Mon Sep 17 00:00:00 2001 From: go-while <137838162+go-while@users.noreply.github.com> Date: Wed, 11 Jun 2025 00:12:46 +0100 Subject: [PATCH 026/152] Update Utils.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- Utils.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Utils.go b/Utils.go index 0dcc700..d136729 100644 --- a/Utils.go +++ b/Utils.go @@ -243,7 +243,10 @@ func AppendFileBytes(nullbytes int, dstPath string) error { if err != nil { return err } - defer dstFile.Close() + // Close the file and check for errors + if closeErr := dstFile.Close(); closeErr != nil { + return fmt.Errorf("error closing file '%s': %v", dstPath, closeErr) + } nul := make([]byte, 0, nullbytes) for i := 0; i < nullbytes; i++ { nul = append(nul, 0x00) From f87d039e9410f4ef4dbcdccd381df0bf24b070b3 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 11 Jun 2025 00:22:46 +0100 Subject: [PATCH 027/152] Update ConnPool.go --- ConnPool.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/ConnPool.go b/ConnPool.go index c874b55..9162ae5 100644 --- a/ConnPool.go +++ b/ConnPool.go @@ -71,17 +71,15 @@ type ConnPool struct { openConns int // counter nextconnId uint64 // unique connection id //mux sync.RWMutex - mux *loggedrwmutex.LoggedSyncRWMutex - Counter *Counter_uint64 // used to count connections, parked conns, etc. - pool chan *ConnItem // idle/parked conns are in here - poolmap map[uint64]*ConnItem // idle/parked conns are in here - //wait []chan *ConnItem - rserver string // "host:port" + mux *loggedrwmutex.LoggedSyncRWMutex + Counter *Counter_uint64 // used to count connections, parked conns, etc. + pool chan *ConnItem // idle/parked conns are in here + rserver string // "host:port" wants_auth bool // we have created an endless loop // provider.ConnPool.provider.ConnPool.provider.ConnPool.provider.ConnPool.provider.ConnPool.provider.ConnPool.CloseConn(provider, connitem) xD // if we want to call closeconn from outside the routine which keep a provider ptr too - // we cann call ConnPools[provider.id].CloseConn(provider, connitem) + // we cann call ConnPools[provider.id].CloseConn(connitem) provider *Provider // pointer to the provider which created this ConnPool and holds the config s *SESSION // session pointer to the session which created this ConnPool (access via c.s.%%SESSIONVARIABLES%% OR provider.ConnPool.s.%%SESSIONVARIABLES%%). } @@ -127,7 +125,6 @@ func NewConnPool(s *SESSION, provider *Provider, workerWGconnReady *sync.WaitGro provider.ConnPool = &ConnPool{ Counter: NewCounter(10), // used to count connections, parked conns, etc. pool: make(chan *ConnItem, provider.MaxConns), - poolmap: make(map[uint64]*ConnItem, provider.MaxConns), // use map rserver: rserver, wants_auth: wants_auth, provider: provider, mux: &loggedrwmutex.LoggedSyncRWMutex{Name: fmt.Sprintf("ConnPool-%s", provider.Name)}, From a7a7dd111d1476de4fa78eb934874549a5304639 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 11 Jun 2025 01:03:15 +0100 Subject: [PATCH 028/152] Update Proxy.go --- Proxy.go | 615 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 594 insertions(+), 21 deletions(-) diff --git a/Proxy.go b/Proxy.go index 3c4edc8..fbf7677 100644 --- a/Proxy.go +++ b/Proxy.go @@ -8,6 +8,8 @@ import ( mrand "math/rand" "net" "net/textproto" + "strconv" + "strings" "sync" "time" ) @@ -62,27 +64,28 @@ type UserData struct { // ProxySession represents an active user session (currently a placeholder, expand as needed) type ProxySession struct { - id uint64 // Unique session ID, can be used for tracking - mux sync.RWMutex // Mutex for session data access - cmdmux sync.RWMutex // Mutex for command handling - Authed bool // Indicates if the user is authenticated - Username string // Username of the authenticated user - Password string // password for the session, can be used for re-authentication - ExpireAt int64 // session expiration time (Unix timestamp) - Conn net.Conn // The user's network connection - Writer *bufio.Writer // bufio writer for the client connection to send articles, headers, bodies, list, xover, xhdr, ... (big data) - tpReader *textproto.Reader // textproto reader for easier command handling - tpWriter *textproto.Writer // textproto writer for easier command handling - CliTp *textproto.Conn // textproto connection for easier command handling - tmpRXBytes uint64 // proxy has RECEIVED this amount of bytes FROM CLIENT via POST/IHAVE/TAKETHIS in last 60 seconds - RXBytes uint64 // proxy has RECEIVED this amount of bytes FROM CLIENT via POST/IHAVE/TAKETHIS in total this session - tmpTXBytes uint64 // proxy has SENT this amount of bytes TO CLIENT via ARTICLE/HEAD/BODY in last 60 seconds - TXBytes uint64 // proxy has SENT this amount of bytes TO CLIENT via ARTICLE/HEAD/BODY in total this session - ConnectedAt time.Time // Timestamp when the session was created - LastCmd time.Time // Timestamp of the last command received - Group string // current group the user is in (used by GROUP command) - MsgNum int64 // current message number in the group (used by STAT, ARTICLE, etc. commands) - Cron time.Time // last run of periodic tasks, e.g., checking session expiration + id uint64 // Unique session ID, can be used for tracking + mux sync.RWMutex // Mutex for session data access + cmdmux sync.RWMutex // Mutex for command handling + Authed bool // Indicates if the user is authenticated + Username string // Username of the authenticated user + Password string // password for the session, can be used for re-authentication + ExpireAt int64 // session expiration time (Unix timestamp) + Conn net.Conn // The user's network connection + Writer *bufio.Writer // bufio writer for the client connection to send articles, headers, bodies, list, xover, xhdr, ... (big data) + tpReader *textproto.Reader // textproto reader for easier command handling + tpWriter *textproto.Writer // textproto writer for easier command handling + CliTp *textproto.Conn // textproto connection for easier command handling + tmpRXBytes uint64 // proxy has RECEIVED this amount of bytes FROM CLIENT via POST/IHAVE/TAKETHIS in last 60 seconds + RXBytes uint64 // proxy has RECEIVED this amount of bytes FROM CLIENT via POST/IHAVE/TAKETHIS in total this session + tmpTXBytes uint64 // proxy has SENT this amount of bytes TO CLIENT via ARTICLE/HEAD/BODY in last 60 seconds + TXBytes uint64 // proxy has SENT this amount of bytes TO CLIENT via ARTICLE/HEAD/BODY in total this session + ConnectedAt time.Time // Timestamp when the session was created + LastCmd time.Time // Timestamp of the last command received + Group string // current group the user is in (used by GROUP command) + MsgNum int64 // current message number in the group (used by STAT, ARTICLE, etc. commands) + Cron time.Time // last run of periodic tasks, e.g., checking session expiration + selectedProvider *Provider // The provider selected for this session, used for routing commands // Add other session-specific data here, e.g., current group, article pointer, etc. } @@ -195,3 +198,573 @@ func StartProxyServers(appOpt *CFG) { dlog(always, "No NNTP proxy servers were started (neither ProxyTCP nor ProxyTLS were configured with valid ports/settings).") } } // end func StartProxyServers + +// GroupInfo holds information about a newsgroup +type GroupInfo struct { + Name string // Group name + Description string // Group description + Count int // Article count + Low int64 // Low water mark (oldest article number) + High int64 // High water mark (newest article number) + Posting bool // Posting allowed flag +} + +// ArticleOverview holds fields for the XOVER/OVER response +type ArticleOverview struct { + Number int64 // Article number + Subject string // Subject header + From string // From header + Date string // Date header + MessageID string // Message-ID header + References string // References header + Bytes int // Size in bytes + Lines int // Lines count + ExtraFields map[string]string // Additional fields +} + +// handleGroupCommand processes the GROUP command by selecting a newsgroup +func (ps *ProxySession) handleGroupCommand(args []string) error { + if len(args) < 1 { + ps.tpWriter.PrintfLine("501 Syntax error: GROUP ") + return nil + } + groupName := args[0] + + // Find a provider that has this group + var group *GroupInfo + + for _, provider := range ProxyParent.providerList { + if provider.NoDownload { + continue // Skip providers that don't allow downloads + } + + connitem, err := provider.ConnPool.GetConn() + if err != nil { + dlog(always, "ERROR GetConn for provider %s: %v", provider.Name, err) + continue // Try next provider + } + + id, err := connitem.srvtp.Cmd("GROUP %s", groupName) + if err != nil { + provider.ConnPool.CloseConn(connitem, nil) + continue // Try next provider + } + + connitem.srvtp.StartResponse(id) + code, msg, err := connitem.srvtp.ReadCodeLine(211) + connitem.srvtp.EndResponse(id) + + // 211 count low high group_name + if err == nil && code == 211 { + parts := strings.Fields(msg) + if len(parts) >= 4 { + count, _ := strconv.Atoi(parts[0]) + low, _ := strconv.ParseInt(parts[1], 10, 64) + high, _ := strconv.ParseInt(parts[2], 10, 64) + + group = &GroupInfo{ + Name: groupName, + Count: count, + Low: low, + High: high, + } + + ps.selectedProvider = provider + provider.ConnPool.ParkConn(0, connitem, "proxy") + break + } + } + + provider.ConnPool.ParkConn(0, connitem, "proxy") + } + + if group == nil { + ps.selectedProvider = nil // Reset selected provider if no group found + ps.tpWriter.PrintfLine("411 No such newsgroup") + return nil + } + + // Update the session with the selected group info + ps.Group = group.Name + ps.MsgNum = group.High // Set to the high water mark as default + + // Return a successful GROUP response: 211 count low high group_name + ps.tpWriter.PrintfLine("211 %d %d %d %s", + group.Count, group.Low, group.High, group.Name) + + dlog(always, "%s | Selected group: %s (articles: %d, range: %d-%d)", + ps.Username, group.Name, group.Count, group.Low, group.High) + + return nil +} + +// handleListCommand processes the LIST command +func (ps *ProxySession) handleListCommand(args []string) error { + var variant string + if len(args) >= 1 { + variant = strings.ToUpper(args[0]) + } + // Find a suitable provider + for _, provider := range ProxyParent.providerList { + if provider.NoDownload { + continue + } + if ps.selectedProvider != nil && provider != ps.selectedProvider { + continue // Skip if this provider is not the selected one for this session + } + + connitem, err := provider.ConnPool.GetConn() + if err != nil { + dlog(always, "ERROR GetConn for provider %s: %v", provider.Name, err) + continue + } + + // Different LIST variants + var command string + if variant == "" { + command = "LIST" + } else { + command = fmt.Sprintf("LIST %s", variant) + } + + id, err := connitem.srvtp.Cmd("%s", command) + if err != nil { + provider.ConnPool.CloseConn(connitem, nil) + continue + } + + connitem.srvtp.StartResponse(id) + code, _, err := connitem.srvtp.ReadCodeLine(215) + if err != nil || code != 215 { + connitem.srvtp.EndResponse(id) + provider.ConnPool.CloseConn(connitem, nil) + continue + } + + // Read the dot-delimited response directly into a list of strings + lines, err := connitem.srvtp.ReadDotLines() + connitem.srvtp.EndResponse(id) + + if err != nil { + provider.ConnPool.CloseConn(connitem, nil) + continue + } + provider.ConnPool.ParkConn(0, connitem, "proxy") + + ps.selectedProvider = provider // Set the selected provider for this session + + // Start our response to the client + ps.tpWriter.PrintfLine("215 List of newsgroups follows") + dw := ps.tpWriter.DotWriter() + + // Pass through the lines + for _, line := range lines { + fmt.Fprintln(dw, line) + } + + dw.Close() + return nil + } + + // If we couldn't find any provider or all failed + if ps.selectedProvider == nil { + ps.tpWriter.PrintfLine("503 No providers available for LIST command") + return nil + } + + return nil +} + +// handleXOverCommand processes the XOVER/OVER command +func (ps *ProxySession) handleXOverCommand(args []string, isXOVER bool) error { + // Must have a selected group first + if ps.Group == "" { + ps.tpWriter.PrintfLine("412 No newsgroup selected") + return nil + } + + var rangeArg string + if len(args) >= 1 { + rangeArg = args[0] + } else { + // If no range specified, use current article number + if ps.MsgNum <= 0 { + ps.tpWriter.PrintfLine("420 No current article selected") + return nil + } + rangeArg = fmt.Sprintf("%d", ps.MsgNum) + } + + // Process the range argument + var startNum, endNum int64 + + if strings.Contains(rangeArg, "-") { + parts := strings.Split(rangeArg, "-") + if len(parts) == 2 { + var err error + startNum, err = strconv.ParseInt(parts[0], 10, 64) + if err != nil { + ps.tpWriter.PrintfLine("501 Invalid article range") + return nil + } + + if parts[1] == "" { + // Format like "1000-" means "1000 to the end" + endNum = 0 // Will be handled as "to the end" by the server + } else { + endNum, err = strconv.ParseInt(parts[1], 10, 64) + if err != nil { + ps.tpWriter.PrintfLine("501 Invalid article range") + return nil + } + } + } + } else { + var err error + startNum, err = strconv.ParseInt(rangeArg, 10, 64) + if err != nil { + ps.tpWriter.PrintfLine("501 Invalid article number") + return nil + } + endNum = startNum // Just one article + } + + // Find provider with this group + for _, provider := range ProxyParent.providerList { + if provider.NoDownload { + continue + } + if ps.selectedProvider != nil && provider != ps.selectedProvider { + continue // Skip if this provider is not the selected one for this session + } + connitem, err := provider.ConnPool.GetConn() + if err != nil { + dlog(always, "ERROR GetConn for provider %s: %v", provider.Name, err) + continue + } + + // Select the group first (required before XOVER/OVER) + id, err := connitem.srvtp.Cmd("GROUP %s", ps.Group) + if err != nil { + provider.ConnPool.CloseConn(connitem, nil) + continue + } + + connitem.srvtp.StartResponse(id) + code, _, err := connitem.srvtp.ReadCodeLine(211) + connitem.srvtp.EndResponse(id) + + if err != nil || code != 211 { + provider.ConnPool.CloseConn(connitem, nil) + continue + } + + // Now run the XOVER/OVER command + var command string + if isXOVER { + command = "XOVER" + } else { + command = "OVER" + } + + if endNum > 0 { + command = fmt.Sprintf("%s %d-%d", command, startNum, endNum) + } else if endNum == 0 { + command = fmt.Sprintf("%s %d-", command, startNum) + } else { + command = fmt.Sprintf("%s %d", command, startNum) + } + + id, err = connitem.srvtp.Cmd("%s", command) + if err != nil { + provider.ConnPool.CloseConn(connitem, nil) + continue + } + + connitem.srvtp.StartResponse(id) + code, _, err = connitem.srvtp.ReadCodeLine(224) + if err != nil || code != 224 { + connitem.srvtp.EndResponse(id) + provider.ConnPool.CloseConn(connitem, nil) + continue + } + + // Read the dot-delimited response + lines, err := connitem.srvtp.ReadDotLines() + connitem.srvtp.EndResponse(id) + + if err != nil { + provider.ConnPool.CloseConn(connitem, nil) + continue + } + + // Start our response to the client + ps.tpWriter.PrintfLine("224 Overview information follows") + dw := ps.tpWriter.DotWriter() + + // Pass through the lines + for _, line := range lines { + fmt.Fprintln(dw, line) + } + + dw.Close() + provider.ConnPool.ParkConn(0, connitem, "proxy") + return nil + } + + // If we couldn't find any provider or all failed + ps.tpWriter.PrintfLine("503 No providers have the selected group") + return nil +} + +// handleXHdrCommand processes the XHDR/HDR command +func (ps *ProxySession) handleXHdrCommand(args []string, isXHDR bool) error { + if len(args) < 1 { + ps.tpWriter.PrintfLine("501 Syntax error: XHDR
[range|]") + return nil + } + + // Must have a selected group first, unless message-id is specified + if ps.Group == "" && !strings.HasPrefix(args[len(args)-1], "<") { + ps.tpWriter.PrintfLine("412 No newsgroup selected") + return nil + } + + headerField := args[0] + + // Get the message ID or range + var messageID string + var rangeSpec string + + if len(args) >= 2 { + if strings.HasPrefix(args[1], "<") { + // It's a message ID + messageID = args[1] + } else { + // It's a range spec + rangeSpec = args[1] + } + } else { + // If no range/msgid specified, use current article number + if ps.MsgNum <= 0 { + ps.tpWriter.PrintfLine("420 No current article selected") + return nil + } + rangeSpec = fmt.Sprintf("%d", ps.MsgNum) + } + + // Find provider with this group + for _, provider := range ProxyParent.providerList { + if provider.NoDownload { + continue + } + if ps.selectedProvider != nil && provider != ps.selectedProvider { + continue // Skip if this provider is not the selected one for this session + } + + connitem, err := provider.ConnPool.GetConn() + if err != nil { + dlog(always, "ERROR GetConn for provider %s: %v", provider.Name, err) + continue + } + + // Select the group first if using an article number range + if messageID == "" { + id, err := connitem.srvtp.Cmd("GROUP %s", ps.Group) + if err != nil { + provider.ConnPool.CloseConn(connitem, nil) + continue + } + + connitem.srvtp.StartResponse(id) + code, _, err := connitem.srvtp.ReadCodeLine(211) + connitem.srvtp.EndResponse(id) + + if err != nil || code != 211 { + provider.ConnPool.CloseConn(connitem, nil) + continue + } + } + + // Now run the XHDR/HDR command + var command string + if isXHDR { + command = "XHDR" + } else { + command = "HDR" + } + + if messageID != "" { + command = fmt.Sprintf("%s %s %s", command, headerField, messageID) + } else { + command = fmt.Sprintf("%s %s %s", command, headerField, rangeSpec) + } + + id, err := connitem.srvtp.Cmd("%s", command) + if err != nil { + provider.ConnPool.CloseConn(connitem, nil) + continue + } + + connitem.srvtp.StartResponse(id) + code, _, err := connitem.srvtp.ReadCodeLine(221) + if err != nil || code != 221 { + connitem.srvtp.EndResponse(id) + provider.ConnPool.CloseConn(connitem, nil) + continue + } + + // Read the dot-delimited response + lines, err := connitem.srvtp.ReadDotLines() + connitem.srvtp.EndResponse(id) + + if err != nil { + provider.ConnPool.CloseConn(connitem, nil) + continue + } + + // Start our response to the client + ps.tpWriter.PrintfLine("221 Header follows") + dw := ps.tpWriter.DotWriter() + + // Pass through the lines + for _, line := range lines { + fmt.Fprintln(dw, line) + } + + dw.Close() + provider.ConnPool.ParkConn(0, connitem, "proxy") + return nil + } + + // If we couldn't find any provider or all failed + ps.tpWriter.PrintfLine("503 No providers have the selected group or article") + return nil +} + +// handleNextOrLastCommand processes the NEXT/LAST command +func (ps *ProxySession) handleNextOrLastCommand(isNext bool) error { + // Must have a selected group first + if ps.Group == "" { + ps.tpWriter.PrintfLine("412 No newsgroup selected") + return nil + } + + // Must have a current article selected + if ps.MsgNum <= 0 { + ps.tpWriter.PrintfLine("420 No current article selected") + return nil + } + + // Find provider with this group + for _, provider := range ProxyParent.providerList { + if provider.NoDownload { + continue + } + if ps.selectedProvider != nil && provider != ps.selectedProvider { + continue // Skip if this provider is not the selected one for this session + } + + connitem, err := provider.ConnPool.GetConn() + if err != nil { + dlog(always, "ERROR GetConn for provider %s: %v", provider.Name, err) + continue + } + + // Select the group first + id, err := connitem.srvtp.Cmd("GROUP %s", ps.Group) + if err != nil { + provider.ConnPool.CloseConn(connitem, nil) + continue + } + + connitem.srvtp.StartResponse(id) + code, _, err := connitem.srvtp.ReadCodeLine(211) + connitem.srvtp.EndResponse(id) + + if err != nil { + if code > 0 { + provider.ConnPool.ParkConn(0, connitem, "proxy") + } else { + provider.ConnPool.CloseConn(connitem, nil) + } + continue + } + + // Set the current article + id, err = connitem.srvtp.Cmd("STAT %d", ps.MsgNum) + if err != nil { + provider.ConnPool.CloseConn(connitem, nil) + continue + } + + connitem.srvtp.StartResponse(id) + code, _, err = connitem.srvtp.ReadCodeLine(223) + connitem.srvtp.EndResponse(id) + + if err != nil { + if code > 0 { + provider.ConnPool.ParkConn(0, connitem, "proxy") + } else { + provider.ConnPool.CloseConn(connitem, nil) + } + continue + } + + // Now run the NEXT/LAST command + var command string + if isNext { + command = "NEXT" + } else { + command = "LAST" + } + + id, err = connitem.srvtp.Cmd("%s", command) + if err != nil { + provider.ConnPool.CloseConn(connitem, nil) + continue + } + + connitem.srvtp.StartResponse(id) + code, msg, err := connitem.srvtp.ReadCodeLine(223) + connitem.srvtp.EndResponse(id) + if code > 0 { + provider.ConnPool.ParkConn(0, connitem, "proxy") + } else { + provider.ConnPool.CloseConn(connitem, nil) + } + if err != nil || code != 223 { + if code == 421 { + // No next/previous article in the group + var which string + if isNext { + which = "next" + } else { + which = "previous" + } + ps.tpWriter.PrintfLine("421 No %s article to retrieve", which) + return nil + } + } else { + // 223 article_number message_id + parts := strings.Fields(msg) + if len(parts) >= 2 { + newNum, _ := strconv.ParseInt(parts[0], 10, 64) + messageID := parts[1] + + // Update the current article pointer + ps.MsgNum = newNum + + // Return successful response + ps.tpWriter.PrintfLine("223 %d %s", newNum, messageID) + } else { + ps.tpWriter.PrintfLine("503 Invalid response from server") + } + } + return nil + } + + // If we couldn't find any provider or all failed + ps.tpWriter.PrintfLine("503 No providers have the selected group") + return nil +} From e40ba2d3f96a55711851e4c724b9bba5ae09e784 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 11 Jun 2025 01:03:17 +0100 Subject: [PATCH 029/152] Update Server.go --- Server.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/Server.go b/Server.go index 6354351..54ad2f0 100644 --- a/Server.go +++ b/Server.go @@ -372,9 +372,23 @@ func (ps *ProxySession) handleRequest(command string, args []string) error { ps.tpWriter.PrintfLine("111 %s", time.Now().Format(time.RFC1123Z)) return nil // No error, date printed - case "LIST", "XOVER", "XHDR", "GROUP", "NEXT", "LAST": - ps.tpWriter.PrintfLine("500 cmd: %s (not implemented)", command) - return fmt.Errorf("unknown command: %s (not implemented)", command) + case "LIST": + return ps.handleListCommand(args) + + case "XOVER", "OVER": + return ps.handleXOverCommand(args, command == "XOVER") + + case "XHDR", "HDR": + return ps.handleXHdrCommand(args, command == "XHDR") + + case "GROUP": + return ps.handleGroupCommand(args) + + case "NEXT": + return ps.handleNextOrLastCommand(true) + + case "LAST": + return ps.handleNextOrLastCommand(false) case "QUIT": ps.tpWriter.PrintfLine("205 Closing connection - goodbye. uploaded=%d downloaded=%d connected='%v'", ps.RXBytes, ps.TXBytes, time.Since(ps.ConnectedAt)) From c559a56111ff91917c41376ee41b9b17b94e1d2c Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 11 Jun 2025 01:40:37 +0100 Subject: [PATCH 030/152] Update Config.go --- Config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/Config.go b/Config.go index 6e590bb..146fed9 100644 --- a/Config.go +++ b/Config.go @@ -121,6 +121,7 @@ type Provider struct { NoCheck bool // if true, no check will be done for this provider NoDownload bool // if true, no download will be done for this provider NoUpload bool // if true, no upload will be done for this provider + Newsreader bool // if true, provider allows newsreader commands (group, list, xhdr, xover,... etc.) Group string // group name is used internally to divide providers accounts into groups Name string // provider name, used for logging and identification Host string // provider host, used for connecting to the server From b2eb2f4b78cc598a90e441c52e5fe8afcbb2b4cf Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 11 Jun 2025 01:46:09 +0100 Subject: [PATCH 031/152] Update Proxy.go --- Proxy.go | 62 ++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/Proxy.go b/Proxy.go index fbf7677..aa17c84 100644 --- a/Proxy.go +++ b/Proxy.go @@ -234,7 +234,7 @@ func (ps *ProxySession) handleGroupCommand(args []string) error { var group *GroupInfo for _, provider := range ProxyParent.providerList { - if provider.NoDownload { + if provider.NoDownload || !provider.Newsreader { continue // Skip providers that don't allow downloads } @@ -269,7 +269,7 @@ func (ps *ProxySession) handleGroupCommand(args []string) error { High: high, } - ps.selectedProvider = provider + ps.selectedProvider = provider // Set the selected provider for this session when requesting GROUP provider.ConnPool.ParkConn(0, connitem, "proxy") break } @@ -305,8 +305,9 @@ func (ps *ProxySession) handleListCommand(args []string) error { variant = strings.ToUpper(args[0]) } // Find a suitable provider + selected := false // Track if we found a provider for this session for _, provider := range ProxyParent.providerList { - if provider.NoDownload { + if provider.NoDownload || !provider.Newsreader { continue } if ps.selectedProvider != nil && provider != ps.selectedProvider { @@ -350,8 +351,10 @@ func (ps *ProxySession) handleListCommand(args []string) error { continue } provider.ConnPool.ParkConn(0, connitem, "proxy") - - ps.selectedProvider = provider // Set the selected provider for this session + selected = true + if ps.selectedProvider == nil { + ps.selectedProvider = provider // Set the selected provider for this session when requesting LIST + } // Start our response to the client ps.tpWriter.PrintfLine("215 List of newsgroups follows") @@ -367,11 +370,13 @@ func (ps *ProxySession) handleListCommand(args []string) error { } // If we couldn't find any provider or all failed - if ps.selectedProvider == nil { + if !selected { + if ps.selectedProvider != nil { + ps.selectedProvider = nil + } ps.tpWriter.PrintfLine("503 No providers available for LIST command") return nil } - return nil } @@ -430,8 +435,9 @@ func (ps *ProxySession) handleXOverCommand(args []string, isXOVER bool) error { } // Find provider with this group + selected := false // Track if we found a provider for this session for _, provider := range ProxyParent.providerList { - if provider.NoDownload { + if provider.NoDownload || !provider.Newsreader { continue } if ps.selectedProvider != nil && provider != ps.selectedProvider { @@ -497,7 +503,7 @@ func (ps *ProxySession) handleXOverCommand(args []string, isXOVER bool) error { provider.ConnPool.CloseConn(connitem, nil) continue } - + selected = true // We found a provider that can handle this command // Start our response to the client ps.tpWriter.PrintfLine("224 Overview information follows") dw := ps.tpWriter.DotWriter() @@ -511,9 +517,13 @@ func (ps *ProxySession) handleXOverCommand(args []string, isXOVER bool) error { provider.ConnPool.ParkConn(0, connitem, "proxy") return nil } - - // If we couldn't find any provider or all failed - ps.tpWriter.PrintfLine("503 No providers have the selected group") + if !selected { + if ps.selectedProvider != nil { + ps.selectedProvider = nil + } + ps.tpWriter.PrintfLine("503 No providers have the selected group") + return nil + } return nil } @@ -554,8 +564,9 @@ func (ps *ProxySession) handleXHdrCommand(args []string, isXHDR bool) error { } // Find provider with this group + selected := false // Track if we found a provider for this session for _, provider := range ProxyParent.providerList { - if provider.NoDownload { + if provider.NoDownload || !provider.Newsreader { continue } if ps.selectedProvider != nil && provider != ps.selectedProvider { @@ -622,7 +633,7 @@ func (ps *ProxySession) handleXHdrCommand(args []string, isXHDR bool) error { provider.ConnPool.CloseConn(connitem, nil) continue } - + selected = true // Start our response to the client ps.tpWriter.PrintfLine("221 Header follows") dw := ps.tpWriter.DotWriter() @@ -636,9 +647,14 @@ func (ps *ProxySession) handleXHdrCommand(args []string, isXHDR bool) error { provider.ConnPool.ParkConn(0, connitem, "proxy") return nil } - + if !selected { + if ps.selectedProvider != nil { + ps.selectedProvider = nil + } + ps.tpWriter.PrintfLine("503 No providers have the selected group or article") + return nil + } // If we couldn't find any provider or all failed - ps.tpWriter.PrintfLine("503 No providers have the selected group or article") return nil } @@ -657,8 +673,9 @@ func (ps *ProxySession) handleNextOrLastCommand(isNext bool) error { } // Find provider with this group + selected := false // Track if we found a provider for this session for _, provider := range ProxyParent.providerList { - if provider.NoDownload { + if provider.NoDownload || !provider.Newsreader { continue } if ps.selectedProvider != nil && provider != ps.selectedProvider { @@ -729,6 +746,7 @@ func (ps *ProxySession) handleNextOrLastCommand(isNext bool) error { code, msg, err := connitem.srvtp.ReadCodeLine(223) connitem.srvtp.EndResponse(id) if code > 0 { + selected = true provider.ConnPool.ParkConn(0, connitem, "proxy") } else { provider.ConnPool.CloseConn(connitem, nil) @@ -763,8 +781,12 @@ func (ps *ProxySession) handleNextOrLastCommand(isNext bool) error { } return nil } - - // If we couldn't find any provider or all failed - ps.tpWriter.PrintfLine("503 No providers have the selected group") + if !selected { + if ps.selectedProvider != nil { + ps.selectedProvider = nil + } + ps.tpWriter.PrintfLine("503 No providers have the selected group") + return nil + } return nil } From 74b0455767306a6e25ec24ed1153ffb5f97bee65 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 11 Jun 2025 01:46:15 +0100 Subject: [PATCH 032/152] Update Server.go --- Server.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Server.go b/Server.go index 54ad2f0..efd814e 100644 --- a/Server.go +++ b/Server.go @@ -251,8 +251,7 @@ forever: tpWriter.PrintfLine("281 Welcome to NZBreX Proxy! Your conns: %d/%d. Exp: '%v'", CountConns[currentUser], userData.MaxConns, time.Unix(userData.ExpireAt, 0).Format(time.RFC1123Z)) - log.Printf("User '%s' authenticated from %s. Active connections for user: %d/%d", - currentUser, conn.RemoteAddr(), CountConns[currentUser], userData.MaxConns) + dlog(cfg.opt.Debug, "User '%s' authenticated. Active connections for user: %d/%d", currentUser, CountConns[currentUser], userData.MaxConns) proxyMutex.Unlock() // Unlock after updating CountConns and user data @@ -392,9 +391,8 @@ func (ps *ProxySession) handleRequest(command string, args []string) error { case "QUIT": ps.tpWriter.PrintfLine("205 Closing connection - goodbye. uploaded=%d downloaded=%d connected='%v'", ps.RXBytes, ps.TXBytes, time.Since(ps.ConnectedAt)) - log.Printf("Client %s issued QUIT.", ps.Conn.RemoteAddr()) - time.Sleep(time.Millisecond) // Sleep to ensure the message is sent before closing the connection - return fmt.Errorf("client %s issued QUIT", ps.Conn.RemoteAddr()) + log.Printf(" %s | quit", ps.Username) + return fmt.Errorf("client quit") default: ps.tpWriter.PrintfLine("502 Unknown command") From ef408f24fe80f266e0d1a8292e3f3d9c9ae91a64 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 11 Jun 2025 02:23:28 +0100 Subject: [PATCH 033/152] Update Server.go --- Server.go | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/Server.go b/Server.go index efd814e..29d8605 100644 --- a/Server.go +++ b/Server.go @@ -359,8 +359,18 @@ func (ps *ProxySession) handleRequest(command string, args []string) error { Id: args[0], }, } - pass = true // we have a valid message ID, so we can pass it to a provider + ps.MsgNum, ps.Group = 0, "" // Reset message number and group on valid + pass = true // we have a valid message ID, so we can pass it to a provider // TODO: add disk caching here? + } else if num > 0 { + // newsreader message number + ps.MsgNum = num // Store the message number in the session + pass = true // we have a valid message number, so we can pass it to a provider + item = &segmentChanItem{ + segment: &nzbparser.NzbSegment{ + Id: args[0], + }, + } } case "CAPABILITIES": @@ -410,12 +420,13 @@ loopProvider: for _, provider := range ProxyParent.providerList { switch command { case "ARTICLE", "BODY", "HEAD", "STAT": - if provider.NoDownload { + if provider.NoDownload || (ps.Group != "" && ps.MsgNum > 0 && !provider.Newsreader) { // yes, doing stat on a provider that does not allow downloading articles does not make sense response = "430 NODL" // 430 No Download, provider does not allow downloading articles continue loopProvider // Skip this provider if it does not allow downloading articles } } + if checkedProviderGroups[provider.Group] || IsArticleNotFoundAtProviderGroup(item.segment.Id, provider.Group) { response = "430 NOPG" // 430 cached Not Found in ProviderGroup @@ -427,7 +438,28 @@ loopProvider: dlog(always, "ERROR GetConn for provider %s: %v", provider.Name, err) continue loopProvider // Skip this provider if connection fails } + if ps.Group != "" && ps.MsgNum > 0 { + // execute GROUP command on remote if we have a group selected + id, err := connitem.srvtp.Cmd("GROUP %s", ps.Group) + if err != nil { + dlog(always, "ERROR CMD_GROUP for provider %s: %v", provider.Name, err) + provider.ConnPool.CloseConn(connitem, nil) + continue loopProvider + } + connitem.srvtp.StartResponse(id) + code, _, err := connitem.srvtp.ReadCodeLine(211) + connitem.srvtp.EndResponse(id) + if err != nil { + dlog(always, "ERROR CMD_GROUP command=%s for provider %s: %v", provider.Name, command, err) + if code > 0 { + provider.ConnPool.ParkConn(0, connitem, "proxy") + } else { + provider.ConnPool.CloseConn(connitem, nil) + } + continue + } + } dlog(cfg.opt.BUG, " %s | provider %s: %s got pc='%v'", ps.Username, provider.Name, command, connitem) // got connection to a provider From 69027ab90226a77d96953609639e6e34beca2283 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 11 Jun 2025 02:30:38 +0100 Subject: [PATCH 034/152] Update Utils.go --- Utils.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Utils.go b/Utils.go index d136729..37d93e9 100644 --- a/Utils.go +++ b/Utils.go @@ -243,17 +243,17 @@ func AppendFileBytes(nullbytes int, dstPath string) error { if err != nil { return err } - // Close the file and check for errors - if closeErr := dstFile.Close(); closeErr != nil { - return fmt.Errorf("error closing file '%s': %v", dstPath, closeErr) - } nul := make([]byte, 0, nullbytes) - for i := 0; i < nullbytes; i++ { + for range nullbytes { nul = append(nul, 0x00) } if _, writeErr := dstFile.Write(nul); writeErr != nil { return writeErr } + // Close the file and check for errors + if closeErr := dstFile.Close(); closeErr != nil { + return fmt.Errorf("error closing file '%s': %v", dstPath, closeErr) + } return nil } // end func AppendFileBytes From fa1e2ea60f2d499d0d2d32f11992a278595ed05a Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 11 Jun 2025 02:31:00 +0100 Subject: [PATCH 035/152] Update Utils.go --- Utils.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Utils.go b/Utils.go index 37d93e9..2c06b38 100644 --- a/Utils.go +++ b/Utils.go @@ -633,10 +633,6 @@ func AdjustMicroSleep(microsleep int64, pushed, todo uint64, lastRunTook time.Du return newSleep } -func fatal() bool { - return true -} - // GetImpossibleCloseCaseVariablesToString returns a string representation of the impossible close case variables. // This is used for debugging purposes to understand the state of the system when an impossible close case occurs. func GetImpossibleCloseCaseVariablesToString(segm, allOk, done, dead, isdl, indl, inup, isup, checked, dmca, nodl, noup, cached, inretry, inyenc, isyenc, dlQ, upQ, yeQ uint64) string { From 54cbcdc2d786b773bdea2d8c248ffab694380f35 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 11 Jun 2025 02:34:04 +0100 Subject: [PATCH 036/152] Update Session.go --- Session.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Session.go b/Session.go index ef76e19..b6af11b 100644 --- a/Session.go +++ b/Session.go @@ -243,7 +243,7 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s // create a new segmentChanItem for each segment item := &segmentChanItem{ segmux, s, &segment, &file, // sync.RWMutex, *SESSION, *nzbparser.Segment, *nzbparser.File - "hashfieldfilllater", // string fields + "", // string fields: 1. hashedId will be filled below &s.nzbHashedname, // *string fields make(chan int, 1), make(chan bool, 1), // chan fields // map fields for segment status From e47ff4603e16439ae28dce768e29bccc2d921d89 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 11 Jun 2025 02:46:38 +0100 Subject: [PATCH 037/152] Update Utils.go --- Utils.go | 122 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 77 insertions(+), 45 deletions(-) diff --git a/Utils.go b/Utils.go index 2c06b38..14a0fc1 100644 --- a/Utils.go +++ b/Utils.go @@ -226,11 +226,10 @@ func LoadHeadersFromFile(path string) ([]string, error) { // AppendFileBytes appends null bytes to the end of a file. // It opens the file in append mode, creates it if it does not exist, and writes the specified number of null bytes. -// If nullbytes is 0, it does nothing. -// If nullbytes is negative, it returns an error. +// If nullbytes is 0 or negative, it returns an error. // If the file does not exist, it creates a new file with the specified number of null bytes. // If the file exists, it appends the specified number of null bytes to the end of the file. -func AppendFileBytes(nullbytes int, dstPath string) error { +func AppendFileBytes(nullbytes int, dstPath string) (err error) { if nullbytes <= 0 { return fmt.Errorf("error AppendFileBytes nullbytes=%d must be greater than 0", nullbytes) } @@ -239,22 +238,38 @@ func AppendFileBytes(nullbytes int, dstPath string) error { } // Open destination file in append mode, create if not exists - dstFile, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) - if err != nil { - return err - } - nul := make([]byte, 0, nullbytes) - for range nullbytes { - nul = append(nul, 0x00) + dstFile, openErr := os.OpenFile(dstPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) + if openErr != nil { + return openErr // Return opening error directly } - if _, writeErr := dstFile.Write(nul); writeErr != nil { - return writeErr + // Defer Close and handle its error. + // If err is already set (e.g., from Write or Flush), this deferred Close error will not overwrite it. + // If err is nil, and Close fails, err will be set to the Close error. + defer func() { + if closeErr := dstFile.Close(); closeErr != nil { + if err == nil { // Only assign closeErr if no other error has occurred + err = fmt.Errorf("error closing file '%s': %v", dstPath, closeErr) + } + // Optional: Log closeErr if err was already set, e.g.: + // else { log.Printf("AppendFileBytes: additionally failed to close '%s' during error handling: %v", dstPath, closeErr) } + } + }() + + writer := bufio.NewWriter(dstFile) + for i := 0; i < nullbytes; i++ { + if err = writer.WriteByte(0x00); err != nil { + // err is assigned (it's the named return), will be returned. Defer will run. + return err + } } - // Close the file and check for errors - if closeErr := dstFile.Close(); closeErr != nil { - return fmt.Errorf("error closing file '%s': %v", dstPath, closeErr) + + // Ensure all buffered data is written to the file + if err = writer.Flush(); err != nil { + // err is assigned, will be returned. Defer will run. + return err } - return nil + + return nil // If everything is successful, err remains nil (or gets set by defer if Close fails) } // end func AppendFileBytes // AppendFile appends (merges) the file contents of srcPath to dstPath. @@ -265,48 +280,65 @@ func AppendFileBytes(nullbytes int, dstPath string) error { // If the source file does not exist, it returns an error. // If the destination file does not exist, it creates a new file. // If the source file is empty, it does nothing. -func AppendFile(srcPath string, dstPath string, delsrc bool) error { - if srcPath == "" || dstPath == "" { - return fmt.Errorf("error AppendFile srcPath='%s' or dstPath='%s' empty", srcPath, dstPath) +func AppendFile(srcPath string, dstPath string, delsrc bool) (err error) { + if srcPath == "" { + return fmt.Errorf("error AppendFile srcPath is empty") + } + if dstPath == "" { + return fmt.Errorf("error AppendFile dstPath is empty") } // Open source file for reading - srcFile, err := os.Open(srcPath) - if err != nil { - return err + srcFile, openSrcErr := os.Open(srcPath) + if openSrcErr != nil { + return fmt.Errorf("error AppendFile opening source file '%s': %w", srcPath, openSrcErr) } - defer srcFile.Close() + defer func() { + if closeErr := srcFile.Close(); closeErr != nil { + if err == nil { // Only assign closeErr if no other error has occurred + err = fmt.Errorf("error AppendFile closing source file '%s': %w", srcPath, closeErr) + } + // Optional: Log closeErr if err was already set + // else { log.Printf("AppendFile: additionally failed to close source '%s' during error handling: %v", srcPath, closeErr) } + } + }() // Open destination file in append mode, create if not exists - dstFile, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) - if err != nil { - return err + dstFile, openDstErr := os.OpenFile(dstPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) + if openDstErr != nil { + return fmt.Errorf("error AppendFile opening destination file '%s': %w", dstPath, openDstErr) } - defer dstFile.Close() - - // Create a buffer and copy in chunks - buf := make([]byte, DefaultYencWriteBuffer) - for { - n, readErr := srcFile.Read(buf) - if n > 0 { - if _, writeErr := dstFile.Write(buf[:n]); writeErr != nil { - return writeErr + defer func() { + if closeErr := dstFile.Close(); closeErr != nil { + if err == nil { // Only assign closeErr if no other error has occurred + err = fmt.Errorf("error AppendFile closing destination file '%s': %w", dstPath, closeErr) } + // Optional: Log closeErr if err was already set + // else { log.Printf("AppendFile: additionally failed to close destination '%s' during error handling: %v", dstPath, closeErr) } } - if readErr == io.EOF { - break - } - if readErr != nil { - return readErr - } + }() + + // Create a buffer and copy in chunks using io.CopyBuffer for efficiency + // DefaultYencWriteBuffer is assumed to be defined elsewhere, e.g., const DefaultYencWriteBuffer = 32 * 1024 + buf := make([]byte, DefaultYencWriteBuffer) // Ensure DefaultYencWriteBuffer is a reasonable size + if _, copyErr := io.CopyBuffer(dstFile, srcFile, buf); copyErr != nil { + return fmt.Errorf("error AppendFile copying from '%s' to '%s': %w", srcPath, dstPath, copyErr) } + + // Ensure all buffered data for dstFile is written to disk before attempting to remove srcFile + if syncErr := dstFile.Sync(); syncErr != nil { + return fmt.Errorf("error AppendFile syncing destination file '%s': %w", dstPath, syncErr) + } + if delsrc { - if err := os.Remove(srcPath); err != nil { - return fmt.Errorf("error Yenc AppendFile Remove err='%v'", err) + if removeErr := os.Remove(srcPath); removeErr != nil { + // Assign to err, so it's returned by the named return, and defer for dstFile can still run. + err = fmt.Errorf("error AppendFile removing source file '%s': %w", srcPath, removeErr) + return err } } - return nil -} // end func AppendFile (written by AI! GPT-4o) + return nil // If everything is successful, err remains nil (or gets set by defers if Close fails) +} // end func AppendFile func SHA256SumFile(path string) (string, error) { // Open the file for reading From dc497ee6fbb53e69f6e6c4d8edae0c8bdc38252a Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 11 Jun 2025 02:48:20 +0100 Subject: [PATCH 038/152] Update Utils.go --- Utils.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/Utils.go b/Utils.go index 14a0fc1..bc9a1ab 100644 --- a/Utils.go +++ b/Utils.go @@ -426,20 +426,6 @@ func (s *SESSION) writeCsvFile() (err error) { return } // end func writeCsv -/* - func setGlobalTimerNow(timer *time.Time) { - globalmux.Lock() - *timer = time.Now() - globalmux.Unlock() - } - - func getGlobalTimerSince(timer time.Time) time.Duration { - globalmux.RLock() - duration := time.Since(timer) - globalmux.RUnlock() - return duration - } -*/ func ConvertSpeed(bytes int64, durationSeconds int64) (kibPerSec int64, mbps float64) { if durationSeconds <= 0 { return 0, 0 From 7c9e4585d8eb870003f0e12fec0e8646f5e140ab Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 11 Jun 2025 02:53:21 +0100 Subject: [PATCH 039/152] Update Utils.go --- Utils.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Utils.go b/Utils.go index bc9a1ab..fb81a8f 100644 --- a/Utils.go +++ b/Utils.go @@ -371,6 +371,14 @@ func (s *SESSION) writeCsvFile() (err error) { if err != nil { return fmt.Errorf("unable to open csv file: %v", err) } + defer func() { + if closeErr := f.Close(); closeErr != nil { + log.Printf("error closing csv file: %v", closeErr) + if err == nil { + err = closeErr + } + } + }() log.Println("writing csv file...") fmt.Print("Writing csv file... ") csvWriter := csv.NewWriter(f) @@ -414,14 +422,7 @@ func (s *SESSION) writeCsvFile() (err error) { if err := csvWriter.Error(); err != nil { return fmt.Errorf("unable to write to the csv file: %v", err) } - defer func() { - if closeErr := f.Close(); closeErr != nil { - log.Printf("error closing csv file: %v", closeErr) - if err == nil { - err = closeErr - } - } - }() + dlog(cfg.opt.Csv, "writeCsv: done") return } // end func writeCsv From 736c090c0a01447fbb79e3d86f24731faf26072e Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 11 Jun 2025 02:54:59 +0100 Subject: [PATCH 040/152] Update Utils.go --- Utils.go | 94 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 54 insertions(+), 40 deletions(-) diff --git a/Utils.go b/Utils.go index fb81a8f..54f8eaa 100644 --- a/Utils.go +++ b/Utils.go @@ -364,67 +364,81 @@ func SHA256SumFile(path string) (string, error) { func (s *SESSION) writeCsvFile() (err error) { // not tested since rewrite if !cfg.opt.Csv { - return + return nil // Not an error, just not enabled + } + if s.nzbPath == "" { + return fmt.Errorf("writeCsvFile: nzbPath is empty, cannot determine CSV filename") } + csvFileName := strings.TrimSuffix(filepath.Base(s.nzbPath), filepath.Ext(filepath.Base(s.nzbPath))) + ".csv" - f, err := os.Create(csvFileName) - if err != nil { - return fmt.Errorf("unable to open csv file: %v", err) + f, createErr := os.Create(csvFileName) + if createErr != nil { + return fmt.Errorf("writeCsvFile: unable to create csv file '%s': %w", csvFileName, createErr) } defer func() { if closeErr := f.Close(); closeErr != nil { - log.Printf("error closing csv file: %v", closeErr) - if err == nil { - err = closeErr + log.Printf("writeCsvFile: error closing csv file '%s': %v", csvFileName, closeErr) // Log the close error regardless + if err == nil { // Only assign closeErr to the return if no other error has occurred + err = fmt.Errorf("writeCsvFile: error closing csv file '%s': %w", csvFileName, closeErr) } } }() - log.Println("writing csv file...") - fmt.Print("Writing csv file... ") + + log.Println("writing csv file...", csvFileName) // Added filename to log + fmt.Printf("Writing csv file '%s'... ", csvFileName) // Added filename to print + csvWriter := csv.NewWriter(f) - firstLine := true - // make sorted provider name slice + + // Prepare header + // Make sorted provider name slice providers := make([]string, 0, len(s.providerList)) - for n := range s.providerList { - providers = append(providers, s.providerList[n].Name) - } - sort.Strings(providers) - for fileName, file := range s.fileStat { - // write first line - if firstLine { - line := make([]string, len(providers)+2) - line[0] = "Filename" - line[1] = "Total segments" - for n, providerName := range providers { - line[n+2] = providerName - } - if err := csvWriter.Write(line); err != nil { - return fmt.Errorf("unable to write to the csv file: %v", err) - } - firstLine = false - } - // write line + for _, p := range s.providerList { // More idiomatic loop + providers = append(providers, p.Name) + } + sort.Strings(providers) // Ensure consistent column order + + header := make([]string, len(providers)+2) + header[0] = "Filename" + header[1] = "TotalSegments" + copy(header[2:], providers) + + if writeErr := csvWriter.Write(header); writeErr != nil { + return fmt.Errorf("writeCsvFile: unable to write header to csv file '%s': %w", csvFileName, writeErr) + } + + // Write data rows + // To ensure consistent row order if fileStat is a map, consider sorting keys + fileNames := make([]string, 0, len(s.fileStat)) + for fn := range s.fileStat { + fileNames = append(fileNames, fn) + } + sort.Strings(fileNames) // Sort filenames for consistent output order + + for _, fileName := range fileNames { + file := s.fileStat[fileName] line := make([]string, len(providers)+2) line[0] = fileName - line[1] = fmt.Sprintf("%v", file.totalSegments) - for n, providerName := range providers { + line[1] = fmt.Sprintf("%d", file.totalSegments) // Use %d for integers + for i, providerName := range providers { if value, ok := file.available[providerName]; ok { - line[n+2] = fmt.Sprintf("%v", value) + line[i+2] = fmt.Sprintf("%d", value) // Use %d for integers } else { - line[n+2] = "0" + line[i+2] = "0" } } - if err := csvWriter.Write(line); err != nil { - return fmt.Errorf("unable to write to the csv file: %v", err) + if writeErr := csvWriter.Write(line); writeErr != nil { + return fmt.Errorf("writeCsvFile: unable to write line for '%s' to csv file '%s': %w", fileName, csvFileName, writeErr) } } + csvWriter.Flush() - if err := csvWriter.Error(); err != nil { - return fmt.Errorf("unable to write to the csv file: %v", err) + if flushErr := csvWriter.Error(); flushErr != nil { + return fmt.Errorf("writeCsvFile: error flushing csv writer for '%s': %w", csvFileName, flushErr) } - dlog(cfg.opt.Csv, "writeCsv: done") - return + fmt.Println("Done.") // Indicate completion for the fmt.Print above + dlog(cfg.opt.Csv, "writeCsvFile: done for '%s'", csvFileName) + return nil // err is nil if execution reaches here successfully } // end func writeCsv func ConvertSpeed(bytes int64, durationSeconds int64) (kibPerSec int64, mbps float64) { From b38836f97fb71d2671dfa7c0519b906295f21a78 Mon Sep 17 00:00:00 2001 From: go-while <137838162+go-while@users.noreply.github.com> Date: Wed, 11 Jun 2025 02:56:16 +0100 Subject: [PATCH 041/152] Update Flags.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- Flags.go | 1 - 1 file changed, 1 deletion(-) diff --git a/Flags.go b/Flags.go index 95e1130..cf76af2 100644 --- a/Flags.go +++ b/Flags.go @@ -178,7 +178,6 @@ func ParseFlags() { } if err := addUserToProxyPasswdFile(userData, newPasswdFile); err != nil { log.Fatalf("Failed to add user to %s: %v", newPasswdFile, err) - os.Exit(1) } log.Printf("User '%s':\n %#v \n added to new passwd file: '%s'. Next reload in <60s", username, userData, newPasswdFile) os.Exit(0) From 5e81c70fafc9c99f81ee740297fa70a10ca9de39 Mon Sep 17 00:00:00 2001 From: go-while Date: Fri, 22 Aug 2025 23:13:39 +0100 Subject: [PATCH 042/152] patches --- Config.go | 6 +- rapidyenc/decoder.go | 117 +++++++++---------------- rapidyenc/decoder_test.go | 14 --- rapidyenc/encoder.go | 56 +----------- rapidyenc/librapidyenc_darwin.a | 1 + rapidyenc/librapidyenc_linux_amd64.a | 1 + rapidyenc/librapidyenc_linux_arm64.a | 1 + rapidyenc/librapidyenc_windows_amd64.a | 1 + 8 files changed, 52 insertions(+), 145 deletions(-) create mode 120000 rapidyenc/librapidyenc_darwin.a create mode 120000 rapidyenc/librapidyenc_linux_amd64.a create mode 120000 rapidyenc/librapidyenc_linux_arm64.a create mode 120000 rapidyenc/librapidyenc_windows_amd64.a diff --git a/Config.go b/Config.go index 146fed9..edbd10a 100644 --- a/Config.go +++ b/Config.go @@ -312,7 +312,7 @@ const ( cmdARTICLE = "ARTICLE" cmdHEAD = "HEAD" cmdBODY = "BODY" - cmdSTAT = "STAT" - cmdIHAVE = "IHAVE" - cmdPOST = "POST" + //cmdSTAT = "STAT" + //cmdIHAVE = "IHAVE" + //cmdPOST = "POST" ) diff --git a/rapidyenc/decoder.go b/rapidyenc/decoder.go index 5b86f5d..5da2b8b 100644 --- a/rapidyenc/decoder.go +++ b/rapidyenc/decoder.go @@ -2,14 +2,10 @@ package rapidyenc /* #cgo CFLAGS: -I${SRCDIR}/src -#cgo darwin LDFLAGS: ${SRCDIR}/librapidyenc.a -lstdc++ -#cgo windows,amd64 LDFLAGS: ${SRCDIR}/librapidyenc.a -lstdc++ -#cgo windows,386 LDFLAGS: ${SRCDIR}/librapidyenc.a -lstdc++ -#cgo windows,arm LDFLAGS: ${SRCDIR}/librapidyenc.a -lstdc++ -#cgo linux,amd64 LDFLAGS: ${SRCDIR}/librapidyenc.a -lstdc++ -#cgo linux,386 LDFLAGS: ${SRCDIR}/librapidyenc.a -lstdc++ -#cgo linux,arm LDFLAGS: ${SRCDIR}/librapidyenc.a -lstdc++ -#cgo linux,arm64 LDFLAGS: ${SRCDIR}/librapidyenc.a -lstdc++ +#cgo darwin LDFLAGS: ${SRCDIR}/librapidyenc_darwin.a -lstdc++ +#cgo windows,amd64 LDFLAGS: ${SRCDIR}/librapidyenc_windows_amd64.a -lstdc++ +#cgo linux,amd64 LDFLAGS: ${SRCDIR}/librapidyenc_linux_amd64.a -lstdc++ +#cgo linux,arm64 LDFLAGS: ${SRCDIR}/librapidyenc_linux_arm64.a -lstdc++ #include "rapidyenc.h" */ import "C" @@ -260,6 +256,31 @@ func (d *Decoder) Read(p []byte) (int, error) { // for the footer and EOF pattern (.\r\n) func (d *Decoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { transform: + if d.body && d.format == FormatYenc { + nd, ns, end, _ := DecodeIncremental(dst[nDst:], src[nSrc:], &d.State) + if nd > 0 { + d.hash.Write(dst[nDst : nDst+nd]) + d.actualSize += int64(nd) + nDst += nd + } + + switch end { + case EndControl: + nSrc += ns - 2 + d.body = false + case EndArticle: + nSrc += ns - 3 + d.body = false + default: + if d.State == StateCRLFEQ { + d.State = StateCRLF + nSrc += ns - 1 + } else { + nSrc += ns + } + return nDst, nSrc, transform.ErrShortSrc + } + } // Line by line processing for { if nSrc < 0 { @@ -270,25 +291,21 @@ transform: d.m.Hash = d.hash.Sum32() if !d.begin { switch d.format { - case FormatUnknown: - err = fmt.Errorf("[rapidyenc] FormatUnknown end of article without finding any *begin header: %w", ErrDataMissing) case FormatYenc: err = fmt.Errorf("[rapidyenc] FormatYenc end of article without finding \"=ybegin\" header: %w", ErrDataMissing) - case FormatUU: - err = fmt.Errorf("[rapidyenc] FormatUU end of article without finding \"begin\" header: %w", ErrDataCorruption) + default: + err = fmt.Errorf("[rapidyenc] FormatUnknown end of article without finding any *begin header: %w", ErrDataMissing) } } else if !d.end { switch d.format { - case FormatUnknown: - err = fmt.Errorf("[rapidyenc] FormatUnknown end of article without finding any *end header: %w", ErrDataMissing) case FormatYenc: err = fmt.Errorf("[rapidyenc] FormatYenc end of article without finding \"=yend\" trailer: %w", ErrDataMissing) - case FormatUU: - err = fmt.Errorf("[rapidyenc] FormatUU end of article without finding \"end\" trailer: %w", ErrDataCorruption) + default: + err = fmt.Errorf("[rapidyenc] FormatUnknown end of article without finding any *end header: %w", ErrDataMissing) } - } else if (d.format != FormatUU && d.format != FormatUnknown) && ((!d.part && d.m.Size != d.endSize) || (d.endSize != d.actualSize)) { + } else if (!d.part && d.m.Size != d.endSize) || (d.endSize != d.actualSize) { err = fmt.Errorf("[rapidyenc] expected size %d but got %d: %w", d.m.Size, d.actualSize, ErrDataCorruption) - } else if d.format == FormatYenc && d.crc && d.expectedCrc != d.m.Hash { + } else if d.crc && d.expectedCrc != d.m.Hash { // If we have a segment ID, use it for debugging otherwise use an empty string. err = fmt.Errorf("[rapidyenc] ERROR CRC32 expected hash '%#08x' but got '%#08x'! seg.Id='%s' err: %w", d.expectedCrc, d.m.Hash, *d.segId, ErrCrcMismatch) } else { @@ -311,61 +328,11 @@ transform: switch d.format { case FormatYenc: - // Header/trailer lines - if bytes.HasPrefix(line, []byte("=ybegin ")) || - bytes.HasPrefix(line, []byte("=ypart ")) || - bytes.HasPrefix(line, []byte("=yend ")) { - d.processYenc(line) - goto transform - } - // If we're in the body, decode this line - if d.body { - // Remove trailing \r\n for decoding - bodyLine := line - if len(bodyLine) >= 2 && bodyLine[len(bodyLine)-2] == '\r' && bodyLine[len(bodyLine)-1] == '\n' { - bodyLine = bodyLine[:len(bodyLine)-2] - } - - dlog(d.debugSpam, "DecodeIncremental input: %q\n", bodyLine) - nd, ns, end, derr := DecodeIncremental(dst[nDst:], bodyLine, &d.State) - if derr != nil && derr != io.EOF { - dlog(always, "ERROR in rapidyenc.DecodeIncremental: nd=%d ns=%d end=%v err=%v\n", nd, ns, end, derr) - } - - if nd > 0 { - d.hash.Write(dst[nDst : nDst+nd]) - d.actualSize += int64(nd) - nDst += nd - } - goto transform - } - + d.processYenc(line) + goto transform case FormatUU: - if bytes.HasPrefix(line, []byte("begin ")) || bytes.Equal(line, []byte("end\r\n")) { - d.processYenc(line) - goto transform - } - if d.body { - bodyLine := line - if len(bodyLine) >= 2 && bodyLine[len(bodyLine)-2] == '\r' && bodyLine[len(bodyLine)-1] == '\n' { - bodyLine = bodyLine[:len(bodyLine)-2] - } - // Decode the UUencoded line - decoded, err := UUdecode(bodyLine) - if err != nil { - d.err = fmt.Errorf("[rapidyenc] error decoding UUencoded line: %w", err) - return nDst, nSrc, d.err - } - if len(decoded) > 0 { - d.hash.Write(decoded) - d.actualSize += int64(len(decoded)) - if nDst+len(decoded) > len(dst) { - d.err = fmt.Errorf("[rapidyenc] destination buffer too small for UUencoded data: %w", errDestinationTooSmall) - return nDst, nSrc, d.err - } - nDst += copy(dst[nDst:], decoded) - } - } + // TODO: does not uudecode, for now just copies encoded data + nDst += copy(dst[nDst:], line) } } @@ -374,12 +341,10 @@ transform: // ! REVIEW ! not sure if we even get here since the formatUnknown is checked above if !d.begin || !d.end { switch d.format { - case FormatUnknown: - return nDst, nSrc, fmt.Errorf("[rapidyenc] FormatUnknown end of article without finding any *begin or *end header: %w", ErrDataMissing) case FormatYenc: return nDst, nSrc, fmt.Errorf("[rapidyenc] FormatYenc end of article without finding \"=ybegin\" or \"=yend\" header: %w", ErrDataMissing) - case FormatUU: - return nDst, nSrc, fmt.Errorf("[rapidyenc] FormatUU end of article without finding \"begin\" or \"end\" header: %w", ErrDataMissing) + default: + return nDst, nSrc, fmt.Errorf("[rapidyenc] FormatUnknown end of article without finding any *begin or *end header: %w", ErrDataMissing) } } return nDst, nSrc, io.EOF diff --git a/rapidyenc/decoder_test.go b/rapidyenc/decoder_test.go index 49a8ca4..8011e95 100644 --- a/rapidyenc/decoder_test.go +++ b/rapidyenc/decoder_test.go @@ -592,19 +592,5 @@ func GenerateTestUUEncodedFiles(t *testing.T) error { return nil } - content1 := []byte("Hello from test1!\nThis is a test file.\n") - content2 := []byte("Another file for test2.\nWith more lines.\n1234567890\r\n") - - uue1 := UUEncode(content1, "test1.txt", 644) - uue2 := UUEncode(content2, "test2.txt", 644) - - if err := os.WriteFile(filepath.Join("uuencode", "test1.uue"), uue1, 0644); err != nil { - t.Errorf("Failed to write uuencode/test1.uue: %v", err) - return err - } - if err := os.WriteFile(filepath.Join("uuencode", "test2.uue"), uue2, 0644); err != nil { - t.Errorf("Failed to write uuencode/test2.uue: %v", err) - return err - } return nil } diff --git a/rapidyenc/encoder.go b/rapidyenc/encoder.go index 4d30baf..4ddd480 100644 --- a/rapidyenc/encoder.go +++ b/rapidyenc/encoder.go @@ -2,21 +2,14 @@ package rapidyenc /* #cgo CFLAGS: -I${SRCDIR}/src -#cgo darwin LDFLAGS: ${SRCDIR}/librapidyenc.a -lstdc++ -#cgo windows,amd64 LDFLAGS: ${SRCDIR}/librapidyenc.a -lstdc++ -#cgo windows,386 LDFLAGS: ${SRCDIR}/librapidyenc.a -lstdc++ -#cgo windows,arm LDFLAGS: ${SRCDIR}/librapidyenc.a -lstdc++ -#cgo linux,amd64 LDFLAGS: ${SRCDIR}/librapidyenc.a -lstdc++ -#cgo linux,386 LDFLAGS: ${SRCDIR}/librapidyenc.a -lstdc++ -#cgo linux,arm LDFLAGS: ${SRCDIR}/librapidyenc.a -lstdc++ -#cgo linux,arm64 LDFLAGS: ${SRCDIR}/librapidyenc.a -lstdc++ +#cgo darwin LDFLAGS: ${SRCDIR}/librapidyenc_darwin.a -lstdc++ +#cgo windows,amd64 LDFLAGS: ${SRCDIR}/librapidyenc_windows_amd64.a -lstdc++ +#cgo linux,amd64 LDFLAGS: ${SRCDIR}/librapidyenc_linux_amd64.a -lstdc++ +#cgo linux,arm64 LDFLAGS: ${SRCDIR}/librapidyenc_linux_arm64.a -lstdc++ #include "rapidyenc.h" */ import "C" import ( - "bytes" - "fmt" - "os" "sync" "unsafe" ) @@ -52,44 +45,3 @@ func (e *Encoder) Encode(src []byte) []byte { return dst[:length] } - -// UUEncode encodes src as UUencoded text with header/footer. -func UUEncode(src []byte, filename string, mode int) []byte { - var buf bytes.Buffer - // Write header - fmt.Fprintf(&buf, "begin %03o %s\r\n", mode, filename) - // Encode in 45-byte chunks - for i := 0; i < len(src); i += 45 { - chunk := src[i:] - if len(chunk) > 45 { - chunk = chunk[:45] - } - // Write length character - buf.WriteByte(byte(len(chunk) + 32)) - // Encode chunk - for j := 0; j < len(chunk); j += 3 { - var a, b, c byte - a = chunk[j] - if j+1 < len(chunk) { - b = chunk[j+1] - } - if j+2 < len(chunk) { - c = chunk[j+2] - } - buf.WriteByte(((a >> 2) & 0x3F) + 32) - buf.WriteByte((((a << 4) | ((b >> 4) & 0x0F)) & 0x3F) + 32) - buf.WriteByte((((b << 2) | ((c >> 6) & 0x03)) & 0x3F) + 32) - buf.WriteByte((c & 0x3F) + 32) - } - buf.WriteString("\r\n") - } - // Write footer - buf.WriteString("end\r\n") - return buf.Bytes() -} - -// WriteUUEncodedFile encodes src and writes it as UUencoded data to the given file. -func WriteUUEncodedFile(filename string, src []byte, outPath string, mode int) error { - encoded := UUEncode(src, filename, mode) - return os.WriteFile(outPath, encoded, 0644) -} diff --git a/rapidyenc/librapidyenc_darwin.a b/rapidyenc/librapidyenc_darwin.a new file mode 120000 index 0000000..6ae5162 --- /dev/null +++ b/rapidyenc/librapidyenc_darwin.a @@ -0,0 +1 @@ +librapidyenc.a \ No newline at end of file diff --git a/rapidyenc/librapidyenc_linux_amd64.a b/rapidyenc/librapidyenc_linux_amd64.a new file mode 120000 index 0000000..6ae5162 --- /dev/null +++ b/rapidyenc/librapidyenc_linux_amd64.a @@ -0,0 +1 @@ +librapidyenc.a \ No newline at end of file diff --git a/rapidyenc/librapidyenc_linux_arm64.a b/rapidyenc/librapidyenc_linux_arm64.a new file mode 120000 index 0000000..6ae5162 --- /dev/null +++ b/rapidyenc/librapidyenc_linux_arm64.a @@ -0,0 +1 @@ +librapidyenc.a \ No newline at end of file diff --git a/rapidyenc/librapidyenc_windows_amd64.a b/rapidyenc/librapidyenc_windows_amd64.a new file mode 120000 index 0000000..6ae5162 --- /dev/null +++ b/rapidyenc/librapidyenc_windows_amd64.a @@ -0,0 +1 @@ +librapidyenc.a \ No newline at end of file From 151a9c0490fc88842e9647519c8aaee45330cc87 Mon Sep 17 00:00:00 2001 From: go-while <137838162+go-while@users.noreply.github.com> Date: Fri, 22 Aug 2025 23:12:48 +0100 Subject: [PATCH 043/152] Merge pull request #66 from go-while/copilot/fix-65 Add comprehensive GitHub Copilot instructions for NZBreX development workflow --- .github/copilot-instructions.md | 180 ++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..f6b97e2 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,180 @@ +# NZBreX - NZB Refresh X + +NZBreX is a Go command-line tool for restoring missing Usenet articles by downloading them from providers where they are still available and re-uploading them to others. The application requires building the rapidyenc C library first, then the Go application with CGO enabled. + +Always reference these instructions first and fallback to search or bash commands only when you encounter unexpected information that does not match the info here. + +## Working Effectively + +### Prerequisites and Dependencies +- Install required system packages: + ```bash + apt-get install build-essential cmake ca-certificates curl git dpkg-dev wget + # For cross-compilation (optional): + apt-get install gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu mingw-w64 gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 g++-aarch64-linux-gnu + ``` +- Go 1.24.3 or later is required (as specified in go.mod) +- CMake 3.31+ is required for building rapidyenc + +### Bootstrap, Build, and Test the Repository +1. **Build rapidyenc C library first** (REQUIRED): + ```bash + cd rapidyenc && ./build_rapidyenc_linux-amd64.sh + ``` + - Takes ~15 seconds. NEVER CANCEL. Set timeout to 60+ seconds. + - Creates `rapidyenc/rapidyenc/build/rapidyenc_static/librapidyenc.a` + - Also builds CLI tools: `rapidyenc_cli` and `rapidyenc_bench` + +2. **Build Go application**: + ```bash + ./local_build_linux-amd64.sh + ``` + - OR manually: `go build -o NZBreX -tags other .` + - Takes ~15-30 seconds total. NEVER CANCEL. Set timeout to 120+ seconds. + - First build downloads dependencies (~25 seconds) + - Subsequent builds are faster (~5 seconds) + +3. **Run tests**: + ```bash + go test ./rapidyenc/ + ``` + - Takes ~5 seconds. Set timeout to 60+ seconds. + +4. **Run tests with race detector**: + ```bash + go test -race ./rapidyenc/ + ``` + - Takes ~5 seconds. Set timeout to 60+ seconds. + +5. **Run code validation**: + ```bash + ./vet.sh + ``` + - Takes ~7 seconds. Set timeout to 60+ seconds. + +6. **Test rapidyenc integration**: + ```bash + ./NZBreX -testrapidyenc + ``` + - Tests yenc encoding/decoding functionality + - Takes ~1 second. Should always work. + +## Validation + +### Essential Validation Steps +- **ALWAYS run through the complete build sequence** after making changes: + 1. Clean build: `rm -rf rapidyenc/rapidyenc/build && rm -f NZBreX` + 2. Build rapidyenc: `cd rapidyenc && ./build_rapidyenc_linux-amd64.sh` + 3. Build application: `./local_build_linux-amd64.sh` + 4. Test rapidyenc: `./NZBreX -testrapidyenc` + 5. Run validation: `./vet.sh` + +- **Manual validation scenarios**: + - Test help: `./NZBreX -help` (should show usage) + - Test version: `./NZBreX -version` (shows build version) + - Validate rapidyenc: `./NZBreX -testrapidyenc` (tests yenc codec) + +- **CRITICAL**: Application functionality cannot be fully tested without Usenet provider access. The provider.ygg.json requires yggdrasil network access which may not be available in all environments. + +- **Before committing**: Always run `./vet.sh` and ensure the application builds and basic commands work. + +## Running the Application + +### Basic Usage +```bash +# Check application help +./NZBreX -help + +# Test with sample NZB (check-only mode, safe) +./NZBreX -checkonly -nzb nzbs/debian-11.6.0-amd64-netinst.iso.nzb.gz -provider provider.sample.json + +# Full operation (requires valid Usenet provider configuration) +./NZBreX -cd /path/to/cache -checkfirst -nzb nzbs/example.nzb.gz -provider provider.json +``` + +### Key Command-Line Options +- `-checkonly`: Safe mode - only check article availability, no downloads/uploads +- `-cd string`: Cache directory path (required for most operations) +- `-nzb string`: Path to NZB file (default: nzbs/ubuntu-24.04-live-server-amd64.iso.nzb.gz) +- `-provider string`: Provider configuration file (default: provider.json) +- `-verbose`: Enable detailed output (default: true) +- `-testrapidyenc`: Test rapidyenc functionality and exit + +## Common Tasks + +### Repository Structure +``` +/home/runner/work/NZBreX/NZBreX/ +├── README.md # Main documentation +├── main.go # Application entry point +├── *.go # Go source files (Config.go, Workers.go, etc.) +├── go.mod, go.sum # Go module files +├── rapidyenc/ # C library for yenc encoding/decoding +│ ├── rapidyenc/ # Actual rapidyenc source code +│ ├── build_rapidyenc_*.sh # Build scripts for different platforms +│ └── required.deb-packages.txt # Required system packages +├── nzbs/ # Sample NZB files for testing +├── yenc/ # Test data for yenc functionality +├── provider.sample.json # Sample provider configuration +├── provider.ygg.json # Yggdrasil network test providers +├── cleanHeaders.txt # Headers to clean during processing +├── local_build_linux-amd64.sh # Main build script +├── vet.sh # Code validation script +├── race_linux.sh # Race detector test script +└── .github/workflows/ # CI/CD workflows +``` + +### Key Files and Locations +- **Build output**: `NZBreX` (main executable) +- **rapidyenc library**: `rapidyenc/rapidyenc/build/rapidyenc_static/librapidyenc.a` +- **Test data**: `nzbs/` directory contains sample NZB files +- **Configuration**: `provider.sample.json` and `config.sample.json` +- **Scripts**: All `.sh` files are executable build/test scripts + +### Cross-compilation Support +- **Linux ARM64**: `cd rapidyenc && ./build_rapidyenc_linux-arm64.sh` +- **Windows AMD64**: `cd rapidyenc && ./crossbuild_rapidyenc_windows-amd64.sh` +- **Darwin AMD64**: `cd rapidyenc && ./crossbuild_rapidyenc_darwin-amd64.sh` + +## Build Times and Timeouts + +### Expected Timing (Never Cancel These Operations) +- **rapidyenc C library build**: 10-20 seconds. NEVER CANCEL. Set timeout to 120+ seconds. +- **Go application build (first time)**: 20-30 seconds. NEVER CANCEL. Set timeout to 180+ seconds. +- **Go application build (subsequent)**: 3-10 seconds. Set timeout to 120+ seconds. +- **Go tests**: 3-10 seconds. Set timeout to 60+ seconds. +- **Go vet**: 5-10 seconds. Set timeout to 60+ seconds. +- **Full clean build**: 30-45 seconds total. NEVER CANCEL. Set timeout to 300+ seconds. + +### Critical Timeout Warnings +- **NEVER CANCEL** any build or test command before the expected time +- **ALWAYS** set timeouts with 3-5x buffer above expected times +- **CMake configuration** can take 3-5 seconds - this is normal +- **Go module downloads** on first build can take 20+ seconds - this is normal + +## Important Notes + +### rapidyenc C Library +- **MUST** be built before Go application +- Built using CMake with extensive CPU optimization (AVX, SSE, NEON) +- Provides high-performance yenc encoding/decoding +- Test with `./NZBreX -testrapidyenc` after any changes + +### Network and Providers +- Application requires Usenet provider access for full functionality +- Test providers (provider.ygg.json) may not be reachable in all environments +- Use `-checkonly` flag for safe testing without network operations +- Provider configuration errors will show as "network unreachable" - this is expected in test environments + +### Development Workflow +- **Always** build rapidyenc first: `cd rapidyenc && ./build_rapidyenc_linux-amd64.sh` +- **Always** test with: `./NZBreX -testrapidyenc` +- **Always** validate with: `./vet.sh` +- **Always** verify help works: `./NZBreX -help` +- Use `-checkonly` for safe testing without network dependencies + +### CI/CD Integration +- GitHub Actions workflows are in `.github/workflows/` +- Self-hosted runners support multiple platforms (Ubuntu, Debian, cross-compilation) +- Build artifacts include static binaries, .deb packages, and checksums +- Race detector tests are included in CI pipeline \ No newline at end of file From f3f1a850c912a401e208ec83915434929c1e3691 Mon Sep 17 00:00:00 2001 From: go-while Date: Fri, 22 Aug 2025 23:43:01 +0100 Subject: [PATCH 044/152] Update Workflows --- .github/workflows/build_and_publish_test.yml | 591 ++++++++----------- .github/workflows/codeql.yml | 105 ++-- 2 files changed, 274 insertions(+), 422 deletions(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 09145cb..7f7766f 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -1,8 +1,9 @@ -name: "TEST Self-Hosted Runners" +name: "Build and Publish (GitHub Runners)" on: push: branches: + - main - 'testing-agent' tags: - "testing" @@ -10,10 +11,10 @@ on: types: [published] workflow_dispatch: repository_dispatch: - #pull_request: - # branches: - # - main - # - "test*" + pull_request: + branches: + - main + - "test*" permissions: contents: write @@ -21,274 +22,66 @@ permissions: jobs: build_matrix: - name: Build ${{ matrix.name }}-${{ matrix.libc }} - runs-on: - - self-hosted - - ${{ matrix.runner_label }} + name: Build ${{ matrix.name }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 120 strategy: fail-fast: false - max-parallel: 8 + max-parallel: 6 matrix: include: - # Alpine Linux cross-builds (musl) - # amd64 - - runner_label: ubu2204 - os: linux - arch: amd64 - name: alpine-amd64 - libc: musl - # arm64 - - runner_label: ubu2204 - os: linux - arch: arm64 - name: alpine-arm64 - libc: musl - # i386 - #- runner_label: ubu2204 - # os: linux - # arch: 386 - # name: alpine-i386 - # libc: musl - # armv7 - #- runner_label: ubu2204 - # os: linux - # arch: arm - # name: alpine-armv7 - # libc: musl - - # FreeBSD cross-builds - # amd64 - #- runner_label: ubu2204 - # os: freebsd - # arch: amd64 - # name: freebsd-amd64 - # arm64 - #- runner_label: ubu2204 - # os: freebsd - # arch: arm64 - # name: freebsd-arm64 - # i386 - #- runner_label: ubu2204 - # os: freebsd - # arch: 386 - # name: freebsd-i386 - # armv7 - #- runner_label: ubu2204 - # os: freebsd - # arch: arm - # name: freebsd-armv7 - - # OpenBSD cross-builds - # amd64 - #- runner_label: ubu2204 - # os: openbsd - # arch: amd64 - # name: openbsd-amd64 - ## arm64 - #- runner_label: ubu2204 - # os: openbsd - # arch: arm64 - # name: openbsd-arm64 - ## i386 - #- runner_label: ubu2204 - # os: openbsd - # arch: 386 - # name: openbsd-i386 - - # NetBSD cross-builds - # amd64 - #- runner_label: ubu2204 - # os: netbsd - # arch: amd64 - # name: netbsd-amd64 - ## arm64 - #- runner_label: ubu2204 - # os: netbsd - # arch: arm64 - # name: netbsd-arm64 - ## i386 - #- runner_label: ubu2204 - # os: netbsd - # arch: 386 - # name: netbsd-i386 - ## armv7 - #- runner_label: ubu2204 - # os: netbsd - # arch: arm - # name: netbsd-armv7 - - # macOS cross-builds - # amd64 - #- runner_label: ubu2204 - # os: darwin - # arch: amd64 - # name: macos-amd64 - ## arm64 - #- runner_label: ubu2204 - # os: darwin - # arch: arm64 - # name: macos-arm64 - - # Windows cross-builds - # amd64 - - runner_label: ubu2204 - os: windows - arch: amd64 - name: windows-amd64 - # arm64 - #- runner_label: ubu2204 - # os: windows - # arch: arm64 - # name: windows-arm64 - # i386 - #- runner_label: ubu2204 - # os: windows - # arch: 386 - # name: windows-i386 - - # Debian 10-12 self-hosted builds amd64 - # amd64 - - runner_label: deb10 - os: linux - arch: amd64 - name: debian-10-amd64 - libc: glibc - - runner_label: deb11 + # Linux builds using GitHub runners + # Ubuntu 22.04 amd64 (glibc) + - runner: ubuntu-22.04 os: linux arch: amd64 - name: debian-11-amd64 + name: ubuntu-22.04-amd64 libc: glibc - - runner_label: deb12 + cross_compile: false + + # Ubuntu latest amd64 (glibc) + - runner: ubuntu-latest os: linux arch: amd64 - name: debian-12-amd64 + name: ubuntu-latest-amd64 libc: glibc + cross_compile: false - # Debian 10-12 self-hosted builds arm64 - # arm64 - - runner_label: deb10 + # Linux ARM64 cross-compilation from Ubuntu + - runner: ubuntu-latest os: linux arch: arm64 - name: debian-10-arm64 - libc: glibc - - runner_label: deb11 - os: linux - arch: arm64 - name: debian-11-arm64 - libc: glibc - - runner_label: deb12 - os: linux - arch: arm64 - name: debian-12-arm64 + name: linux-arm64 libc: glibc + cross_compile: true - # Debian 10-12 self-hosted builds 32-bit ARM - # armv7 - #- runner_label: deb10 - # os: linux - # arch: arm - # name: debian-10-armv7 - # libc: glibc - #- runner_label: deb11 - # os: linux - # arch: arm - # name: debian-11-armv7 - # libc: glibc - #- runner_label: deb12 - # os: linux - # arch: arm - # name: debian-12-armv7 - # libc: glibc - - # Debian 10-12 self-hosted builds i386 - # i386 - #- runner_label: deb10 - # os: linux - # arch: 386 - # name: debian-10-i386 - # libc: glibc - #- runner_label: deb11 - # os: linux - # arch: 386 - # name: debian-11-i386 - # libc: glibc - #- runner_label: deb12 - # os: linux - # arch: 386 - # name: debian-12-i386 - # libc: glibc - - # Ubuntu 20.04 - 24.04 self-hosted builds amd64 - # amd64 - - runner_label: ubu2004 + # Alpine Linux (musl) cross-compilation + - runner: ubuntu-latest os: linux arch: amd64 - name: ubuntu-20.04-amd64 - libc: glibc - - runner_label: ubu2204 - os: linux + name: alpine-amd64 + libc: musl + cross_compile: true + + # Windows cross-compilation from Ubuntu + - runner: ubuntu-latest + os: windows arch: amd64 - name: ubuntu-22.04-amd64 - libc: glibc - - runner_label: ubu2404 - os: linux + name: windows-amd64 + cross_compile: true + + # macOS builds + - runner: macos-13 + os: darwin arch: amd64 - name: ubuntu-24.04-amd64 - libc: glibc + name: macos-amd64 + cross_compile: false - # Ubuntu 20.04 - 24.04 self-hosted builds arm64 - # arm64 - - runner_label: ubu2004 - os: linux - arch: arm64 - name: ubuntu-20.04-arm64 - libc: glibc - - runner_label: ubu2204 - os: linux - arch: arm64 - name: ubuntu-22.04-arm64 - libc: glibc - - runner_label: ubu2404 - os: linux + - runner: macos-14 + os: darwin arch: arm64 - name: ubuntu-24.04-arm64 - libc: glibc - - # Ubuntu 20.04 - 24.04 self-hosted builds 32-bit ARM - # armv7 - #- runner_label: ubu2004 - # os: linux - # arch: arm - # name: ubuntu-20.04-armv7 - # libc: glibc - #- runner_label: ubu2204 - # os: linux - # arch: arm - # name: ubuntu-22.04-armv7 - # libc: glibc - #- runner_label: ubu2404 - # os: linux - # arch: arm - # name: ubuntu-24.04-armv7 - # libc: glibc - - # Ubuntu 20.04 - 24.04 self-hosted builds i386 - # i386 - #- runner_label: ubu2004 - # os: linux - # arch: 386 - # name: ubuntu-20.04-i386 - # libc: glibc - #- runner_label: ubu2204 - # os: linux - # arch: 386 - # name: ubuntu-22.04-i386 - # libc: glibc - #- runner_label: ubu2404 - # os: linux - # arch: 386 - # name: ubuntu-24.04-i386 - # libc: glibc + name: macos-arm64 + cross_compile: false steps: - name: Show actor @@ -308,16 +101,117 @@ jobs: echo "Pull request not allowed for user ${{ github.event.pull_request.user.login }}" exit 1 - - name: "@Start Clean directory" - run: | - echo "deleting workspace" - rm -rf dist build ${{ github.workspace }}/ - mkdir -vp ${{ github.workspace }} - - name: "Checkout code" uses: actions/checkout@v4 -# - name: Clean Go module cache + - name: "Set up Go" + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + cache: true + + # Install system dependencies based on runner OS + - name: "Install build dependencies (Ubuntu)" + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y build-essential cmake ca-certificates curl git dpkg-dev wget + # For cross-compilation + if [ "${{ matrix.cross_compile }}" = "true" ]; then + case "${{ matrix.os }}-${{ matrix.arch }}" in + linux-arm64) + sudo apt-get install -y gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu g++-aarch64-linux-gnu + ;; + windows-amd64) + sudo apt-get install -y mingw-w64 gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 + ;; + linux-amd64) + # For Alpine/musl builds + if [ "${{ matrix.libc }}" = "musl" ]; then + sudo apt-get install -y musl-tools musl-dev + fi + ;; + esac + fi + + - name: "Install build dependencies (macOS)" + if: runner.os == 'macOS' + run: | + brew install cmake + + - name: "Clone rapidyenc" + run: | + cd rapidyenc + if [ ! -e rapidyenc ]; then + ./clone_rapidyenc.sh + if [ ! -e rapidyenc/.git ]; then + echo "rapidyenc/ src not found, exiting" + exit 1 + fi + else + echo "rapidyenc/ src exists, skipping clone" + fi + + # Build rapidyenc for different platforms + - name: "Build rapidyenc (Linux amd64)" + if: matrix.os == 'linux' && matrix.arch == 'amd64' && !matrix.cross_compile + timeout-minutes: 5 + run: cd rapidyenc && ./build_rapidyenc_linux-amd64.sh + + - name: "Build rapidyenc (Linux arm64 cross-compile)" + if: matrix.os == 'linux' && matrix.arch == 'arm64' && matrix.cross_compile + timeout-minutes: 5 + run: | + cd rapidyenc && ./build_rapidyenc_linux-arm64.sh + echo "CC=aarch64-linux-gnu-gcc" >> $GITHUB_ENV + echo "CXX=aarch64-linux-gnu-g++" >> $GITHUB_ENV + echo "CGO_ENABLED=1" >> $GITHUB_ENV + + - name: "Build rapidyenc (Windows amd64 cross-compile)" + if: matrix.os == 'windows' && matrix.arch == 'amd64' + timeout-minutes: 5 + run: | + cd rapidyenc && ./crossbuild_rapidyenc_windows-amd64.sh + echo "CC=x86_64-w64-mingw32-gcc" >> $GITHUB_ENV + echo "CGO_ENABLED=1" >> $GITHUB_ENV + + - name: "Build rapidyenc (macOS)" + if: matrix.os == 'darwin' + timeout-minutes: 5 + run: | + cd rapidyenc + if [ "${{ matrix.arch }}" = "amd64" ]; then + ./crossbuild_rapidyenc_darwin-amd64.sh || ./build_rapidyenc_linux-amd64.sh + else + # For arm64, we might need to build natively or cross-compile + ./build_rapidyenc_linux-amd64.sh + fi + + - name: "Build rapidyenc (Alpine/musl)" + if: matrix.libc == 'musl' + timeout-minutes: 5 + run: | + cd rapidyenc && ./build_rapidyenc_linux-amd64.sh + # Note: May need Alpine-specific build script + + # Run tests with race detector for supported platforms + - name: "Run race detector tests (amd64/linux only)" + if: matrix.os == 'linux' && matrix.arch == 'amd64' && !matrix.cross_compile + timeout-minutes: 5 + run: go test -race ./rapidyenc/ + + - name: "Run normal tests (other platforms)" + if: ${{ !(matrix.os == 'linux' && matrix.arch == 'amd64' && !matrix.cross_compile) }} + timeout-minutes: 5 + run: go test ./rapidyenc/ + + - name: "Test rapidyenc integration" + if: ${{ !(matrix.cross_compile) }} + timeout-minutes: 2 + run: | + # Build first if not cross-compiling + go build -o NZBreX -tags other . + ./NZBreX -testrapidyenc# - name: Clean Go module cache # run: | # rm -rf ~/.cache/go-build ~/go/pkg/mod # - name: Restore Go modules cache @@ -413,7 +307,6 @@ jobs: GITHUB_REF_NAME: ${{ github.ref_name }} GITHUB_SHA: ${{ github.sha }} run: | - echo "COMPILER=SHR" >> "$GITHUB_ENV" # If event is tag or release, use the tag; otherwise, use 0.0.0-SNAPSHOT SAFE_VERSION="$(echo "${GITHUB_REF_NAME}" | sed 's|/|-|g')" if [[ "${GITHUB_REF_TYPE}" == "tag" || "${GITHUB_EVENT_NAME}" == "release" ]]; then @@ -503,88 +396,93 @@ jobs: dpkg-deb --root-owner-group --build builds/${{ matrix.name }} NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.deb fi - - name: "Upload binary to private GaRuS (Git-actions-Runner-upload-Server)" - run: | - upload_with_retry() { - local file="$1" - local filename="$2" - local attempt=1 - local delay=30 - local max_attempts=30 - local FNstr="" - if [ -n "$filename" ]; then - # filename to use has been supplied in $2 - FNstr=";filename=$filename" - fi - size=$(du -b $file|cut -f1) - human=$(du -h $file) - while [ $attempt -le $max_attempts ]; do - test $attempt -gt 1 && echo "Upload attempt $attempt for $file..." - if curl --silent -f -F "file=@$file$FNstr" \ - -H "X-Git-Repo: ${{ github.repository }}" \ - -H "X-Git-Ref: ${{ env.VERSION }}" \ - -H "X-Git-SHA7: ${{ env.SHA7 }}" \ - -H "X-Git-Comp: ${{ env.COMPILER }}" \ - -H "X-Git-MATRIX: ${{ matrix.name }}" \ - -H "X-Auth-Token: ${{ secrets.BUILD_TEST_UPLOAD_TOKEN }}" \ - http://10.20.0.1:58080/upload.php; then - echo "Upload succeeded for $file size=$size [$human]" - return 0 - else - echo "Upload failed for $file. Retrying in $delay seconds..." - sleep $delay - attempt=$(( attempt + 1 )) - fi - done - echo "Upload failed for $file after $max_attempts attempts." - return 1 - } - set -e - for ext in zip deb tgz xz; do - file="NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.${ext}" - if [ -e "$file" ]; then - for algo in 256 512; do - sha="sha${algo}sum" - $sha "$file" > "$file.$sha" - echo -e "\n$file.$sha" - cat "$file.$sha" | cut -d" " -f1 - upload_with_retry "$file.$sha" - done - upload_with_retry "$file" - fi - done - # catch the hash sums of the binary files for uploading - sha256file="builds/${{ matrix.name }}/usr/bin/${{ env.BINARY }}.sha256sum" - sha512file="builds/${{ matrix.name }}/usr/bin/${{ env.BINARY }}.sha512sum" - # finally rename the sha sums to good filenames - cp -v "$sha256file" "NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha256sum" - cp -v "$sha512file" "NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha512sum" - upload_with_retry "NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha256sum" - upload_with_retry "NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha512sum" - # - DIST="dist.${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.$(hostname).tar.gz" - if [ -e "$DIST" ]; then - echo "$DIST already exists. Skipping tar." - else - ls -lha - tar -czvf "$DIST" builds/ - echo "Created $DIST from builds/" - du -b "$DIST"; du -hs "$DIST"; - sha256sum "$DIST" > "${DIST}.sha256sum" - sha512sum "$DIST" > "${DIST}.sha512sum" - upload_with_retry "${DIST}.sha512sum" - upload_with_retry "${DIST}.sha256sum" - upload_with_retry "${DIST}" - fi - - -# - name: "Upload Binary to github blob storage" -# uses: actions/upload-artifact@v4 -# with: -# name: binary-${{ matrix.name }} -# path: builds/${{ matrix.name }}/usr/bin/* -# retention-days: 1 -# if-no-files-found: warn + # Private GaRuS upload commented out - only accessible from self-hosted runners + # The following upload section is commented out because it's only accessible from self-hosted runners + # Uncomment when using self-hosted runners with access to the private upload server + #- name: "Upload binary to private GaRuS (Git-actions-Runner-upload-Server)" + # run: | + # upload_with_retry() { + # local file="$1" + # local filename="$2" + # local attempt=1 + # local delay=30 + # local max_attempts=30 + # local FNstr="" + # if [ -n "$filename" ]; then + # # filename to use has been supplied in $2 + # FNstr=";filename=$filename" + # fi + # size=$(du -b $file|cut -f1) + # human=$(du -h $file) + # while [ $attempt -le $max_attempts ]; do + # test $attempt -gt 1 && echo "Upload attempt $attempt for $file..." + # if curl --silent -f -F "file=@$file$FNstr" \ + # -H "X-Git-Repo: ${{ github.repository }}" \ + # -H "X-Git-Ref: ${{ env.VERSION }}" \ + # -H "X-Git-SHA7: ${{ env.SHA7 }}" \ + # -H "X-Git-Comp: ${{ env.COMPILER }}" \ + # -H "X-Git-MATRIX: ${{ matrix.name }}" \ + # -H "X-Auth-Token: ${{ secrets.BUILD_TEST_UPLOAD_TOKEN }}" \ + # http://10.20.0.1:58080/upload.php; then + # echo "Upload succeeded for $file size=$size [$human]" + # return 0 + # else + # echo "Upload failed for $file. Retrying in $delay seconds..." + # sleep $delay + # attempt=$(( attempt + 1 )) + # fi + # done + # echo "Upload failed for $file after $max_attempts attempts." + # return 1 + # } + # set -e + # for ext in zip deb tgz xz; do + # file="NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.${ext}" + # if [ -e "$file" ]; then + # for algo in 256 512; do + # sha="sha${algo}sum" + # $sha "$file" > "$file.$sha" + # echo -e "\n$file.$sha" + # cat "$file.$sha" | cut -d" " -f1 + # upload_with_retry "$file.$sha" + # done + # upload_with_retry "$file" + # fi + # done + # # catch the hash sums of the binary files for uploading + # sha256file="builds/${{ matrix.name }}/usr/bin/${{ env.BINARY }}.sha256sum" + # sha512file="builds/${{ matrix.name }}/usr/bin/${{ env.BINARY }}.sha512sum" + # # finally rename the sha sums to good filenames + # cp -v "$sha256file" "NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha256sum" + # cp -v "$sha512file" "NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha512sum" + # upload_with_retry "NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha256sum" + # upload_with_retry "NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha512sum" + # # + # DIST="dist.${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.$(hostname).tar.gz" + # if [ -e "$DIST" ]; then + # echo "$DIST already exists. Skipping tar." + # else + # ls -lha + # tar -czvf "$DIST" builds/ + # echo "Created $DIST from builds/" + # du -b "$DIST"; du -hs "$DIST"; + # sha256sum "$DIST" > "${DIST}.sha256sum" + # sha512sum "$DIST" > "${DIST}.sha512sum" + # upload_with_retry "${DIST}.sha512sum" + # upload_with_retry "${DIST}.sha256sum" + # upload_with_retry "${DIST}" + # fi + + # GitHub Artifacts upload for builds (optional - can be enabled if needed) + - name: "Upload Binary to GitHub Artifacts" + uses: actions/upload-artifact@v4 + with: + name: binary-${{ matrix.name }} + path: | + builds/${{ matrix.name }}/usr/bin/* + NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.* + retention-days: 7 + if-no-files-found: warn - name: "Upload Release Assets to github" if: github.event_name == 'release' @@ -597,12 +495,3 @@ jobs: NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.zip NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha256sum NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha512sum - -# - name: "Backup go cache" -# run: runner/go-cache-backup.sh - - - name: "@End Clean directory" - run: | - echo "deleting workspace" - rm -rf dist build ${{ github.workspace }}/ - mkdir -vp ${{ github.workspace }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f9bcb81..0ab8abf 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -2,18 +2,19 @@ name: "CodeQL Advanced" on: push: - # branches: - # - 'testing' + branches: + - main + - testing tags: - - "testing" + - testing release: types: [published] workflow_dispatch: repository_dispatch: - #pull_request: - # branches: - # - main - # - "test*" + pull_request: + branches: + - main + - testing schedule: - cron: '30 05 * * 1' @@ -21,6 +22,7 @@ jobs: analyze: name: Analyze (${{ matrix.runner_name }}-${{ matrix.language }}) runs-on: ${{ matrix.runner }} + timeout-minutes: 60 permissions: security-events: write packages: read @@ -31,63 +33,21 @@ jobs: fail-fast: false matrix: include: - # run codeQL on SHR: Go + # GitHub hosted runners only - language: go build-mode: autobuild - runner: [self-hosted, ubu2204-gor] - runner_name: "runner-ubu2204-gor-001" - ram: 8 - cache: false + runner: ubuntu-22.04 + runner_name: "GH-ubuntu-22.04" + ram: 6 + cache: true - # run codeQL on SHR: Go - language: go build-mode: autobuild - runner: [self-hosted, ubu2404-gor] - runner_name: "runner-ubu2404-gor-001" - ram: 8 - cache: false - - # run codeQL on SHR: C/C++ - - language: c-cpp - build-mode: manual - runner: [self-hosted, ubu2204-gor] - runner_name: "runner-ubu2204-gor-001" - ram: 8 - cache: false - - # run codeQL on SHR: C/C++ - - language: c-cpp - build-mode: manual - runner: [self-hosted, ubu2404-gor] - runner_name: "runner-ubu2404-gor-001" - ram: 8 - cache: false - - # run codeQL on SHR: Actions - - language: actions - build-mode: none - runner: [self-hosted, ubu2204-gor] - runner_name: "runner-ubu2204-gor-001" - ram: 8 - cache: false - - # run codeQL on SHR: Actions - - language: actions - build-mode: none - runner: [self-hosted, ubu2404-gor] - runner_name: "runner-ubu2404-gor-001" - ram: 8 - cache: false - - # run codeQL on GH: Actions - - language: actions - build-mode: none - runner: ubuntu-latest - runner_name: "GH-ubuntu-latest" + runner: ubuntu-24.04 + runner_name: "GH-ubuntu-24.04" ram: 6 cache: true - # run codeQL on GH: Go - language: go build-mode: autobuild runner: ubuntu-latest @@ -95,7 +55,7 @@ jobs: ram: 6 cache: true - # run codeQL on GH: C/C++ + # C/C++ analysis for rapidyenc - language: c-cpp build-mode: manual runner: ubuntu-latest @@ -103,6 +63,14 @@ jobs: ram: 6 cache: true + # Actions analysis + - language: actions + build-mode: none + runner: ubuntu-latest + runner_name: "GH-ubuntu-latest" + ram: 6 + cache: true + steps: - name: Show actor @@ -122,22 +90,22 @@ jobs: echo "Pull request not allowed for user ${{ github.event.pull_request.user.login }}" exit 1 - - name: "@Start Clean directory" - run: | - echo "deleting workspace" - rm -rf dist build ${{ github.workspace }}/ - mkdir -vp ${{ github.workspace }} - - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Install build dependencies + if: matrix.language == 'c-cpp' + run: | + sudo apt-get update + sudo apt-get install -y build-essential cmake ca-certificates curl git dpkg-dev wget + - name: Set up Go if: matrix.language == 'go' uses: actions/setup-go@v5 with: - go-version: '1.24.3' # or whatever version you need + go-version-file: 'go.mod' cache: ${{ matrix.cache }} - name: Initialize CodeQL @@ -156,6 +124,7 @@ jobs: - name: Build (manual, for C/C++) if: matrix.language == 'c-cpp' && matrix.build-mode != 'autobuild' # Run the build step only for C/C++ jobs + timeout-minutes: 10 run: | cd rapidyenc if [ ! -e rapidyenc ]; then @@ -170,10 +139,4 @@ jobs: ./build_rapidyenc_linux-amd64.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 - - - name: "@End Clean directory" - run: | - echo "deleting workspace" - rm -rf dist build ${{ github.workspace }}/ - mkdir -vp ${{ github.workspace }} \ No newline at end of file + uses: github/codeql-action/analyze@v3 \ No newline at end of file From 9d2fbb3a470df342e163fc26d08ae5a730d4a2fe Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 11:04:05 +0000 Subject: [PATCH 045/152] test build --- .github/workflows/build_and_publish_test.yml | 114 +++++++++++-------- rapidyenc/clone_rapidyenc-org.sh | 1 + rapidyenc/clone_rapidyenc.sh | 2 +- rapidyenc/decoder.go | 31 ++++- rapidyenc/decoder_test.go | 55 +++++---- rapidyenc/encoder.go | 37 ++++++ 6 files changed, 166 insertions(+), 74 deletions(-) create mode 100755 rapidyenc/clone_rapidyenc-org.sh diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 7f7766f..94e38e9 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -31,6 +31,14 @@ jobs: matrix: include: # Linux builds using GitHub runners + # Ubuntu 20.04 amd64 (glibc) + - runner: ubuntu-20.04 + os: linux + arch: amd64 + name: ubuntu-20.04-amd64 + libc: glibc + cross_compile: false + # Ubuntu 22.04 amd64 (glibc) - runner: ubuntu-22.04 os: linux @@ -221,29 +229,17 @@ jobs: uses: actions/setup-go@v5 with: go-version-file: 'go.mod' - cache: false - # if your self-hosted runners are not reset (i.e., they persist their workspace and their Go module cache between jobs), - # using the GitHub Actions dependency cache provides little to no benefit. - # In fact, it can sometimes cause issues (like stale or conflicting cache states) - # and may slow down your pipeline due to unnecessary cache save/restore steps. - # On self-hosted runners where the module cache persists between jobs, - # letting Go’s native cache do its job is almost always faster and more reliable. - # Results with 44 jobs: - # Before (with Actions cache): ~8 minutes - # After (cache: false): ~2m 27s - -# - name: Cache Go modules -# uses: actions/cache@v4 -# with: -# path: | -# ~/.cache/go-build -# ~/go/pkg/mod -# key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} -# restore-keys: | -# ${{ runner.os }}-go- - -# - name: "Restore go cache" -# run: runner/go-cache-restore.sh + cache: true + + - name: Cache Go modules + uses: actions/cache@v4 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- - name: "clone rapidyenc" run: | @@ -258,45 +254,66 @@ jobs: echo "rapidyenc/ src exists, skipping clone" fi - - name: "compile rapidyenc linux-amd64" - if: matrix.os == 'linux' && matrix.arch == 'amd64' + # Build rapidyenc for different platforms + - name: "Build rapidyenc (Linux amd64)" + if: matrix.os == 'linux' && matrix.arch == 'amd64' && !matrix.cross_compile + timeout-minutes: 5 run: cd rapidyenc && ./build_rapidyenc_linux-amd64.sh - #- name: "compile rapidyenc linux-386" - # if: matrix.os == 'linux' && matrix.arch == '386' - # run: cd rapidyenc && ./build_rapidyenc_linux-386.sh + - name: "Build rapidyenc (Linux arm64 cross-compile)" + if: matrix.os == 'linux' && matrix.arch == 'arm64' && matrix.cross_compile + timeout-minutes: 5 + run: | + cd rapidyenc && ./build_rapidyenc_linux-arm64.sh + echo "CC=aarch64-linux-gnu-gcc" >> $GITHUB_ENV + echo "CXX=aarch64-linux-gnu-g++" >> $GITHUB_ENV + echo "CGO_ENABLED=1" >> $GITHUB_ENV - - name: "compile rapidyenc windows-amd64" + - name: "Build rapidyenc (Windows amd64 cross-compile)" if: matrix.os == 'windows' && matrix.arch == 'amd64' + timeout-minutes: 5 run: | cd rapidyenc && ./crossbuild_rapidyenc_windows-amd64.sh echo "CC=x86_64-w64-mingw32-gcc" >> $GITHUB_ENV echo "CGO_ENABLED=1" >> $GITHUB_ENV - - name: "compile rapidyenc linux-arm64" - if: matrix.os == 'linux' && matrix.arch == 'arm64' + - name: "Build rapidyenc (macOS)" + if: matrix.os == 'darwin' + timeout-minutes: 5 run: | - cd rapidyenc && ./build_rapidyenc_linux-arm64.sh - echo "CC=aarch64-linux-gnu-gcc" >> $GITHUB_ENV - echo "CXX=aarch64-linux-gnu-g++" >> $GITHUB_ENV - echo "CGO_ENABLED=1" >> $GITHUB_ENV + cd rapidyenc + if [ "${{ matrix.arch }}" = "amd64" ]; then + ./crossbuild_rapidyenc_darwin-amd64.sh || ./build_rapidyenc_linux-amd64.sh + else + # For arm64, we might need to build natively or cross-compile + ./build_rapidyenc_linux-amd64.sh + fi - #- name: "Set up Win386 cross-compiler" - # if: matrix.goarch == '386' && matrix.os == 'windows' - # run: | - # cd rapidyenc && ./build_rapidyenc_win_386.sh - # echo "CC=x86_64-w64-mingw32-gcc" >> $GITHUB_ENV - # echo "CXX=x86_64-w64-mingw32-g++" >> $GITHUB_ENV - # echo "CGO_ENABLED=1" >> $GITHUB_ENV + - name: "Build rapidyenc (Alpine/musl)" + if: matrix.libc == 'musl' + timeout-minutes: 5 + run: | + cd rapidyenc && ./build_rapidyenc_linux-amd64.sh + # Note: May need Alpine-specific build script # Run tests with race detector for supported platforms - #- name: "Run race detector tests (amd64/linux only)" - # if: matrix.os == 'linux' && matrix.arch == 'amd64' - # run: go test -race ./... + - name: "Run race detector tests (amd64/linux only)" + if: matrix.os == 'linux' && matrix.arch == 'amd64' && !matrix.cross_compile + timeout-minutes: 5 + run: go test -race ./rapidyenc/ + + - name: "Run normal tests (other platforms)" + if: ${{ !(matrix.os == 'linux' && matrix.arch == 'amd64' && !matrix.cross_compile) }} + timeout-minutes: 5 + run: go test ./rapidyenc/ - #- name: "Run normal tests (other platforms)" - # if: ${{ !(matrix.os == 'linux' && matrix.arch == 'amd64') }} - # run: go test ./... + - name: "Test rapidyenc integration" + if: ${{ !(matrix.cross_compile) }} + timeout-minutes: 2 + run: | + # Build first if not cross-compiling + go build -o NZBreX -tags other . + ./NZBreX -testrapidyenc - name: "Set GaRuS Variables" id: vars @@ -307,6 +324,7 @@ jobs: GITHUB_REF_NAME: ${{ github.ref_name }} GITHUB_SHA: ${{ github.sha }} run: | + echo "COMPILER=GHR" >> "$GITHUB_ENV" # Changed from SHR to GHR (GitHub Runners) # If event is tag or release, use the tag; otherwise, use 0.0.0-SNAPSHOT SAFE_VERSION="$(echo "${GITHUB_REF_NAME}" | sed 's|/|-|g')" if [[ "${GITHUB_REF_TYPE}" == "tag" || "${GITHUB_EVENT_NAME}" == "release" ]]; then diff --git a/rapidyenc/clone_rapidyenc-org.sh b/rapidyenc/clone_rapidyenc-org.sh new file mode 100755 index 0000000..b80d71d --- /dev/null +++ b/rapidyenc/clone_rapidyenc-org.sh @@ -0,0 +1 @@ +git clone https://github.com/animetosho/rapidyenc.git || exit 1 diff --git a/rapidyenc/clone_rapidyenc.sh b/rapidyenc/clone_rapidyenc.sh index b80d71d..fefd3d5 100755 --- a/rapidyenc/clone_rapidyenc.sh +++ b/rapidyenc/clone_rapidyenc.sh @@ -1 +1 @@ -git clone https://github.com/animetosho/rapidyenc.git || exit 1 +# test ! -d rapidyenc && git clone https://github.com/animetosho/rapidyenc.git diff --git a/rapidyenc/decoder.go b/rapidyenc/decoder.go index 5da2b8b..26cf40e 100644 --- a/rapidyenc/decoder.go +++ b/rapidyenc/decoder.go @@ -3,7 +3,9 @@ package rapidyenc /* #cgo CFLAGS: -I${SRCDIR}/src #cgo darwin LDFLAGS: ${SRCDIR}/librapidyenc_darwin.a -lstdc++ -#cgo windows,amd64 LDFLAGS: ${SRCDIR}/librapidyenc_windows_amd64.a -lstdc++ +#cgo windows,amd64 LDFLAGS: ${SRCDIR}/librapidyenc.a -static-libstdc++ -static-libgcc +#cgo windows,386 LDFLAGS: ${SRCDIR}/librapidyenc.a -static-libstdc++ -static-libgcc +#cgo windows,arm LDFLAGS: ${SRCDIR}/librapidyenc.a -static-libstdc++ -static-libgcc #cgo linux,amd64 LDFLAGS: ${SRCDIR}/librapidyenc_linux_amd64.a -lstdc++ #cgo linux,arm64 LDFLAGS: ${SRCDIR}/librapidyenc_linux_arm64.a -lstdc++ #include "rapidyenc.h" @@ -293,6 +295,8 @@ transform: switch d.format { case FormatYenc: err = fmt.Errorf("[rapidyenc] FormatYenc end of article without finding \"=ybegin\" header: %w", ErrDataMissing) + case FormatUU: + err = fmt.Errorf("[rapidyenc] FormatUU end of article without finding \"begin\" header: %w", ErrDataMissing) default: err = fmt.Errorf("[rapidyenc] FormatUnknown end of article without finding any *begin header: %w", ErrDataMissing) } @@ -300,10 +304,12 @@ transform: switch d.format { case FormatYenc: err = fmt.Errorf("[rapidyenc] FormatYenc end of article without finding \"=yend\" trailer: %w", ErrDataMissing) + case FormatUU: + err = fmt.Errorf("[rapidyenc] FormatUU end of article without finding \"end\" trailer: %w", ErrDataCorruption) default: err = fmt.Errorf("[rapidyenc] FormatUnknown end of article without finding any *end header: %w", ErrDataMissing) } - } else if (!d.part && d.m.Size != d.endSize) || (d.endSize != d.actualSize) { + } else if d.format == FormatYenc && ((!d.part && d.m.Size != d.endSize) || (d.endSize != d.actualSize)) { err = fmt.Errorf("[rapidyenc] expected size %d but got %d: %w", d.m.Size, d.actualSize, ErrDataCorruption) } else if d.crc && d.expectedCrc != d.m.Hash { // If we have a segment ID, use it for debugging otherwise use an empty string. @@ -331,8 +337,19 @@ transform: d.processYenc(line) goto transform case FormatUU: - // TODO: does not uudecode, for now just copies encoded data - nDst += copy(dst[nDst:], line) + d.processYenc(line) // Process UU headers (begin/end) + if d.body && !bytes.HasPrefix(line, []byte("begin ")) && !bytes.HasPrefix(line, []byte("end")) { + // Decode UU data line + decoded, err := UUdecode(line) + if err != nil { + return nDst, nSrc, fmt.Errorf("[rapidyenc] UUdecode error: %w", err) + } + if decoded != nil { + nDst += copy(dst[nDst:], decoded) + d.actualSize += int64(len(decoded)) + d.hash.Write(decoded) + } + } } } @@ -343,6 +360,12 @@ transform: switch d.format { case FormatYenc: return nDst, nSrc, fmt.Errorf("[rapidyenc] FormatYenc end of article without finding \"=ybegin\" or \"=yend\" header: %w", ErrDataMissing) + case FormatUU: + if !d.begin { + return nDst, nSrc, fmt.Errorf("[rapidyenc] FormatUU end of article without finding \"begin\" header: %w", ErrDataMissing) + } else { + return nDst, nSrc, fmt.Errorf("[rapidyenc] FormatUU end of article without finding \"end\" trailer: %w", ErrDataCorruption) + } default: return nDst, nSrc, fmt.Errorf("[rapidyenc] FormatUnknown end of article without finding any *begin or *end header: %w", ErrDataMissing) } diff --git a/rapidyenc/decoder_test.go b/rapidyenc/decoder_test.go index 8011e95..1972059 100644 --- a/rapidyenc/decoder_test.go +++ b/rapidyenc/decoder_test.go @@ -519,6 +519,40 @@ func RapidyencDecoderFilesTest(t *testing.T) (errs []error) { return errs } +// GenerateTestUUEncodedFiles creates uuencode/test1.uue and uuencode/test2.uue with test content. +func GenerateTestUUEncodedFiles(t *testing.T) error { + _ = os.MkdirAll("uuencode", 0755) + t.Logf("Generating test UUencoded files...") + // Create test content for uuencode files + // These contents are just examples, you can modify them as needed. + // The files will be created in the "uuencode" directory. + + // check if files already exist + if _, err := os.Stat(filepath.Join("uuencode", "test1.uue")); err == nil { + t.Logf("File uuencode/test1.uue already exists, skipping creation.") + return nil + } + if _, err := os.Stat(filepath.Join("uuencode", "test2.uue")); err == nil { + t.Logf("File uuencode/test2.uue already exists, skipping creation.") + return nil + } + content1 := []byte("Hello from test1!\nThis is a test file.\n") + content2 := []byte("Another file for test2.\nWith more lines.\n1234567890\r\n") + + uue1 := UUEncode(content1, "test1.txt", 644) + uue2 := UUEncode(content2, "test2.txt", 644) + + if err := os.WriteFile(filepath.Join("uuencode", "test1.uue"), uue1, 0644); err != nil { + t.Errorf("Failed to write uuencode/test1.uue: %v", err) + return err + } + if err := os.WriteFile(filepath.Join("uuencode", "test2.uue"), uue2, 0644); err != nil { + t.Errorf("Failed to write uuencode/test2.uue: %v", err) + return err + } + return nil +} + // TestUUdecodeFiles runs UUdecode tests on sample files. // It reads UUencoded files, decodes them, and checks for integrity. func TestUUdecodeFiles(t *testing.T) { @@ -573,24 +607,3 @@ func TestUUdecodeFiles(t *testing.T) { t.Logf("Successfully checked %d UUencoded files.", checked) } } - -// GenerateTestUUEncodedFiles creates uuencode/test1.uue and uuencode/test2.uue with test content. -func GenerateTestUUEncodedFiles(t *testing.T) error { - _ = os.MkdirAll("uuencode", 0755) - t.Logf("Generating test UUencoded files...") - // Create test content for uuencode files - // These contents are just examples, you can modify them as needed. - // The files will be created in the "uuencode" directory. - - // check if files already exist - if _, err := os.Stat(filepath.Join("uuencode", "test1.uue")); err == nil { - t.Logf("File uuencode/test1.uue already exists, skipping creation.") - return nil - } - if _, err := os.Stat(filepath.Join("uuencode", "test2.uue")); err == nil { - t.Logf("File uuencode/test2.uue already exists, skipping creation.") - return nil - } - - return nil -} diff --git a/rapidyenc/encoder.go b/rapidyenc/encoder.go index 4ddd480..7554221 100644 --- a/rapidyenc/encoder.go +++ b/rapidyenc/encoder.go @@ -10,6 +10,8 @@ package rapidyenc */ import "C" import ( + "bytes" + "fmt" "sync" "unsafe" ) @@ -45,3 +47,38 @@ func (e *Encoder) Encode(src []byte) []byte { return dst[:length] } + +// UUEncode encodes src as UUencoded text with header/footer. +func UUEncode(src []byte, filename string, mode int) []byte { + var buf bytes.Buffer + // Write header + fmt.Fprintf(&buf, "begin %03o %s\r\n", mode, filename) + // Encode in 45-byte chunks + for i := 0; i < len(src); i += 45 { + chunk := src[i:] + if len(chunk) > 45 { + chunk = chunk[:45] + } + // Write length character + buf.WriteByte(byte(len(chunk) + 32)) + // Encode chunk + for j := 0; j < len(chunk); j += 3 { + var a, b, c byte + a = chunk[j] + if j+1 < len(chunk) { + b = chunk[j+1] + } + if j+2 < len(chunk) { + c = chunk[j+2] + } + buf.WriteByte(((a >> 2) & 0x3F) + 32) + buf.WriteByte((((a << 4) | ((b >> 4) & 0x0F)) & 0x3F) + 32) + buf.WriteByte((((b << 2) | ((c >> 6) & 0x03)) & 0x3F) + 32) + buf.WriteByte((c & 0x3F) + 32) + } + buf.WriteString("\r\n") + } + // Write footer + buf.WriteString("end\r\n") + return buf.Bytes() +} From ff6e13c7676f1be10c9d1ec82bd14641b01d92d0 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 11:14:17 +0000 Subject: [PATCH 046/152] Update build_and_publish_test.yml --- .github/workflows/build_and_publish_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 94e38e9..f7bac5a 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -4,7 +4,7 @@ on: push: branches: - main - - 'testing-agent' + - 'testing' tags: - "testing" release: From 38968698b9bc07be987b32a1d3f75e2a8114fd5e Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 11:22:55 +0000 Subject: [PATCH 047/152] Update build_and_publish_test.yml --- .github/workflows/build_and_publish_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index f7bac5a..68060e5 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -219,7 +219,8 @@ jobs: run: | # Build first if not cross-compiling go build -o NZBreX -tags other . - ./NZBreX -testrapidyenc# - name: Clean Go module cache + ./NZBreX -testrapidyenc +# - name: Clean Go module cache # run: | # rm -rf ~/.cache/go-build ~/go/pkg/mod # - name: Restore Go modules cache From 062c66438b9b49be903cff80e2e9b409cff52aed Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 11:28:46 +0000 Subject: [PATCH 048/152] Update build_and_publish_test.yml --- .github/workflows/build_and_publish_test.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 68060e5..ce2eb6c 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -174,6 +174,7 @@ jobs: echo "CC=aarch64-linux-gnu-gcc" >> $GITHUB_ENV echo "CXX=aarch64-linux-gnu-g++" >> $GITHUB_ENV echo "CGO_ENABLED=1" >> $GITHUB_ENV + echo "GOARCH=arm64" >> $GITHUB_ENV - name: "Build rapidyenc (Windows amd64 cross-compile)" if: matrix.os == 'windows' && matrix.arch == 'amd64' @@ -182,6 +183,7 @@ jobs: cd rapidyenc && ./crossbuild_rapidyenc_windows-amd64.sh echo "CC=x86_64-w64-mingw32-gcc" >> $GITHUB_ENV echo "CGO_ENABLED=1" >> $GITHUB_ENV + echo "GOOS=windows" >> $GITHUB_ENV - name: "Build rapidyenc (macOS)" if: matrix.os == 'darwin' @@ -208,11 +210,15 @@ jobs: timeout-minutes: 5 run: go test -race ./rapidyenc/ - - name: "Run normal tests (other platforms)" - if: ${{ !(matrix.os == 'linux' && matrix.arch == 'amd64' && !matrix.cross_compile) }} + - name: "Run normal tests (non-cross-compile platforms)" + if: ${{ !(matrix.cross_compile) && !(matrix.os == 'linux' && matrix.arch == 'amd64') }} timeout-minutes: 5 run: go test ./rapidyenc/ + - name: "Skip tests for cross-compiled platforms" + if: matrix.cross_compile + run: echo "Skipping tests for cross-compiled platform ${{ matrix.name }} (requires emulation or native hardware)" + - name: "Test rapidyenc integration" if: ${{ !(matrix.cross_compile) }} timeout-minutes: 2 @@ -269,6 +275,7 @@ jobs: echo "CC=aarch64-linux-gnu-gcc" >> $GITHUB_ENV echo "CXX=aarch64-linux-gnu-g++" >> $GITHUB_ENV echo "CGO_ENABLED=1" >> $GITHUB_ENV + echo "GOARCH=arm64" >> $GITHUB_ENV - name: "Build rapidyenc (Windows amd64 cross-compile)" if: matrix.os == 'windows' && matrix.arch == 'amd64' @@ -277,6 +284,7 @@ jobs: cd rapidyenc && ./crossbuild_rapidyenc_windows-amd64.sh echo "CC=x86_64-w64-mingw32-gcc" >> $GITHUB_ENV echo "CGO_ENABLED=1" >> $GITHUB_ENV + echo "GOOS=windows" >> $GITHUB_ENV - name: "Build rapidyenc (macOS)" if: matrix.os == 'darwin' @@ -303,11 +311,15 @@ jobs: timeout-minutes: 5 run: go test -race ./rapidyenc/ - - name: "Run normal tests (other platforms)" - if: ${{ !(matrix.os == 'linux' && matrix.arch == 'amd64' && !matrix.cross_compile) }} + - name: "Run normal tests (non-cross-compile platforms)" + if: ${{ !(matrix.cross_compile) && !(matrix.os == 'linux' && matrix.arch == 'amd64') }} timeout-minutes: 5 run: go test ./rapidyenc/ + - name: "Skip tests for cross-compiled platforms" + if: matrix.cross_compile + run: echo "Skipping tests for cross-compiled platform ${{ matrix.name }} (requires emulation or native hardware)" + - name: "Test rapidyenc integration" if: ${{ !(matrix.cross_compile) }} timeout-minutes: 2 From eeb4d241f30d77a4706a2d09c66aea14ddcdc8b5 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 11:30:05 +0000 Subject: [PATCH 049/152] Update build_and_publish_test.yml --- .github/workflows/build_and_publish_test.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index ce2eb6c..81c1117 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -32,12 +32,12 @@ jobs: include: # Linux builds using GitHub runners # Ubuntu 20.04 amd64 (glibc) - - runner: ubuntu-20.04 - os: linux - arch: amd64 - name: ubuntu-20.04-amd64 - libc: glibc - cross_compile: false + #- runner: ubuntu-20.04 + # os: linux + # arch: amd64 + # name: ubuntu-20.04-amd64 + # libc: glibc + # cross_compile: false # Ubuntu 22.04 amd64 (glibc) - runner: ubuntu-22.04 From 7c54911e6959fd17d06d7768a3e9288ca0650248 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 11:33:25 +0000 Subject: [PATCH 050/152] Update build_and_publish_test.yml --- .github/workflows/build_and_publish_test.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 81c1117..39f560e 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -116,7 +116,7 @@ jobs: uses: actions/setup-go@v5 with: go-version-file: 'go.mod' - cache: true + cache: false # Install system dependencies based on runner OS - name: "Install build dependencies (Ubuntu)" @@ -238,15 +238,15 @@ jobs: go-version-file: 'go.mod' cache: true - - name: Cache Go modules - uses: actions/cache@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- + #- name: Cache Go modules + # uses: actions/cache@v4 + # with: + # path: | + # ~/.cache/go-build + # ~/go/pkg/mod + # key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + # restore-keys: | + # ${{ runner.os }}-go- - name: "clone rapidyenc" run: | From 95e1c6fa9f57d5e4713741f25d2a451921ba686e Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 11:49:38 +0000 Subject: [PATCH 051/152] Update build_and_publish_test.yml --- .github/workflows/build_and_publish_test.yml | 67 +++++++++++++------- 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 39f560e..14a7503 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -31,27 +31,19 @@ jobs: matrix: include: # Linux builds using GitHub runners - # Ubuntu 20.04 amd64 (glibc) - #- runner: ubuntu-20.04 - # os: linux - # arch: amd64 - # name: ubuntu-20.04-amd64 - # libc: glibc - # cross_compile: false - - # Ubuntu 22.04 amd64 (glibc) - - runner: ubuntu-22.04 + # Ubuntu 24.04 amd64 (glibc) - Latest GA + - runner: ubuntu-24.04 os: linux arch: amd64 - name: ubuntu-22.04-amd64 + name: ubuntu-24.04-amd64 libc: glibc cross_compile: false - # Ubuntu latest amd64 (glibc) - - runner: ubuntu-latest + # Ubuntu 22.04 amd64 (glibc) - Stable LTS + - runner: ubuntu-22.04 os: linux arch: amd64 - name: ubuntu-latest-amd64 + name: ubuntu-22.04-amd64 libc: glibc cross_compile: false @@ -78,17 +70,40 @@ jobs: name: windows-amd64 cross_compile: true - # macOS builds - - runner: macos-13 + # macOS builds - Multiple versions for maximum compatibility + # macOS 15 Intel (amd64) - Latest GA + - runner: macos-15 + os: darwin + arch: amd64 + name: macos-15-amd64 + cross_compile: false + + # macOS 14 Intel (amd64) - Stable + - runner: macos-14 os: darwin arch: amd64 - name: macos-amd64 + name: macos-14-amd64 cross_compile: false + # macOS 15 ARM64 - Latest GA + - runner: macos-15 + os: darwin + arch: arm64 + name: macos-15-arm64 + cross_compile: false + + # macOS 14 ARM64 - Stable - runner: macos-14 os: darwin arch: arm64 - name: macos-arm64 + name: macos-14-arm64 + cross_compile: false + + # macOS 26 ARM64 - Public Preview (Beta) - For early compatibility testing + - runner: macos-26 + os: darwin + arch: arm64 + name: macos-26-arm64-beta cross_compile: false steps: @@ -380,10 +395,18 @@ jobs: echo "provider.ygg.json not found! Build cannot continue." >&2 exit 1 fi - sha256sum builds/${{ matrix.name }}/usr/bin/$binary_name > builds/${{ matrix.name }}/usr/bin/$binary_name.sha256sum - cat builds/${{ matrix.name }}/usr/bin/$binary_name.sha256sum - sha512sum builds/${{ matrix.name }}/usr/bin/$binary_name > builds/${{ matrix.name }}/usr/bin/$binary_name.sha512sum - cat builds/${{ matrix.name }}/usr/bin/$binary_name.sha512sum + # Use appropriate checksum commands based on OS + if [ "${{ runner.os }}" == "macOS" ]; then + shasum -a 256 builds/${{ matrix.name }}/usr/bin/$binary_name | awk '{print $1" "$2}' > builds/${{ matrix.name }}/usr/bin/$binary_name.sha256sum + cat builds/${{ matrix.name }}/usr/bin/$binary_name.sha256sum + shasum -a 512 builds/${{ matrix.name }}/usr/bin/$binary_name | awk '{print $1" "$2}' > builds/${{ matrix.name }}/usr/bin/$binary_name.sha512sum + cat builds/${{ matrix.name }}/usr/bin/$binary_name.sha512sum + else + sha256sum builds/${{ matrix.name }}/usr/bin/$binary_name > builds/${{ matrix.name }}/usr/bin/$binary_name.sha256sum + cat builds/${{ matrix.name }}/usr/bin/$binary_name.sha256sum + sha512sum builds/${{ matrix.name }}/usr/bin/$binary_name > builds/${{ matrix.name }}/usr/bin/$binary_name.sha512sum + cat builds/${{ matrix.name }}/usr/bin/$binary_name.sha512sum + fi # # packing the build # From 8861850bf1a089d7c2e2bb93ffb80b9f5106d3d7 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 11:56:17 +0000 Subject: [PATCH 052/152] Update build_and_publish_test.yml --- .github/workflows/build_and_publish_test.yml | 22 ++++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 14a7503..d91948e 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -3,18 +3,18 @@ name: "Build and Publish (GitHub Runners)" on: push: branches: - - main - - 'testing' + # - main + - testing tags: - - "testing" + - testing release: types: [published] workflow_dispatch: repository_dispatch: - pull_request: - branches: - - main - - "test*" + #pull_request: + # branches: + # - main + # - "test*" permissions: contents: write @@ -71,15 +71,15 @@ jobs: cross_compile: true # macOS builds - Multiple versions for maximum compatibility - # macOS 15 Intel (amd64) - Latest GA - - runner: macos-15 + # macOS 15 Intel (amd64) - Latest GA (requires -large for Intel) + - runner: macos-15-large os: darwin arch: amd64 name: macos-15-amd64 cross_compile: false - # macOS 14 Intel (amd64) - Stable - - runner: macos-14 + # macOS 14 Intel (amd64) - Stable (requires -large for Intel) + - runner: macos-14-large os: darwin arch: amd64 name: macos-14-amd64 From 634051100184b71be23847e5a54c7c824d9019cd Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 12:08:03 +0000 Subject: [PATCH 053/152] Update build_and_publish_test.yml --- .github/workflows/build_and_publish_test.yml | 26 ++++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index d91948e..832e57a 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -64,26 +64,26 @@ jobs: cross_compile: true # Windows cross-compilation from Ubuntu + # Windows amd64 (64-bit x86) - runner: ubuntu-latest os: windows arch: amd64 name: windows-amd64 cross_compile: true - # macOS builds - Multiple versions for maximum compatibility - # macOS 15 Intel (amd64) - Latest GA (requires -large for Intel) - - runner: macos-15-large - os: darwin - arch: amd64 - name: macos-15-amd64 - cross_compile: false + # Windows 386 (32-bit x86) + - runner: ubuntu-latest + os: windows + arch: 386 + name: windows-386 + cross_compile: true - # macOS 14 Intel (amd64) - Stable (requires -large for Intel) - - runner: macos-14-large - os: darwin - arch: amd64 - name: macos-14-amd64 - cross_compile: false + # Windows ARM64 + - runner: ubuntu-latest + os: windows + arch: arm64 + name: windows-arm64 + cross_compile: true # macOS 15 ARM64 - Latest GA - runner: macos-15 From cbbc0dabe0ed909d4a73d1770e5152eb18198185 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 12:17:36 +0000 Subject: [PATCH 054/152] Update build_and_publish_test.yml --- .github/workflows/build_and_publish_test.yml | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 832e57a..13663d8 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -64,26 +64,18 @@ jobs: cross_compile: true # Windows cross-compilation from Ubuntu - # Windows amd64 (64-bit x86) + # Windows amd64 (64-bit x86) - Covers vast majority of Windows users - runner: ubuntu-latest os: windows arch: amd64 name: windows-amd64 cross_compile: true - # Windows 386 (32-bit x86) - - runner: ubuntu-latest - os: windows - arch: 386 - name: windows-386 - cross_compile: true - - # Windows ARM64 - - runner: ubuntu-latest - os: windows - arch: arm64 - name: windows-arm64 - cross_compile: true + # Note: Windows 386 (32-bit) and ARM64 removed due to: + # - No rapidyenc cross-compilation scripts for these architectures + # - Windows 32-bit is legacy/deprecated + # - Windows ARM64 has minimal market share + # - Windows amd64 covers 95%+ of users # macOS 15 ARM64 - Latest GA - runner: macos-15 From ec992777b619a902d2137c2a5d15a604e6822e84 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 12:19:10 +0000 Subject: [PATCH 055/152] Update codeql.yml --- .github/workflows/codeql.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0ab8abf..7bee3a9 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -38,21 +38,21 @@ jobs: build-mode: autobuild runner: ubuntu-22.04 runner_name: "GH-ubuntu-22.04" - ram: 6 + ram: 12 cache: true - language: go build-mode: autobuild runner: ubuntu-24.04 runner_name: "GH-ubuntu-24.04" - ram: 6 + ram: 12 cache: true - language: go build-mode: autobuild runner: ubuntu-latest runner_name: "GH-ubuntu-latest" - ram: 6 + ram: 12 cache: true # C/C++ analysis for rapidyenc @@ -60,7 +60,7 @@ jobs: build-mode: manual runner: ubuntu-latest runner_name: "GH-ubuntu-latest" - ram: 6 + ram: 12 cache: true # Actions analysis @@ -68,7 +68,7 @@ jobs: build-mode: none runner: ubuntu-latest runner_name: "GH-ubuntu-latest" - ram: 6 + ram: 12 cache: true From e6e0b3c1f86417fb05cb91f6ab95c5bbba801000 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 12:25:53 +0000 Subject: [PATCH 056/152] test 1.25.1 --- .github/workflows/build_and_publish_test.yml | 4 ++-- go.mod | 10 +++++----- go.sum | 18 ++++++++---------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 13663d8..af7b8c0 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -122,7 +122,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v5 with: - go-version-file: 'go.mod' + go-version: '1.25.1' cache: false # Install system dependencies based on runner OS @@ -242,7 +242,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v5 with: - go-version-file: 'go.mod' + go-version: '1.24.3' cache: true #- name: Cache Go modules diff --git a/go.mod b/go.mod index e635711..17cf8b6 100644 --- a/go.mod +++ b/go.mod @@ -1,24 +1,24 @@ module github.com/go-while/NZBreX -go 1.24.3 +go 1.25.1 require ( github.com/Tensai75/nzbparser v0.1.0 github.com/go-while/go-cpu-mem-profiler v0.0.0-20240612221627-856954a5fc83 + golang.org/x/crypto v0.43.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/crypto v0.39.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( - github.com/Tensai75/subjectparser v0.1.0 // indirect + github.com/Tensai75/subjectparser v0.1.1 // indirect github.com/go-while/go-loggedrwmutex v0.0.0-20250601032232-b0aa20cdcb2c github.com/gorilla/mux v1.8.1 // indirect github.com/stretchr/testify v1.10.0 - golang.org/x/net v0.40.0 // indirect - golang.org/x/text v0.26.0 + golang.org/x/net v0.46.0 // indirect + golang.org/x/text v0.30.0 ) diff --git a/go.sum b/go.sum index fad6773..aa98f0d 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/Tensai75/nzbparser v0.1.0 h1:6RppAuWFahqu/kKjWO5Br0xuEYcxGz+XBTxYc+qvPo4= github.com/Tensai75/nzbparser v0.1.0/go.mod h1:IUIIaeGaYp2dLAAF29BWYeKTfI4COvXaeQAzQiTOfMY= -github.com/Tensai75/subjectparser v0.1.0 h1:6fEWnRov8lDHxJS2EWqY6VonwYfrIRN+k8h8H7fFwHA= -github.com/Tensai75/subjectparser v0.1.0/go.mod h1:PNBFBnkOGbVDfX+56ZmC4GKSpqoRMCF1Y44xYd7NLGI= +github.com/Tensai75/subjectparser v0.1.1 h1:SAlaEKUmaalt4QH+UFBzEP6iHqA994iouFqvFPSM9y0= +github.com/Tensai75/subjectparser v0.1.1/go.mod h1:PNBFBnkOGbVDfX+56ZmC4GKSpqoRMCF1Y44xYd7NLGI= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-while/go-cpu-mem-profiler v0.0.0-20240612221627-856954a5fc83 h1:vehfiL7LsK8bJQZdVuJxcMem4AP2HwPEQ8orUUnIA+E= @@ -14,14 +14,12 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= -golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= -golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= -golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= +golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= +golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From 47d7b9833d9359c8792ab3ed28605d4f92f67a95 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 12:41:06 +0000 Subject: [PATCH 057/152] build scripts --- README.md | 58 ++++++++++++++++- build_windows.bat | 93 +++++++++++++++++++++++++++ local_crossbuild_windows-amd64.sh | 74 ++++++++++++++++++++- local_crossbuild_windows-amd64_old.sh | 11 ++++ 4 files changed, 230 insertions(+), 6 deletions(-) create mode 100644 build_windows.bat create mode 100755 local_crossbuild_windows-amd64_old.sh diff --git a/README.md b/README.md index 3781219..1a8d0d0 100644 --- a/README.md +++ b/README.md @@ -75,9 +75,10 @@ The tool uses the NNTP commands: ## Installation -1. Compile from source or download the latest executable from the [Releases](../../releases) page. -2. Configure `provider.json` with your Usenet provider details. -3. Run the program from the command line: +1. **Download pre-built binaries** from the [Releases](../../releases) page, or +2. **Compile from source** (see Build Instructions below) +3. Configure `provider.json` with your Usenet provider details. +4. Run the program from the command line: ```sh ./nzbrex -checkonly -nzb nzbs/ubuntu-24.04-live-server-amd64.iso.nzb.gz @@ -85,6 +86,37 @@ The tool uses the NNTP commands: ./nzbrex --cd=cache --checkfirst --nzb=nzbs/ubuntu-24.04-live-server-amd64.iso.nzb.gz ``` +### Build Instructions + +NZBreX requires the rapidyenc C++ library to be compiled before building the Go application. + +#### Linux Build +```sh +# Build rapidyenc library +cd rapidyenc && ./build_rapidyenc_linux-amd64.sh && cd .. + +# Build NZBreX +go build . +``` + +#### Windows Native Build +```cmd +REM Install dependencies: Go, MinGW-w64, CMake +REM Then run: +build_windows.bat +``` + +#### Windows Cross-Compilation (from Linux) +```sh +# Install dependencies +sudo apt install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 cmake + +# Run cross-compilation script +./local_crossbuild_windows-amd64.sh +``` + +The Windows executable will be statically linked and only depend on core Windows system libraries. + --- ## Running NZBreX @@ -218,6 +250,26 @@ Example output: --- +## Troubleshooting + +### Windows Executable Issues + +If you encounter an error like "The code execution cannot proceed because libc++.so.6.dll was not found" on Windows: + +1. **Download the latest release** - This issue has been fixed in recent builds +2. **Ensure you're using the correct Windows version** - Download the Windows-specific executable from the releases page +3. **The executable is self-contained** - No additional runtime libraries should be required + +The Windows executable is statically linked and only depends on core Windows system libraries that are guaranteed to be available. + +### Build Issues + +- **rapidyenc library missing**: Run the appropriate rapidyenc build script before building NZBreX +- **Cross-compilation fails**: Ensure MinGW-w64 and cmake are installed +- **Link errors**: Make sure the rapidyenc library matches your target platform + +--- + ## Contributing & Reporting Issues Please open an issue on [GitHub Issues](../../issues) or a [Discussion](../../discussions) if you encounter problems. diff --git a/build_windows.bat b/build_windows.bat new file mode 100644 index 0000000..d4c5452 --- /dev/null +++ b/build_windows.bat @@ -0,0 +1,93 @@ +@echo off +REM Windows build script for NZBreX +REM Requires: Git, Go 1.19+, MinGW-w64, CMake + +echo Building NZBreX for Windows... + +REM Check if Go is available +go version >nul 2>&1 +if errorlevel 1 ( + echo ERROR: Go not found. Please install Go 1.24.3 or later. + echo Download from: https://golang.org/dl/ + pause + exit /b 1 +) + +REM Check if CMake is available +cmake --version >nul 2>&1 +if errorlevel 1 ( + echo ERROR: CMake not found. Please install CMake. + echo Download from: https://cmake.org/download/ + pause + exit /b 1 +) + +REM Check if we're in the right directory +if not exist "rapidyenc" ( + echo ERROR: rapidyenc directory not found. Please run this script from the NZBreX root directory. + pause + exit /b 1 +) + +echo Building rapidyenc library for Windows... +cd rapidyenc +if not exist "rapidyenc" ( + echo ERROR: rapidyenc submodule not initialized. Please run: git submodule update --init --recursive + pause + exit /b 1 +) + +REM Build rapidyenc +rmdir /s /q rapidyenc\build 2>nul +mkdir rapidyenc\build +cd rapidyenc\build + +REM Try to configure with MinGW Makefiles generator +cmake .. -G "MinGW Makefiles" +if errorlevel 1 ( + REM Try Ninja generator if MinGW Makefiles fails + cmake .. -G "Ninja" + if errorlevel 1 ( + REM Try MSYS Makefiles generator if Ninja fails + cmake .. -G "MSYS Makefiles" + if errorlevel 1 ( + echo ERROR: CMake configuration failed with all known generators. + echo Please ensure MinGW-w64, Ninja, or MSYS2 are properly installed. + echo You may need to install MSYS2 and MinGW-w64 from: https://www.msys2.org/ + echo Or install Ninja from: https://ninja-build.org/ + pause + cd ..\..\..\ + exit /b 1 + ) + ) +) + +cmake --build . --config Release +if errorlevel 1 ( + echo ERROR: Failed to build rapidyenc library. + pause + cd ..\..\..\ + exit /b 1 +) + +REM Copy library files +copy rapidyenc_static\librapidyenc.a ..\..\librapidyenc.a +cd ..\..\..\ + +echo rapidyenc library built successfully. + +echo Building NZBreX executable... +set CGO_ENABLED=1 +go build -o NZBreX_ry.exe -tags "windows rapidyenc" . +if errorlevel 1 ( + echo ERROR: Failed to build NZBreX executable. + pause + exit /b 1 +) + +echo. +echo Build completed successfully! +echo Executable: NZBreX_ry.exe +echo. +echo To test the build, run: NZBreX_ry.exe -version +pause diff --git a/local_crossbuild_windows-amd64.sh b/local_crossbuild_windows-amd64.sh index 571a25e..a3ad878 100755 --- a/local_crossbuild_windows-amd64.sh +++ b/local_crossbuild_windows-amd64.sh @@ -1,11 +1,79 @@ +#!/usr/bin/env bash + +# Windows cross-compilation script for NZBreX +# Requires: gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 cmake + +set -e # Exit on any error + +echo "Building NZBreX for Windows (amd64)..." + +# Check if required tools are available +if ! command -v x86_64-w64-mingw32-gcc &> /dev/null; then + echo "ERROR: x86_64-w64-mingw32-gcc not found. Please install MinGW-w64:" + echo " Ubuntu/Debian: sudo apt install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64" + echo " CentOS/RHEL: sudo yum install mingw64-gcc mingw64-gcc-c++" + exit 1 +fi + +if ! command -v cmake &> /dev/null; then + echo "ERROR: cmake not found. Please install cmake:" + echo " Ubuntu/Debian: sudo apt install cmake" + echo " CentOS/RHEL: sudo yum install cmake" + exit 1 +fi + +# Build rapidyenc for Windows +echo "Building rapidyenc library for Windows..." rm -rf rapidyenc/rapidyenc/build mkdir -p rapidyenc/rapidyenc/build -cd rapidyenc && ./crossbuild_rapidyenc_windows-amd64.sh && cd ../ +cd rapidyenc +if ! ./crossbuild_rapidyenc_windows-amd64.sh; then + echo "ERROR: Failed to build rapidyenc library for Windows" + exit 1 +fi +cd ../ +# Check if rapidyenc library was built successfully +if [ ! -f "rapidyenc/librapidyenc.a" ]; then + echo "ERROR: rapidyenc static library not found at rapidyenc/librapidyenc.a" + exit 1 +fi + +echo "rapidyenc library built successfully" + +# Set up cross-compilation environment export GOOS=windows export GOARCH=amd64 export CGO_ENABLED=1 export CC=x86_64-w64-mingw32-gcc -go build -o NZBreX_ry.exe -tags "windows rapidyenc" . -exit $? +echo "Cross-compiling Go application for Windows..." +# Build with static linking to avoid dependency on MinGW runtime DLLs +if ! go build -o NZBreX.exe -ldflags "-linkmode external -extldflags '-static'" -tags "windows rapidyenc" .; then + echo "ERROR: Failed to build Windows executable" + exit 1 +fi + +# Verify the executable was created +if [ ! -f "NZBreX.exe" ]; then + echo "ERROR: Windows executable not created" + exit 1 +fi + +echo "Windows executable built successfully: NZBreX.exe" + +# Check dependencies (optional, for debugging) +if command -v x86_64-w64-mingw32-objdump &> /dev/null; then + echo "DLL dependencies:" + if ! x86_64-w64-mingw32-objdump -p "NZBreX.exe" > objdump_output.txt; then + echo " ERROR: objdump failed to analyze NZBreX.exe" + elif ! grep -q "DLL Name" objdump_output.txt; then + echo " (none found)" + else + grep "DLL Name" objdump_output.txt + fi + rm -f objdump_output.txt +fi + +echo "Build completed successfully!" +exit 0 diff --git a/local_crossbuild_windows-amd64_old.sh b/local_crossbuild_windows-amd64_old.sh new file mode 100755 index 0000000..571a25e --- /dev/null +++ b/local_crossbuild_windows-amd64_old.sh @@ -0,0 +1,11 @@ +rm -rf rapidyenc/rapidyenc/build +mkdir -p rapidyenc/rapidyenc/build +cd rapidyenc && ./crossbuild_rapidyenc_windows-amd64.sh && cd ../ + +export GOOS=windows +export GOARCH=amd64 +export CGO_ENABLED=1 +export CC=x86_64-w64-mingw32-gcc + +go build -o NZBreX_ry.exe -tags "windows rapidyenc" . +exit $? From e0b7ced753ebf54c736c27bd7f94790a280b9cd5 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 12:48:38 +0000 Subject: [PATCH 058/152] Update codeql.yml --- .github/workflows/codeql.yml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7bee3a9..80aa423 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -38,21 +38,14 @@ jobs: build-mode: autobuild runner: ubuntu-22.04 runner_name: "GH-ubuntu-22.04" - ram: 12 + ram: 16 cache: true - language: go build-mode: autobuild runner: ubuntu-24.04 runner_name: "GH-ubuntu-24.04" - ram: 12 - cache: true - - - language: go - build-mode: autobuild - runner: ubuntu-latest - runner_name: "GH-ubuntu-latest" - ram: 12 + ram: 16 cache: true # C/C++ analysis for rapidyenc @@ -60,7 +53,7 @@ jobs: build-mode: manual runner: ubuntu-latest runner_name: "GH-ubuntu-latest" - ram: 12 + ram: 16 cache: true # Actions analysis @@ -68,7 +61,7 @@ jobs: build-mode: none runner: ubuntu-latest runner_name: "GH-ubuntu-latest" - ram: 12 + ram: 16 cache: true @@ -113,7 +106,7 @@ jobs: with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} - queries: security-extended,security-and-quality + queries: ${{ matrix.language == 'c-cpp' && 'security-extended' || 'security-extended,security-and-quality' }} ram: ${{ matrix.ram }} - name: Autobuild From 8ad2d83693c93f9b0dc3214a0bece12cda1337b6 Mon Sep 17 00:00:00 2001 From: go-while <137838162+go-while@users.noreply.github.com> Date: Tue, 14 Oct 2025 13:35:14 +0000 Subject: [PATCH 059/152] Update ProxyHelper.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- ProxyHelper.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ProxyHelper.go b/ProxyHelper.go index fd52707..550f972 100644 --- a/ProxyHelper.go +++ b/ProxyHelper.go @@ -314,7 +314,11 @@ func (ps *ProxySession) Close() { ps.Username = "" // Clear username to avoid dangling pointer ps.Password = "" // Clear username to avoid dangling pointer ps.ExpireAt = 0 // Clear expiration time - log.Printf("Closed session for user '%s'", ps.Username) + username := ps.Username // Store username for logging + ps.Username = "" // Clear username to avoid dangling pointer + ps.Password = "" // Clear username to avoid dangling pointer + ps.ExpireAt = 0 // Clear expiration time + log.Printf("Closed session for user '%s'", username) } func (ps *ProxySession) IsExpired() (isExpired bool) { From 2ec6267f1826d75109c198b582e31e723ab990e1 Mon Sep 17 00:00:00 2001 From: go-while <137838162+go-while@users.noreply.github.com> Date: Tue, 14 Oct 2025 13:35:50 +0000 Subject: [PATCH 060/152] Update Server.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- Server.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Server.go b/Server.go index 29d8605..badbbf1 100644 --- a/Server.go +++ b/Server.go @@ -42,7 +42,9 @@ func handleConnection(conn net.Conn) { CountConns[s.Username]-- log.Printf("Decremented connection count for user '%s'. Active connections: %d", currentUser, CountConns[currentUser]) } else { - log.Printf("Connection count for user '%s' was already 0 or less, not decrementing. This might indicate an issue.", currentUser) + log.Printf("Decremented connection count for user '%s'. Active connections: %d", s.Username, CountConns[s.Username]) + } else { + log.Printf("Connection count for user '%s' was already 0 or less, not decrementing. This might indicate an issue.", s.Username) } // ProxySessions is used to track this specific session, remove it here. delete(ProxySessions, s.Username) From 674a7e8b965bc2c9a0ae550091cfe71398c9e016 Mon Sep 17 00:00:00 2001 From: go-while <137838162+go-while@users.noreply.github.com> Date: Tue, 14 Oct 2025 13:36:21 +0000 Subject: [PATCH 061/152] Update Server.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- Server.go | 1 - 1 file changed, 1 deletion(-) diff --git a/Server.go b/Server.go index badbbf1..be493d4 100644 --- a/Server.go +++ b/Server.go @@ -66,7 +66,6 @@ func handleConnection(conn net.Conn) { ps.Username = "" // Clear username to avoid dangling pointer ps.ConnectedAt = time.Time{} // Clear timestamp to avoid dangling pointer ps.LastCmd = time.Time{} // Clear last command time to avoid dangling pointer - ps = nil // Clear session to avoid dangling pointer }(ps) // Send initial welcome message (RFC 3977: 200 or 201) From 0654c56914aaa0fd40a12da356dfa1d6c12604a2 Mon Sep 17 00:00:00 2001 From: go-while <137838162+go-while@users.noreply.github.com> Date: Tue, 14 Oct 2025 13:37:58 +0000 Subject: [PATCH 062/152] Potential fix for code scanning alert no. 71: Clear-text logging of sensitive information Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- Flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Flags.go b/Flags.go index cf76af2..dc25e99 100644 --- a/Flags.go +++ b/Flags.go @@ -179,7 +179,7 @@ func ParseFlags() { if err := addUserToProxyPasswdFile(userData, newPasswdFile); err != nil { log.Fatalf("Failed to add user to %s: %v", newPasswdFile, err) } - log.Printf("User '%s':\n %#v \n added to new passwd file: '%s'. Next reload in <60s", username, userData, newPasswdFile) + // Redact password before logging os.Exit(0) } // test rapidyenc decoder From 6a36f32ea99d9bd4bc216b78b2447fc6311c5198 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 16:02:21 +0000 Subject: [PATCH 063/152] try patch --- Cache.go | 108 ++++++++++++++++++++++++++++++++-------------------- ConnPool.go | 33 +++++++--------- NetConn.go | 10 ++++- Routines.go | 30 +++++++++++++-- Workers.go | 57 +-------------------------- 5 files changed, 117 insertions(+), 121 deletions(-) diff --git a/Cache.go b/Cache.go index 6df7892..a6000e2 100644 --- a/Cache.go +++ b/Cache.go @@ -3,6 +3,7 @@ package main import ( "bufio" "fmt" + "log" "os" "path/filepath" "strings" @@ -156,19 +157,25 @@ func (c *Cache) GoCacheWriter(cid int) { } n := c.CacheWriter(item) wrote_bytes += uint64(n) + // Memory slot already released by GoDownsRoutine before Add2Cache } } // end func c.GoCacheWriter func (c *Cache) GoYencWriter(cid int) { var wrote_bytes uint64 + dlog(always, "GoYencWriter %d started", cid) for { + dlog(cfg.opt.DebugCache, "GoYencWriter %d waiting for item...", cid) yitem := <-c.yenc_writer_chan if yitem == nil { + log.Printf("YencWriter %d received nil item. closing. wrote total %d bytes", cid, wrote_bytes) c.yenc_writer_chan <- nil return } + dlog(always, "GoYencWriter %d processing seg.Id='%s'", cid, yitem.item.segment.Id) n := c.YencWriter(yitem) wrote_bytes += uint64(n) + dlog(always, "GoYencWriter %d finished seg.Id='%s' wrote=%d bytes", cid, yitem.item.segment.Id, n) } } // end func c.GoCacheWriter @@ -295,20 +302,31 @@ func (c *Cache) WriteYenc(item *segmentChanItem, yPart *yenc.Part) { dlog(always, "ERROR WriteYenc: empty Body seg.Id='%s'", item.segment.Id) return } - GCounter.Incr("yencQueueCnt") - GCounter.Incr("TOTAL_yencQueueCnt") + + // Set flag BEFORE incrementing counter to avoid race item.mux.Lock() item.flaginYenc = true item.mux.Unlock() - c.yenc_writer_chan <- ¥c_item{ - item: item, - yPart: yPart, + +wait: + for { + select { + case c.yenc_writer_chan <- ¥c_item{item: item, yPart: yPart}: // enqueue + // Increment counter AFTER successful enqueue + GCounter.Incr("yencQueueCnt") + GCounter.Incr("TOTAL_yencQueueCnt") + break wait + default: + // chan full + log.Printf("WriteYenc waiting to enqueue seg.Id='%s' yenc_queue_cnt=%d channel=%d/%d", item.segment.Id, GCounter.GetValue("yencQueueCnt"), len(c.yenc_writer_chan), cap(c.yenc_writer_chan)) + time.Sleep(time.Second) + } } } // emd func WriteYenc func (c *Cache) YencWriter(yitem *yenc_item) (wrote_bytes int) { defer GCounter.Decr("yencQueueCnt") - + dlog(always, "YencWriter processing seg.Id='%s'", yitem.item.segment.Id) yitem.item.mux.Lock() if yitem.item.hashedId == "" { yitem.item.hashedId = SHA256str("<" + yitem.item.segment.Id + ">") @@ -318,53 +336,59 @@ func (c *Cache) YencWriter(yitem *yenc_item) (wrote_bytes int) { _, _, yencdir, fp, fp_tmp := c.GetYenc(yitem.item) if FileExists(fp) { - c.resetYencFlagsOnErr(yitem.item) + dlog(c.debug, "YencWriter file already exists, skipping: '%s'", fp) + yitem.item.mux.Lock() + yitem.item.flaginYenc = false + yitem.item.flagisYenc = true // File exists, so yenc IS done + yitem.item.mux.Unlock() return 0 } if !Mkdir(yencdir) { + dlog(always, "ERROR YencWriter Mkdir failed dir='%s'", yencdir) c.resetYencFlagsOnErr(yitem.item) return 0 } - dlog(c.debug, "Writing yenc part: '%s'", fp_tmp) - if file, err := os.OpenFile(fp_tmp, os.O_CREATE|os.O_WRONLY, 0644); err == nil { - defer func() { - if cerr := file.Close(); cerr != nil { - dlog(always, "ERROR YencWriter file.Close err='%v'", cerr) - } - }() - datawriter := bufio.NewWriterSize(file, DefaultYencWriteBuffer) - if n, err := datawriter.Write(yitem.yPart.Body); err != nil { - dlog(always, "ERROR YencWriter datawriter.Write err='%v'", err) - c.resetYencFlagsOnErr(yitem.item) - return 0 - } else { - wrote_bytes += n - } + dlog(always, "DEBUG: Writing yenc part: '%s'", fp_tmp) + file, err := os.OpenFile(fp_tmp, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + dlog(always, "ERROR YencWriter OpenFile failed fp_tmp='%s' err='%v'", fp_tmp, err) + c.resetYencFlagsOnErr(yitem.item) + return 0 + } - if err := datawriter.Flush(); err != nil { - dlog(always, "ERROR YencWriter datawriter.Flush err='%v'", err) - c.resetYencFlagsOnErr(yitem.item) - return 0 - } + datawriter := bufio.NewWriterSize(file, DefaultYencWriteBuffer) + if n, err := datawriter.Write(yitem.yPart.Body); err != nil { + dlog(always, "ERROR YencWriter datawriter.Write err='%v'", err) + c.resetYencFlagsOnErr(yitem.item) + return 0 + } else { + wrote_bytes += n + } - if err := os.Rename(fp_tmp, fp); err != nil { - dlog(always, "ERROR YencWriter move .tmp failed err='%v'", err) - c.resetYencFlagsOnErr(yitem.item) - return 0 - } + if err := datawriter.Flush(); err != nil { + dlog(always, "ERROR YencWriter datawriter.Flush err='%v'", err) + c.resetYencFlagsOnErr(yitem.item) + return 0 + } - yitem.item.mux.Lock() - yitem.item.flaginYenc = false - yitem.item.flagisYenc = true - /* // watch out for broken wings #99ffff! - if yitem.item.flaginUP { - doMemReturn = false - } - */ - yitem.item.mux.Unlock() - } // end OpenFile + if err := os.Rename(fp_tmp, fp); err != nil { + dlog(always, "ERROR YencWriter move .tmp failed err='%v'", err) + c.resetYencFlagsOnErr(yitem.item) + return 0 + } + + if err := file.Close(); err != nil { + dlog(always, "ERROR YencWriter file.Close err='%v'", err) + c.resetYencFlagsOnErr(yitem.item) + return 0 + } + + yitem.item.mux.Lock() + yitem.item.flaginYenc = false + yitem.item.flagisYenc = true + yitem.item.mux.Unlock() yitem.yPart.Body = nil yitem.yPart = nil return diff --git a/ConnPool.go b/ConnPool.go index 9162ae5..4c11b8a 100644 --- a/ConnPool.go +++ b/ConnPool.go @@ -266,7 +266,7 @@ func (c *ConnPool) auth(srvtp *textproto.Conn, conn net.Conn, start time.Time) ( conn.Close() } dlog(always, "ERROR AUTH#1 Cmd(AUTHINFO USER ...) '%s' retry in %.0fs err='%v' ", c.provider.Name, DefaultConnectErrSleep.Seconds(), err) - //time.Sleep(DefaultConnectErrSleep) + time.Sleep(DefaultConnectErrSleep) return nil, err } time2 := time.Now() @@ -290,7 +290,7 @@ func (c *ConnPool) auth(srvtp *textproto.Conn, conn net.Conn, start time.Time) ( conn.Close() } dlog(always, "ERROR AUTH#3 Cmd(AUTHINFO PASS ...) '%s' retry in %.0fs err='%v'", c.provider.Name, DefaultConnectErrSleep.Seconds(), err) - //time.Sleep(DefaultConnectErrSleep) + time.Sleep(DefaultConnectErrSleep) return nil, err } time4 := time.Now() @@ -303,7 +303,7 @@ func (c *ConnPool) auth(srvtp *textproto.Conn, conn net.Conn, start time.Time) ( conn.Close() } dlog(always, "ERROR AUTH#4 ReadCodeLine(281) '%s' retry in %.0fs code=%d msg='%s' err='%v'", c.provider.Name, DefaultConnectErrSleep.Seconds(), code, msg, err) - //time.Sleep(DefaultConnectErrSleep) + time.Sleep(DefaultConnectErrSleep) return nil, err } time5 := time.Now() @@ -320,13 +320,12 @@ func (c *ConnPool) NewConn() (connitem *ConnItem, err error) { // another routine was faster... dlog(cfg.opt.DebugConnPool, "ConnPool NewConn: another routine was faster! openConns=%d provider.MaxConns=%d '%s'", c.openConns, c.provider.MaxConns, c.provider.Name) c.mux.Unlock() // mutex c459a - err = fmt.Errorf("retry getConn, not newConn! all conns are already established") - return + return nil, fmt.Errorf("retry getConn, not newConn! all conns are already established") } if c.openConns < c.provider.MaxConns { dlog(cfg.opt.DebugConnPool, "ConnPool NewConn: connect to '%s' openConns=%d provider.MaxConns=%d", c.provider.Name, c.openConns, c.provider.MaxConns) - c.openConns++ openConnsBefore := c.openConns + c.openConns++ c.mux.Unlock() // mutex c459a start := time.Now() // connect to the provider @@ -336,11 +335,9 @@ func (c *ConnPool) NewConn() (connitem *ConnItem, err error) { if err != nil || connitem == nil || connitem.conn == nil { continue // retry connect } - if cfg.opt.DebugConnPool { - c.mux.RLock() // mutex c461 - dlog(always, "ConnPool NewConn: got new connid=%d '%s' openConns after=%d/%d before=%d connectTook=(%d ms) err='%v", connitem.connid, c.provider.Name, c.openConns, c.provider.MaxConns, openConnsBefore, time.Since(start).Milliseconds(), err) - c.mux.RUnlock() // mutex c461 - } + c.mux.RLock() // mutex c461 + dlog(cfg.opt.DebugConnPool, "ConnPool NewConn: got new connid=%d '%s' openConns after=%d/%d before=%d connectTook=(%d ms) err='%v", connitem.connid, c.provider.Name, c.openConns, c.provider.MaxConns, openConnsBefore, time.Since(start).Milliseconds(), err) + c.mux.RUnlock() // mutex c461 // we have a new connection! GCounter.Incr("TOTAL_NewConns") return // established new connection and returns connitem @@ -351,24 +348,22 @@ func (c *ConnPool) NewConn() (connitem *ConnItem, err error) { if err != nil || connitem == nil || connitem.conn == nil { continue // retry connect } - if cfg.opt.DebugConnPool { - c.mux.RLock() // mutex c461 - dlog(always, "ConnPool NewConn: got new connid=%d '%s' openConns after=%d/%d before=%d connectTook=(%d ms) err='%v", connitem.connid, c.provider.Name, c.openConns, c.provider.MaxConns, openConnsBefore, time.Since(start).Milliseconds(), err) - c.mux.RUnlock() // mutex c461 - } + c.mux.RLock() // mutex c461 + dlog(cfg.opt.DebugConnPool, "ConnPool NewConn: got new connid=%d '%s' openConns after=%d/%d before=%d connectTook=(%d ms) err='%v", connitem.connid, c.provider.Name, c.openConns, c.provider.MaxConns, openConnsBefore, time.Since(start).Milliseconds(), err) + c.mux.RUnlock() // mutex c461 // we have a new connection! GCounter.Incr("TOTAL_NewConns") - return // established new connection and returns connitem + return connitem, nil // established new connection and returns connitem } } c.mux.Lock() // mutex c460 c.openConns-- dlog(always, "ERROR in ConnPool NewConn: connect failed '%s' openConns=%d connitem='%v' err='%v'", c.provider.Name, c.openConns, connitem, err) c.mux.Unlock() // mutex c460 - return + return nil, fmt.Errorf("error in ConnPool NewConn: connect failed '%s' err='%v'", c.provider.Name, err) } c.mux.Unlock() // mutex c459a - return + return nil, fmt.Errorf("error in ConnPool NewConn: openConns=%d > provider.MaxConns", c.openConns) } // end func NewConn func (c *ConnPool) GetConn() (connitem *ConnItem, err error) { diff --git a/NetConn.go b/NetConn.go index 3a0c157..e999743 100644 --- a/NetConn.go +++ b/NetConn.go @@ -876,8 +876,14 @@ readlines: dlog(cfg.opt.DebugRapidYenc, "readDotLines: rapidyenc yenc.Part.Validate OK seg.Id='%s' @ '%s' part.Body=%d Number=%d crc32=%x", item.segment.Id, connitem.c.provider.Name, len(part.Body), part.Number, part.Crc32) } // end if cfg.opt.DoubleCheckRapidYencCRC - // Now write to cache - cache.WriteYenc(item, part) + // Now write to cache (only if yencout flag is enabled) + if cfg.opt.YencWrite && cacheON { + cache.WriteYenc(item, part) + } else { + // Free memory if not writing to cache + part.Body = nil + part = nil + } } // end switch yencTest dlog(cfg.opt.DebugWorker, "readDotLines: YencCRC yenctest=%d brokenYenc=%t seg.Id='%s' @ '%s' rxb=%d content=(%d lines) Part.Validate:took=(%d µs) readDotLines:took=(%d µs) startReadSignals:took=(%d µs) cfg.opt.YencWrite=%t err='%v'", cfg.opt.YencTest, brokenYenc, item.segment.Id, connitem.c.provider.Name, rxb, len(content), time.Since(startReadLines).Microseconds(), time.Since(yencstart).Microseconds(), time.Since(startReadSignals).Microseconds(), cfg.opt.YencWrite, err) if isBadCrc || brokenYenc { diff --git a/Routines.go b/Routines.go index a7f4e4b..7e7d2e2 100644 --- a/Routines.go +++ b/Routines.go @@ -151,9 +151,28 @@ func (s *SESSION) GoDownsRoutine(wid int, provider *Provider, item *segmentChanI // check cache before download if cacheON && cache.ReadCache(item) > 0 { - // item has been read from cache - //DecreaseDLQueueCnt() // decrease when read from cache // DISABLED - //memlim.MemReturn(who+":cacheRead", item) + // item has been read from cache - set flags as if downloaded + // Mark all providers in this group as having the article + flagProviderCache: + for pid, prov := range s.providerList { + if prov.Group != provider.Group { + continue flagProviderCache + } + item.mux.Lock() + item.availableOn[pid] = true + delete(item.missingOn, pid) + item.mux.Unlock() + } + + item.mux.Lock() + item.flagisDL = true + item.flaginDL = false + item.flaginDLMEM = false + item.mux.Unlock() + + // Free memory slot immediately since no actual download happens + who := fmt.Sprintf("DR=%d@'%s'#'%s' seg.Id='%s'", wid, provider.Name, provider.Group, item.segment.Id) + memlim.MemReturn(who+":cacheRead", item) return 920, nil } start := time.Now() // start time for this routine @@ -242,6 +261,11 @@ func (s *SESSION) GoDownsRoutine(wid int, provider *Provider, item *segmentChanI // pass item to cache. dlog(cfg.opt.DebugWorker, "DEBUG GoDownsRoutine CMD_ARTICLE: reached cache.Add2Cache seg.Id='%s' @ '%s'#'%s'", item.segment.Id, provider.Name, provider.Group) + + // Release memory BEFORE adding to cache to prevent deadlock + // Cache writer will not release memory since we already did it here + memlim.MemReturn("GoDownsRoutine:beforeCache", item) + cache.Add2Cache(item) // pass to ParkConn diff --git a/Workers.go b/Workers.go index 3d489d4..0dcdc0d 100644 --- a/Workers.go +++ b/Workers.go @@ -270,6 +270,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro case 920: // 920 is a special code for GoDownsRoutine to indicate that the item has been read from cache mode = "cache read" + // Memory was already returned in GoDownsRoutine for cache reads } dlog(cfg.opt.DebugWorker && cfg.opt.BUG, "GoDownsRoutine: %s (wid=%d) seg.Id='%s' @ '%s' took='%v' speedInKBytes=%.2f", mode, wid, item.segment.Id, provider.Name, time.Since(StartDowns), speedInKBytes) @@ -459,7 +460,7 @@ providerDl: dlog(cfg.opt.DebugWorker, " | [DV-pushDL] pushed to dlchan seg.Id='%s' @ #'%s'", item.segment.Id, s.providerList[pid].Group) return // return after 1st push! default: - dlog(cfg.opt.BUG, "DEBUG SPAM pushDL: chan is full @ #'%s'", s.providerList[pid].Group) + dlog(cfg.opt.BUG, "DEBUG SPAM pushDL: chan is full @ #'%s', retry next", s.providerList[pid].Group) // chan is full means we cannot push the item to the download queue to this provider group // either app is blocked or we're just checking faster than we can download at all... //err = fmt.Errorf(" | [DV-pushDL] chans full @ '%s'#'%s'", s.providerList[pid].Name, s.providerList[pid].Group) @@ -594,61 +595,7 @@ func (s *SESSION) GoWorkDivider(waitDivider *sync.WaitGroup, waitDividerDone *sy // strings var logstring, log00, log01, log02, log03, log04, log05, log06, log07, log08, log09, log10, log11, log99 string - /* - go func(s *SESSION) { - if !testing { - return - } - //var funcmux sync.Mutex - //inup, indl, nodl, noup, inretry := uint64(0), uint64(0), uint64(0), uint64(0), uint64(0) - //replychan := make(chan bool, 1) // internal replychan for the worker loop - for { - wrappedItem := <-s.WorkDividerChan - item := wrappedItem.wItem // unwrap the item from the channel - item.mux.RLock() - if item.checkedOn != providersCnt { - //dlog( " | [DV] WorkDividerChan debug#1 received item seg.Id='%s' checkedOn=%d", item.segment.Id, item.checkedOn) - item.mux.RUnlock() - continue // ignore item, will retry next run - } - item.mux.RUnlock() - if wrappedItem.src != "CR" { - dlog( " | [DV] WorkDividerChan debug#2 received item seg.Id='%s' checkedOn=%d src=%s", item.segment.Id, item.checkedOn, wrappedItem.src) - } - switch wrappedItem.src { - - case "DR": - go func(item *segmentChanItem) { - for { - pushedUp, nNoUp, nInRetry := s.pushUP(allowUp, item) - if pushedUp { - dlog( " | [DV] PUSHED to Up seg.Id='%s' pushedUp=%t nNoUp=%d nInRetry=%d", item.segment.Id, pushedUp, nNoUp, nInRetry) - break - } - dlog( " | [DV] WorkDividerChan retrying pushUP seg.Id='%s'", item.segment.Id) - time.Sleep(1000 * time.Millisecond) // wait a bit before retrying - } - }(item) - - case "CR": - go func(item *segmentChanItem) { - for { - pushedDl, nNoDl := s.pushDL(allowDl, item) - if pushedDl { - if cfg.opt.BUG { - dlog( " | [DV] PUSHED to DL seg.Id='%s' pushedDl=%t nNoDl=%d", item.segment.Id, pushedDl, nNoDl) - } - break - } - dlog( " | [DV] WorkDividerChan retrying pushDL seg.Id='%s'", item.segment.Id) - time.Sleep(1000 * time.Millisecond) // wait a bit before retrying - } - }(item) - } // end switch wrappedItem.src - } - }(s) - */ // loops forever over the s.segmentList and checks if there is anything to do for an item var minsleep int64 = 10 // 0.01 second in milliseconds var baseline int64 = 1000 From 8996aab1fcce3b3c9167e4ae86eccdb9f5bbf370 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 16:07:40 +0000 Subject: [PATCH 064/152] Update codeql.yml --- .github/workflows/codeql.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 80aa423..2bfa177 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,22 +1,22 @@ name: "CodeQL Advanced" on: - push: - branches: - - main - - testing - tags: - - testing - release: - types: [published] + #push: + # branches: + # - main + # - testing + # tags: + # - testing + #release: + # types: [published] workflow_dispatch: - repository_dispatch: - pull_request: - branches: - - main - - testing - schedule: - - cron: '30 05 * * 1' + #repository_dispatch: + #pull_request: + # branches: + # - main + # - testing + #schedule: + # - cron: '30 05 * * 1' jobs: analyze: From 15ff454703ba730ea63fe6159a9acd88440dd8d2 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 16:11:11 +0000 Subject: [PATCH 065/152] Update Server.go --- Server.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/Server.go b/Server.go index be493d4..de00a85 100644 --- a/Server.go +++ b/Server.go @@ -41,8 +41,6 @@ func handleConnection(conn net.Conn) { if CountConns[s.Username] > 0 { CountConns[s.Username]-- log.Printf("Decremented connection count for user '%s'. Active connections: %d", currentUser, CountConns[currentUser]) - } else { - log.Printf("Decremented connection count for user '%s'. Active connections: %d", s.Username, CountConns[s.Username]) } else { log.Printf("Connection count for user '%s' was already 0 or less, not decrementing. This might indicate an issue.", s.Username) } From 8031f5fc57e4a0681f24a7e8b9086e3e29697106 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 16:25:16 +0000 Subject: [PATCH 066/152] testing --- Routines.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/Routines.go b/Routines.go index 7e7d2e2..64c85b1 100644 --- a/Routines.go +++ b/Routines.go @@ -153,19 +153,21 @@ func (s *SESSION) GoDownsRoutine(wid int, provider *Provider, item *segmentChanI if cacheON && cache.ReadCache(item) > 0 { // item has been read from cache - set flags as if downloaded // Mark all providers in this group as having the article - flagProviderCache: - for pid, prov := range s.providerList { - if prov.Group != provider.Group { - continue flagProviderCache + /* disabled, this looks wrong + flagProviderCache: + for pid, prov := range s.providerList { + if prov.Group != provider.Group { + continue flagProviderCache + } + item.mux.Lock() + item.availableOn[pid] = true + delete(item.missingOn, pid) + item.mux.Unlock() } - item.mux.Lock() - item.availableOn[pid] = true - delete(item.missingOn, pid) - item.mux.Unlock() - } - + */ item.mux.Lock() - item.flagisDL = true + item.flagCache = true + item.flagisDL = false item.flaginDL = false item.flaginDLMEM = false item.mux.Unlock() From 01286ffb07726f9c430c4a13a5635cfcddf14f81 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 16:55:31 +0000 Subject: [PATCH 067/152] testing --- Routines.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Routines.go b/Routines.go index 64c85b1..79554aa 100644 --- a/Routines.go +++ b/Routines.go @@ -167,14 +167,14 @@ func (s *SESSION) GoDownsRoutine(wid int, provider *Provider, item *segmentChanI */ item.mux.Lock() item.flagCache = true - item.flagisDL = false + item.flagisDL = true item.flaginDL = false item.flaginDLMEM = false item.mux.Unlock() // Free memory slot immediately since no actual download happens - who := fmt.Sprintf("DR=%d@'%s'#'%s' seg.Id='%s'", wid, provider.Name, provider.Group, item.segment.Id) - memlim.MemReturn(who+":cacheRead", item) + //who := fmt.Sprintf("DR=%d@'%s'#'%s' seg.Id='%s'", wid, provider.Name, provider.Group, item.segment.Id) + //memlim.MemReturn(who+":cacheRead", item) return 920, nil } start := time.Now() // start time for this routine @@ -266,7 +266,7 @@ func (s *SESSION) GoDownsRoutine(wid int, provider *Provider, item *segmentChanI // Release memory BEFORE adding to cache to prevent deadlock // Cache writer will not release memory since we already did it here - memlim.MemReturn("GoDownsRoutine:beforeCache", item) + //memlim.MemReturn("GoDownsRoutine:beforeCache", item) cache.Add2Cache(item) // pass to ParkConn @@ -410,9 +410,11 @@ func (s *SESSION) GoReupsRoutine(wid int, provider *Provider, item *segmentChanI // pass case 436: retry = true + err = nil // pass case 437: unwanted = true + err = nil // pass default: dlog(always, "ERROR in GoReupsRoutine: CMD_IHAVE seg.Id='%s' @ '%s'#'%s' code=%d msg='%s' err='%v'", item.segment.Id, provider.Name, provider.Group, code, msg, err) From d203fe0ae587591bfe11663b9ca1afa4b87f5e00 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 16:58:04 +0000 Subject: [PATCH 068/152] testing --- Routines.go | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/Routines.go b/Routines.go index 79554aa..556169a 100644 --- a/Routines.go +++ b/Routines.go @@ -151,30 +151,7 @@ func (s *SESSION) GoDownsRoutine(wid int, provider *Provider, item *segmentChanI // check cache before download if cacheON && cache.ReadCache(item) > 0 { - // item has been read from cache - set flags as if downloaded - // Mark all providers in this group as having the article - /* disabled, this looks wrong - flagProviderCache: - for pid, prov := range s.providerList { - if prov.Group != provider.Group { - continue flagProviderCache - } - item.mux.Lock() - item.availableOn[pid] = true - delete(item.missingOn, pid) - item.mux.Unlock() - } - */ - item.mux.Lock() - item.flagCache = true - item.flagisDL = true - item.flaginDL = false - item.flaginDLMEM = false - item.mux.Unlock() - - // Free memory slot immediately since no actual download happens - //who := fmt.Sprintf("DR=%d@'%s'#'%s' seg.Id='%s'", wid, provider.Name, provider.Group, item.segment.Id) - //memlim.MemReturn(who+":cacheRead", item) + dlog(cfg.opt.DebugDR, "GoDownsRoutine: cache hit seg.Id='%s' @ '%s'#'%s'", item.segment.Id, provider.Name, provider.Group) return 920, nil } start := time.Now() // start time for this routine @@ -264,10 +241,6 @@ func (s *SESSION) GoDownsRoutine(wid int, provider *Provider, item *segmentChanI // pass item to cache. dlog(cfg.opt.DebugWorker, "DEBUG GoDownsRoutine CMD_ARTICLE: reached cache.Add2Cache seg.Id='%s' @ '%s'#'%s'", item.segment.Id, provider.Name, provider.Group) - // Release memory BEFORE adding to cache to prevent deadlock - // Cache writer will not release memory since we already did it here - //memlim.MemReturn("GoDownsRoutine:beforeCache", item) - cache.Add2Cache(item) // pass to ParkConn From 6d96418cfbbb193e955af83f6095f2324f7ca522 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 17:07:33 +0000 Subject: [PATCH 069/152] Update build_and_publish_test.yml --- .github/workflows/build_and_publish_test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index af7b8c0..6b25311 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -10,7 +10,13 @@ on: release: types: [published] workflow_dispatch: + inputs: + version: + description: 'Version to release (e.g., 1.0.0)' + required: true + type: string repository_dispatch: + types: [build] #pull_request: # branches: # - main From d238ed903c306c5996df4b46f4da9c8615e52627 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 17:15:42 +0000 Subject: [PATCH 070/152] Update build_and_publish_test.yml --- .github/workflows/build_and_publish_test.yml | 26 ++++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 6b25311..935d6cd 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -349,15 +349,27 @@ jobs: GITHUB_EVENT_NAME: ${{ github.event_name }} GITHUB_REF_NAME: ${{ github.ref_name }} GITHUB_SHA: ${{ github.sha }} + INPUT_VERSION: ${{ inputs.version }} run: | echo "COMPILER=GHR" >> "$GITHUB_ENV" # Changed from SHR to GHR (GitHub Runners) - # If event is tag or release, use the tag; otherwise, use 0.0.0-SNAPSHOT - SAFE_VERSION="$(echo "${GITHUB_REF_NAME}" | sed 's|/|-|g')" - if [[ "${GITHUB_REF_TYPE}" == "tag" || "${GITHUB_EVENT_NAME}" == "release" ]]; then + + # Determine version based on event type and inputs + if [[ -n "$INPUT_VERSION" ]]; then + # Use version from workflow_dispatch or repository_dispatch input + VERSION="$INPUT_VERSION" + echo "SHOULD_RELEASE=true" >> $GITHUB_ENV + elif [[ "${GITHUB_REF_TYPE}" == "tag" || "${GITHUB_EVENT_NAME}" == "release" ]]; then + # Use tag name for tagged releases + SAFE_VERSION="$(echo "${GITHUB_REF_NAME}" | sed 's|/|-|g')" VERSION="${SAFE_VERSION}" + echo "SHOULD_RELEASE=true" >> $GITHUB_ENV else + # Default to branch name with SNAPSHOT suffix + SAFE_VERSION="$(echo "${GITHUB_REF_NAME}" | sed 's|/|-|g')" VERSION="${SAFE_VERSION}-SNAPSHOT" + echo "SHOULD_RELEASE=false" >> $GITHUB_ENV fi + # Strip leading 'v' if present VERSION="${VERSION#v}" SHA7="${GITHUB_SHA::7}" @@ -537,13 +549,17 @@ jobs: if-no-files-found: warn - name: "Upload Release Assets to github" - if: github.event_name == 'release' + if: env.SHOULD_RELEASE == 'true' uses: softprops/action-gh-release@da05d552573ad5aba039eaac05058a918a7bf631 # v2.2.2 with: + tag_name: v${{ env.VERSION }} + name: Release ${{ env.VERSION }} + draft: false + prerelease: ${{ contains(env.VERSION, 'test') || contains(env.VERSION, 'beta') || contains(env.VERSION, 'alpha') }} files: | NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.deb NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.tgz - NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.xz + NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.tar.xz NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.zip NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha256sum NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha512sum From 06fe0c0291d02598c17395054680e936a25a38fa Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 17:17:23 +0000 Subject: [PATCH 071/152] Create grep.sh --- grep.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100755 grep.sh diff --git a/grep.sh b/grep.sh new file mode 100755 index 0000000..894d4bf --- /dev/null +++ b/grep.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# tool to search for strings in GO source files +find . -iname "*.go" -exec grep -in "$1" {} + From 7e500988268509da79a1b25c30881cc95f020260 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 17:49:42 +0000 Subject: [PATCH 072/152] 0.0.6-test-p2 --- Workers.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Workers.go b/Workers.go index 0dcdc0d..01af320 100644 --- a/Workers.go +++ b/Workers.go @@ -197,7 +197,11 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro code, err := s.GoCheckRoutine(wid, provider, item, sharedCC) item.PrintItemFlags(cfg.opt.DebugFlags, true, fmt.Sprintf("post-GoCheckRoutine: code=%d", code)) if err != nil { // re-queue? - dlog(always, "ERROR in GoCheckRoutine err='%v'", err) + dlog(always, "ERROR in GoCheckRoutine err='%v' flag retry item", err) + item.mux.Lock() + item.retryIn = time.Now().Unix() + 9 // retry in 9s + item.retryOn[provider.id] = true + item.mux.Unlock() } case true: log.Fatal("you should not be here! Quitting...") // FIXME TODO: remove this fatal error @@ -253,8 +257,12 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro if err != nil || (code != 220 && code != 920) { if code != 430 { // 430 is a normal error code for GoDownsRoutine, so we don't log it as an error - errStr = fmt.Sprintf("ERROR in GoDownsRoutine code='%d' err='%v'", code, err) + errStr = fmt.Sprintf("ERROR in GoDownsRoutine code='%d' err='%v' flag retry item", code, err) dlog(always, "%s", errStr) + item.mux.Lock() + item.retryIn = time.Now().Unix() + 9 // retry in 9s + item.retryOn[provider.id] = true + item.mux.Unlock() } memlim.MemReturn("MemRetOnERR: "+errStr, item) // memfree GoDownsRoutine on error continue forGoDownsRoutine @@ -317,9 +325,13 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro DecreaseUPQueueCnt() if err != nil { - errStr := fmt.Sprintf("ERROR in GoReupsRoutine code='%d' err='%v'", code, err) + errStr := fmt.Sprintf("ERROR in GoReupsRoutine code='%d' err='%v' flag retry item", code, err) dlog(always, "%s", errStr) memlim.MemReturn("MemRetOnERR: "+errStr, item) // memfree GoReupsRoutine on error + item.mux.Lock() + item.retryIn = time.Now().Unix() + 9 // retry in 9s + item.retryOn[provider.id] = true + item.mux.Unlock() continue forGoReupsRoutine } speedInKBytes := (float64(item.size) / 1024) / float64(time.Since(StartReUps).Seconds()) From 19fb9629572ba2cd431c421466231ad9e196bc18 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 18:39:48 +0000 Subject: [PATCH 073/152] Update Session.go --- Session.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Session.go b/Session.go index b6af11b..77ad131 100644 --- a/Session.go +++ b/Session.go @@ -131,7 +131,7 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s defer waitSession.Done() } - if !proxy { + if s != nil && !s.proxy { // not booting proxy, no session supplied, create a new session // checks for the correct inputs if nzbfilepath == "" && s == nil { From 9266e6d64a489e1598dd39506c22587529ac968f Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 18:46:18 +0000 Subject: [PATCH 074/152] Update build_and_publish_test.yml --- .github/workflows/build_and_publish_test.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 935d6cd..64b9911 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -245,12 +245,6 @@ jobs: # - name: Restore Go modules cache # # your cache restore step here - - name: "Set up Go" - uses: actions/setup-go@v5 - with: - go-version: '1.24.3' - cache: true - #- name: Cache Go modules # uses: actions/cache@v4 # with: From 7c73aabe0a37cb9664bb56a92e2918aa223c6c36 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 18:46:44 +0000 Subject: [PATCH 075/152] Update build_and_publish_test.yml --- .github/workflows/build_and_publish_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 64b9911..a79ec4f 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -128,7 +128,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v5 with: - go-version: '1.25.1' + go-version: '1.25.2' cache: false # Install system dependencies based on runner OS From fda537106726ed9e433cc0d1c46f507870781356 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 18:48:31 +0000 Subject: [PATCH 076/152] go 1.25.3 --- .github/workflows/build_and_publish_test.yml | 2 +- go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index a79ec4f..0544a09 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -128,7 +128,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v5 with: - go-version: '1.25.2' + go-version: '1.25.3' cache: false # Install system dependencies based on runner OS diff --git a/go.mod b/go.mod index 17cf8b6..bf76bc1 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/go-while/NZBreX -go 1.25.1 +go 1.25.3 require ( github.com/Tensai75/nzbparser v0.1.0 From 6efaa7bd0a63ede6a0bacb6862731d35acd51f30 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 21:01:51 +0000 Subject: [PATCH 077/152] AI fucked up --- Session.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Session.go b/Session.go index 77ad131..8ba935f 100644 --- a/Session.go +++ b/Session.go @@ -131,7 +131,7 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s defer waitSession.Done() } - if s != nil && !s.proxy { + if proxy { // there was a purpose to use NOT the session variable here... // not booting proxy, no session supplied, create a new session // checks for the correct inputs if nzbfilepath == "" && s == nil { From 49b3fc5a19a215055ed702c0ddb33590cc1ea866 Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 21:05:50 +0000 Subject: [PATCH 078/152] I too, !proxy --- Session.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Session.go b/Session.go index 8ba935f..b6af11b 100644 --- a/Session.go +++ b/Session.go @@ -131,7 +131,7 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s defer waitSession.Done() } - if proxy { // there was a purpose to use NOT the session variable here... + if !proxy { // not booting proxy, no session supplied, create a new session // checks for the correct inputs if nzbfilepath == "" && s == nil { From 0001efe8ed4691ba8cdb5d6d63bb110a8299598f Mon Sep 17 00:00:00 2001 From: go-while Date: Tue, 14 Oct 2025 21:06:48 +0000 Subject: [PATCH 079/152] comment for AI --- Session.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Session.go b/Session.go index b6af11b..8464550 100644 --- a/Session.go +++ b/Session.go @@ -131,7 +131,7 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s defer waitSession.Done() } - if !proxy { + if !proxy { // the normal case, we are processing an nzb file. don't use the session var here! // not booting proxy, no session supplied, create a new session // checks for the correct inputs if nzbfilepath == "" && s == nil { From 1e0795944966ee7b84ea4402668978812830da57 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 06:01:35 +0000 Subject: [PATCH 080/152] item.FlagError(provider.id) --- Routines.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Routines.go b/Routines.go index 556169a..248e1b2 100644 --- a/Routines.go +++ b/Routines.go @@ -49,6 +49,7 @@ func (s *SESSION) GoCheckRoutine(wid int, provider *Provider, item *segmentChanI code, msg, err := CMD_STAT(connitem, item) if code == 0 && err != nil { // connection problem, closed? + item.FlagError(provider.id) provider.ConnPool.CloseConn(connitem, sharedCC) // close conn on error dlog(always, "WARN checking seg.Id='%s' failed @ '%s' code=%d msg='%s' err='%v'", item.segment.Id, provider.Name, code, msg, err) return code, err @@ -173,10 +174,10 @@ func (s *SESSION) GoDownsRoutine(wid int, provider *Provider, item *segmentChanI startArticle := time.Now() code, msg, rxb, err := CMD(connitem, item, cmdARTICLE) - if err != nil { dlog(always, "ERROR in GoDownsRoutine: CMD_ARTICLE seg.Id='%s' @ '%s'#'%s' err='%v'", item.segment.Id, provider.Name, provider.Group, err) // handle connection problem / closed connection + item.FlagError(provider.id) provider.ConnPool.CloseConn(connitem, sharedCC) // close conn on error return 0, fmt.Errorf("error in GoDownsRoutine: CMD_ARTICLE seg.Id='%s' @ '%s'#'%s' err='%v'", item.segment.Id, provider.Name, provider.Group, err) } @@ -366,6 +367,12 @@ func (s *SESSION) GoReupsRoutine(wid int, provider *Provider, item *segmentChanI switch cmd { case 1: code, msg, txb, err = CMD_POST(connitem, item) + if code == 0 && err != nil { + // connection problem, closed? + item.FlagError(provider.id) + provider.ConnPool.CloseConn(connitem, sharedCC) + return 0, fmt.Errorf("error in GoReupsRoutine: CMD_POST seg.Id='%s' @ '%s'#'%s' err='%v'", item.segment.Id, provider.Name, provider.Group, err) + } switch code { case 240: uploaded = true @@ -377,6 +384,12 @@ func (s *SESSION) GoReupsRoutine(wid int, provider *Provider, item *segmentChanI case 2: code, msg, txb, err = CMD_IHAVE(connitem, item) + if code == 0 && err != nil { + // connection problem, closed? + item.FlagError(provider.id) + provider.ConnPool.CloseConn(connitem, sharedCC) // close conn on error + return 0, fmt.Errorf("error in GoReupsRoutine: CMD_IHAVE seg.Id='%s' @ '%s'#'%s' err='%v'", item.segment.Id, provider.Name, provider.Group, err) + } switch code { case 235: uploaded = true From 406238b024b1d6db1ff52dae88d594984bcef43f Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 06:07:02 +0000 Subject: [PATCH 081/152] item.FlagError --- Workers.go | 45 +++++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/Workers.go b/Workers.go index 01af320..134e247 100644 --- a/Workers.go +++ b/Workers.go @@ -198,10 +198,9 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro item.PrintItemFlags(cfg.opt.DebugFlags, true, fmt.Sprintf("post-GoCheckRoutine: code=%d", code)) if err != nil { // re-queue? dlog(always, "ERROR in GoCheckRoutine err='%v' flag retry item", err) - item.mux.Lock() - item.retryIn = time.Now().Unix() + 9 // retry in 9s - item.retryOn[provider.id] = true - item.mux.Unlock() + go func(item *segmentChanItem) { + segCC <- item // re-queue the item for checking + }(item) } case true: log.Fatal("you should not be here! Quitting...") // FIXME TODO: remove this fatal error @@ -253,16 +252,14 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro StartDowns := time.Now() code, err := s.GoDownsRoutine(wid, provider, item, sharedCC) item.PrintItemFlags(cfg.opt.DebugFlags, true, fmt.Sprintf("post-GoDownsRoutine: code=%d", code)) + DecreaseDLQueueCnt() if err != nil || (code != 220 && code != 920) { if code != 430 { // 430 is a normal error code for GoDownsRoutine, so we don't log it as an error - errStr = fmt.Sprintf("ERROR in GoDownsRoutine code='%d' err='%v' flag retry item", code, err) + errStr = fmt.Sprintf("ERROR in GoDownsRoutine code='%d' err='%v'. no retry", code, err) dlog(always, "%s", errStr) - item.mux.Lock() - item.retryIn = time.Now().Unix() + 9 // retry in 9s - item.retryOn[provider.id] = true - item.mux.Unlock() + item.FlagError(provider.id) // mark item with error for this provider } memlim.MemReturn("MemRetOnERR: "+errStr, item) // memfree GoDownsRoutine on error continue forGoDownsRoutine @@ -325,13 +322,10 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro DecreaseUPQueueCnt() if err != nil { - errStr := fmt.Sprintf("ERROR in GoReupsRoutine code='%d' err='%v' flag retry item", code, err) + errStr := fmt.Sprintf("ERROR in GoReupsRoutine code='%d' err='%v' no retry", code, err) dlog(always, "%s", errStr) memlim.MemReturn("MemRetOnERR: "+errStr, item) // memfree GoReupsRoutine on error - item.mux.Lock() - item.retryIn = time.Now().Unix() + 9 // retry in 9s - item.retryOn[provider.id] = true - item.mux.Unlock() + item.FlagError(provider.id) // mark item with error for this provider continue forGoReupsRoutine } speedInKBytes := (float64(item.size) / 1024) / float64(time.Since(StartReUps).Seconds()) @@ -386,6 +380,25 @@ func matchThisDL(item *segmentChanItem) bool { return (!item.flaginDL && !item.flagisDL && !item.flaginUP && !item.flagisUP && !item.flaginDLMEM) } // end func matchThisDL +func (item *segmentChanItem) FlagError(providerId int) { + item.mux.Lock() + defer item.mux.Unlock() + item.errorOn[providerId] = true +} + +func (item *segmentChanItem) FlagDLFailed(providerList []*Provider, providerGroup string) { + item.mux.Lock() + defer item.mux.Unlock() + for id, prov := range providerList { + if prov.Group != providerGroup { + continue + } + item.ignoreDlOn[id] = true + item.missingOn[id] = true + item.errorOn[id] = true + } +} + // matchThisUP checks if the item is a candidate for upload // it returns true if the item is a candidate for upload func matchThisUP(item *segmentChanItem) bool { @@ -543,10 +556,10 @@ providerUp: if item.retryOn[pid] { if item.retryIn > time.Now().Unix() { inretry++ - dlog(always, " | [DV-pushUP] (noup) retryOn seg.Id='%s' @ #'%s' retryIn=%d > now=%d", item.segment.Id, s.providerList[pid].Group, item.retryIn, time.Now().Unix()) + dlog(always, " | [DV-pushUP] (noup) retryOn seg.Id='%s' @ #'%s' retryIn=%d > now=%d, continue", item.segment.Id, s.providerList[pid].Group, item.retryIn, time.Now().Unix()) continue providerUp } else { - dlog(cfg.opt.DebugWorker, " | [DV-pushUP] (noup) retryOn seg.Id='%s' @ #'%s' retryIn=%d <= now=%d", item.segment.Id, s.providerList[pid].Group, item.retryIn, time.Now().Unix()) + dlog(cfg.opt.DebugWorker, " | [DV-pushUP] (noup) retryOn seg.Id='%s' @ #'%s' retryIn=%d <= now=%d, pass", item.segment.Id, s.providerList[pid].Group, item.retryIn, time.Now().Unix()) delete(item.retryOn, pid) // remove retryOn flag for this provider // pass } From bd22e72dcbee2849205b7fb382dc15c4b85ef473 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 06:07:12 +0000 Subject: [PATCH 082/152] comment --- ConnPool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ConnPool.go b/ConnPool.go index 4c11b8a..43c0a97 100644 --- a/ConnPool.go +++ b/ConnPool.go @@ -726,7 +726,7 @@ func isNetworkUnreachable(err error) bool { } // end func isNetworkUnreachable (written by AI! GPT-4o) // GetNewSharedConnChannel creates a new shared connection channel. -// The sharedCC channel is used to share connections between (anonymous) goroutines which will work on the same item. +// The sharedCC channel is used to share connections between (anonymous) goroutines // which will work on the same item. func GetNewSharedConnChannel(wid int, provider *Provider) (sharedCC chan *ConnItem, err error) { sharedCC = make(chan *ConnItem, 1) // buffered channel with size 1 From c0377813335b7ebaac50d3655c3de013dc4ddb63 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 11:10:43 +0000 Subject: [PATCH 083/152] Update build_and_publish_test.yml --- .github/workflows/build_and_publish_test.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 0544a09..971047e 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -421,18 +421,18 @@ jobs: LICENSE README.md rapidyenc/LICENSE rapidyenc/rapidyenc/README.md rapidyenc/rapidyenc/crcutil-1.0/LICENSE rapidyenc/rapidyenc/build/rapidyenc_* rapidyenc/rapidyenc/build/librapidyenc.* # # .tgz (tar + gzip) - pwd && ls -lha && echo ".tgz Packing builds/${{ matrix.name }}/usr/bin/$binary_name" - tar -czf "NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.tgz" \ - -C builds/${{ matrix.name }}/usr/bin $binary_name $binary_name.sha256sum $binary_name.sha512sum \ - -C ${{ github.workspace }} cleanHeaders.txt provider.sample.json provider.ygg.json \ - -C ${{ github.workspace }} LICENSE README.md rapidyenc/LICENSE rapidyenc/rapidyenc/README.md rapidyenc/rapidyenc/crcutil-1.0/LICENSE rapidyenc/rapidyenc/build/rapidyenc_* rapidyenc/rapidyenc/build/librapidyenc.* + #pwd && ls -lha && echo ".tgz Packing builds/${{ matrix.name }}/usr/bin/$binary_name" + #tar -czf "NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.tgz" \ + # -C builds/${{ matrix.name }}/usr/bin $binary_name $binary_name.sha256sum $binary_name.sha512sum \ + # -C ${{ github.workspace }} cleanHeaders.txt provider.sample.json provider.ygg.json \ + # -C ${{ github.workspace }} LICENSE README.md rapidyenc/LICENSE rapidyenc/rapidyenc/README.md rapidyenc/rapidyenc/crcutil-1.0/LICENSE rapidyenc/rapidyenc/build/rapidyenc_* rapidyenc/rapidyenc/build/librapidyenc.* # # .xz (tar + xz) - pwd && ls -lha && echo ".xz Packing builds/${{ matrix.name }}/usr/bin/$binary_name" - tar -cJf "NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.tar.xz" \ - -C builds/${{ matrix.name }}/usr/bin $binary_name $binary_name.sha256sum $binary_name.sha512sum \ - -C ${{ github.workspace }} cleanHeaders.txt provider.sample.json provider.ygg.json \ - -C ${{ github.workspace }} LICENSE README.md rapidyenc/LICENSE rapidyenc/rapidyenc/README.md rapidyenc/rapidyenc/crcutil-1.0/LICENSE rapidyenc/rapidyenc/build/rapidyenc_* rapidyenc/rapidyenc/build/librapidyenc.* + #pwd && ls -lha && echo ".xz Packing builds/${{ matrix.name }}/usr/bin/$binary_name" + #tar -cJf "NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.tar.xz" \ + # -C builds/${{ matrix.name }}/usr/bin $binary_name $binary_name.sha256sum $binary_name.sha512sum \ + # -C ${{ github.workspace }} cleanHeaders.txt provider.sample.json provider.ygg.json \ + # -C ${{ github.workspace }} LICENSE README.md rapidyenc/LICENSE rapidyenc/rapidyenc/README.md rapidyenc/rapidyenc/crcutil-1.0/LICENSE rapidyenc/rapidyenc/build/rapidyenc_* rapidyenc/rapidyenc/build/librapidyenc.* # # done packing # From 0f6547d0413aaa29f20e032c10c5f51b125ef076 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 11:58:51 +0000 Subject: [PATCH 084/152] debugs --- Cache.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cache.go b/Cache.go index a6000e2..a9a65e6 100644 --- a/Cache.go +++ b/Cache.go @@ -168,14 +168,14 @@ func (c *Cache) GoYencWriter(cid int) { dlog(cfg.opt.DebugCache, "GoYencWriter %d waiting for item...", cid) yitem := <-c.yenc_writer_chan if yitem == nil { - log.Printf("YencWriter %d received nil item. closing. wrote total %d bytes", cid, wrote_bytes) + dlog(always, "YencWriter %d received nil item. closing. wrote total %d bytes", cid, wrote_bytes) c.yenc_writer_chan <- nil return } - dlog(always, "GoYencWriter %d processing seg.Id='%s'", cid, yitem.item.segment.Id) + dlog(c.debug, "GoYencWriter %d processing seg.Id='%s'", cid, yitem.item.segment.Id) n := c.YencWriter(yitem) wrote_bytes += uint64(n) - dlog(always, "GoYencWriter %d finished seg.Id='%s' wrote=%d bytes", cid, yitem.item.segment.Id, n) + dlog(c.debug, "GoYencWriter %d finished seg.Id='%s' wrote=%d bytes", cid, yitem.item.segment.Id, n) } } // end func c.GoCacheWriter From c0bc43fd314a1ec4fa97441f913c61f29cc3a71a Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 12:08:56 +0000 Subject: [PATCH 085/152] proxy: replace tpReader/tpWriter with CliTp --- Proxy.go | 96 +++++++++++++++++++++++++------------------------- ProxyHelper.go | 42 +++++++++++----------- Server.go | 91 +++++++++++++++++++++-------------------------- 3 files changed, 109 insertions(+), 120 deletions(-) diff --git a/Proxy.go b/Proxy.go index aa17c84..60b5e82 100644 --- a/Proxy.go +++ b/Proxy.go @@ -64,28 +64,28 @@ type UserData struct { // ProxySession represents an active user session (currently a placeholder, expand as needed) type ProxySession struct { - id uint64 // Unique session ID, can be used for tracking - mux sync.RWMutex // Mutex for session data access - cmdmux sync.RWMutex // Mutex for command handling - Authed bool // Indicates if the user is authenticated - Username string // Username of the authenticated user - Password string // password for the session, can be used for re-authentication - ExpireAt int64 // session expiration time (Unix timestamp) - Conn net.Conn // The user's network connection - Writer *bufio.Writer // bufio writer for the client connection to send articles, headers, bodies, list, xover, xhdr, ... (big data) - tpReader *textproto.Reader // textproto reader for easier command handling - tpWriter *textproto.Writer // textproto writer for easier command handling - CliTp *textproto.Conn // textproto connection for easier command handling - tmpRXBytes uint64 // proxy has RECEIVED this amount of bytes FROM CLIENT via POST/IHAVE/TAKETHIS in last 60 seconds - RXBytes uint64 // proxy has RECEIVED this amount of bytes FROM CLIENT via POST/IHAVE/TAKETHIS in total this session - tmpTXBytes uint64 // proxy has SENT this amount of bytes TO CLIENT via ARTICLE/HEAD/BODY in last 60 seconds - TXBytes uint64 // proxy has SENT this amount of bytes TO CLIENT via ARTICLE/HEAD/BODY in total this session - ConnectedAt time.Time // Timestamp when the session was created - LastCmd time.Time // Timestamp of the last command received - Group string // current group the user is in (used by GROUP command) - MsgNum int64 // current message number in the group (used by STAT, ARTICLE, etc. commands) - Cron time.Time // last run of periodic tasks, e.g., checking session expiration - selectedProvider *Provider // The provider selected for this session, used for routing commands + id uint64 // Unique session ID, can be used for tracking + mux sync.RWMutex // Mutex for session data access + cmdmux sync.RWMutex // Mutex for command handling + Authed bool // Indicates if the user is authenticated + Username string // Username of the authenticated user + Password string // password for the session, can be used for re-authentication + ExpireAt int64 // session expiration time (Unix timestamp) + Conn net.Conn // The user's network connection + Writer *bufio.Writer // bufio writer for the client connection to send articles, headers, bodies, list, xover, xhdr, ... (big data) + //tpReader *textproto.Reader // textproto reader for easier command handling + //tpWriter *textproto.Writer // textproto writer for easier command handling + CliTp *textproto.Conn // textproto connection for easier command handling + tmpRXBytes uint64 // proxy has RECEIVED this amount of bytes FROM CLIENT via POST/IHAVE/TAKETHIS in last 60 seconds + RXBytes uint64 // proxy has RECEIVED this amount of bytes FROM CLIENT via POST/IHAVE/TAKETHIS in total this session + tmpTXBytes uint64 // proxy has SENT this amount of bytes TO CLIENT via ARTICLE/HEAD/BODY in last 60 seconds + TXBytes uint64 // proxy has SENT this amount of bytes TO CLIENT via ARTICLE/HEAD/BODY in total this session + ConnectedAt time.Time // Timestamp when the session was created + LastCmd time.Time // Timestamp of the last command received + Group string // current group the user is in (used by GROUP command) + MsgNum int64 // current message number in the group (used by STAT, ARTICLE, etc. commands) + Cron time.Time // last run of periodic tasks, e.g., checking session expiration + selectedProvider *Provider // The provider selected for this session, used for routing commands // Add other session-specific data here, e.g., current group, article pointer, etc. } @@ -225,7 +225,7 @@ type ArticleOverview struct { // handleGroupCommand processes the GROUP command by selecting a newsgroup func (ps *ProxySession) handleGroupCommand(args []string) error { if len(args) < 1 { - ps.tpWriter.PrintfLine("501 Syntax error: GROUP ") + ps.CliTp.PrintfLine("501 Syntax error: GROUP ") return nil } groupName := args[0] @@ -280,7 +280,7 @@ func (ps *ProxySession) handleGroupCommand(args []string) error { if group == nil { ps.selectedProvider = nil // Reset selected provider if no group found - ps.tpWriter.PrintfLine("411 No such newsgroup") + ps.CliTp.PrintfLine("411 No such newsgroup") return nil } @@ -289,7 +289,7 @@ func (ps *ProxySession) handleGroupCommand(args []string) error { ps.MsgNum = group.High // Set to the high water mark as default // Return a successful GROUP response: 211 count low high group_name - ps.tpWriter.PrintfLine("211 %d %d %d %s", + ps.CliTp.PrintfLine("211 %d %d %d %s", group.Count, group.Low, group.High, group.Name) dlog(always, "%s | Selected group: %s (articles: %d, range: %d-%d)", @@ -357,8 +357,8 @@ func (ps *ProxySession) handleListCommand(args []string) error { } // Start our response to the client - ps.tpWriter.PrintfLine("215 List of newsgroups follows") - dw := ps.tpWriter.DotWriter() + ps.CliTp.PrintfLine("215 List of newsgroups follows") + dw := ps.CliTp.DotWriter() // Pass through the lines for _, line := range lines { @@ -374,7 +374,7 @@ func (ps *ProxySession) handleListCommand(args []string) error { if ps.selectedProvider != nil { ps.selectedProvider = nil } - ps.tpWriter.PrintfLine("503 No providers available for LIST command") + ps.CliTp.PrintfLine("503 No providers available for LIST command") return nil } return nil @@ -384,7 +384,7 @@ func (ps *ProxySession) handleListCommand(args []string) error { func (ps *ProxySession) handleXOverCommand(args []string, isXOVER bool) error { // Must have a selected group first if ps.Group == "" { - ps.tpWriter.PrintfLine("412 No newsgroup selected") + ps.CliTp.PrintfLine("412 No newsgroup selected") return nil } @@ -394,7 +394,7 @@ func (ps *ProxySession) handleXOverCommand(args []string, isXOVER bool) error { } else { // If no range specified, use current article number if ps.MsgNum <= 0 { - ps.tpWriter.PrintfLine("420 No current article selected") + ps.CliTp.PrintfLine("420 No current article selected") return nil } rangeArg = fmt.Sprintf("%d", ps.MsgNum) @@ -409,7 +409,7 @@ func (ps *ProxySession) handleXOverCommand(args []string, isXOVER bool) error { var err error startNum, err = strconv.ParseInt(parts[0], 10, 64) if err != nil { - ps.tpWriter.PrintfLine("501 Invalid article range") + ps.CliTp.PrintfLine("501 Invalid article range") return nil } @@ -419,7 +419,7 @@ func (ps *ProxySession) handleXOverCommand(args []string, isXOVER bool) error { } else { endNum, err = strconv.ParseInt(parts[1], 10, 64) if err != nil { - ps.tpWriter.PrintfLine("501 Invalid article range") + ps.CliTp.PrintfLine("501 Invalid article range") return nil } } @@ -428,7 +428,7 @@ func (ps *ProxySession) handleXOverCommand(args []string, isXOVER bool) error { var err error startNum, err = strconv.ParseInt(rangeArg, 10, 64) if err != nil { - ps.tpWriter.PrintfLine("501 Invalid article number") + ps.CliTp.PrintfLine("501 Invalid article number") return nil } endNum = startNum // Just one article @@ -505,8 +505,8 @@ func (ps *ProxySession) handleXOverCommand(args []string, isXOVER bool) error { } selected = true // We found a provider that can handle this command // Start our response to the client - ps.tpWriter.PrintfLine("224 Overview information follows") - dw := ps.tpWriter.DotWriter() + ps.CliTp.PrintfLine("224 Overview information follows") + dw := ps.CliTp.DotWriter() // Pass through the lines for _, line := range lines { @@ -521,7 +521,7 @@ func (ps *ProxySession) handleXOverCommand(args []string, isXOVER bool) error { if ps.selectedProvider != nil { ps.selectedProvider = nil } - ps.tpWriter.PrintfLine("503 No providers have the selected group") + ps.CliTp.PrintfLine("503 No providers have the selected group") return nil } return nil @@ -530,13 +530,13 @@ func (ps *ProxySession) handleXOverCommand(args []string, isXOVER bool) error { // handleXHdrCommand processes the XHDR/HDR command func (ps *ProxySession) handleXHdrCommand(args []string, isXHDR bool) error { if len(args) < 1 { - ps.tpWriter.PrintfLine("501 Syntax error: XHDR
[range|]") + ps.CliTp.PrintfLine("501 Syntax error: XHDR
[range|]") return nil } // Must have a selected group first, unless message-id is specified if ps.Group == "" && !strings.HasPrefix(args[len(args)-1], "<") { - ps.tpWriter.PrintfLine("412 No newsgroup selected") + ps.CliTp.PrintfLine("412 No newsgroup selected") return nil } @@ -557,7 +557,7 @@ func (ps *ProxySession) handleXHdrCommand(args []string, isXHDR bool) error { } else { // If no range/msgid specified, use current article number if ps.MsgNum <= 0 { - ps.tpWriter.PrintfLine("420 No current article selected") + ps.CliTp.PrintfLine("420 No current article selected") return nil } rangeSpec = fmt.Sprintf("%d", ps.MsgNum) @@ -635,8 +635,8 @@ func (ps *ProxySession) handleXHdrCommand(args []string, isXHDR bool) error { } selected = true // Start our response to the client - ps.tpWriter.PrintfLine("221 Header follows") - dw := ps.tpWriter.DotWriter() + ps.CliTp.PrintfLine("221 Header follows") + dw := ps.CliTp.DotWriter() // Pass through the lines for _, line := range lines { @@ -651,7 +651,7 @@ func (ps *ProxySession) handleXHdrCommand(args []string, isXHDR bool) error { if ps.selectedProvider != nil { ps.selectedProvider = nil } - ps.tpWriter.PrintfLine("503 No providers have the selected group or article") + ps.CliTp.PrintfLine("503 No providers have the selected group or article") return nil } // If we couldn't find any provider or all failed @@ -662,13 +662,13 @@ func (ps *ProxySession) handleXHdrCommand(args []string, isXHDR bool) error { func (ps *ProxySession) handleNextOrLastCommand(isNext bool) error { // Must have a selected group first if ps.Group == "" { - ps.tpWriter.PrintfLine("412 No newsgroup selected") + ps.CliTp.PrintfLine("412 No newsgroup selected") return nil } // Must have a current article selected if ps.MsgNum <= 0 { - ps.tpWriter.PrintfLine("420 No current article selected") + ps.CliTp.PrintfLine("420 No current article selected") return nil } @@ -760,7 +760,7 @@ func (ps *ProxySession) handleNextOrLastCommand(isNext bool) error { } else { which = "previous" } - ps.tpWriter.PrintfLine("421 No %s article to retrieve", which) + ps.CliTp.PrintfLine("421 No %s article to retrieve", which) return nil } } else { @@ -774,9 +774,9 @@ func (ps *ProxySession) handleNextOrLastCommand(isNext bool) error { ps.MsgNum = newNum // Return successful response - ps.tpWriter.PrintfLine("223 %d %s", newNum, messageID) + ps.CliTp.PrintfLine("223 %d %s", newNum, messageID) } else { - ps.tpWriter.PrintfLine("503 Invalid response from server") + ps.CliTp.PrintfLine("503 Invalid response from server") } } return nil @@ -785,7 +785,7 @@ func (ps *ProxySession) handleNextOrLastCommand(isNext bool) error { if ps.selectedProvider != nil { ps.selectedProvider = nil } - ps.tpWriter.PrintfLine("503 No providers have the selected group") + ps.CliTp.PrintfLine("503 No providers have the selected group") return nil } return nil diff --git a/ProxyHelper.go b/ProxyHelper.go index 550f972..498326b 100644 --- a/ProxyHelper.go +++ b/ProxyHelper.go @@ -8,8 +8,8 @@ import ( "log" mrand "math/rand" "net" - "net/textproto" "os" + "path/filepath" "strconv" "strings" "time" @@ -64,10 +64,10 @@ func LinesWriter(cliwriter *bufio.Writer, conn net.Conn, code int, item *segment return txb, nil } // end func LinesWriter -func printCapabilities(tpWriter *textproto.Writer) { +func (ps *ProxySession) printCapabilities() { // Respond with server capabilities (RFC 3977 Section 5.3) - tpWriter.PrintfLine("101 Capability list:") - dw := tpWriter.DotWriter() + ps.CliTp.PrintfLine("101 Capability list:") + dw := ps.CliTp.DotWriter() fmt.Fprintln(dw, "VERSION 2") // Indicates RFC 3977 support fmt.Fprintln(dw, "READER") // Indicates MODE READER support fmt.Fprintln(dw, "AUTHINFO USER PASS") // Supports AUTHINFO USER and AUTHINFO PASS @@ -79,13 +79,10 @@ func printCapabilities(tpWriter *textproto.Writer) { // File format: username:bcrypt_hash:maxconns:ExpireAt_unix_timestamp func loadPasswdFile(filename string) error { // Before loading, merge any .new.* files into the main passwd file - base := filename - if idx := strings.LastIndex(filename, "/"); idx != -1 { - base = filename[idx+1:] - } - dir := "." - if idx := strings.LastIndex(filename, "/"); idx != -1 { - dir = filename[:idx] + dir := filepath.Dir(filename) + base := filepath.Base(filename) + if dir == "" { + dir = "." } pattern := fmt.Sprintf("%s.new.", base) entries, err := os.ReadDir(dir) @@ -96,7 +93,7 @@ func loadPasswdFile(filename string) error { } name := entry.Name() if strings.HasPrefix(name, pattern) { - newfile := dir + "/" + name + newfile := filepath.Join(dir, name) // Append contents to main passwd file func() { mainf, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640) @@ -281,13 +278,14 @@ func addUserToProxyPasswdFile(userData *UserData, filename string) error { return nil } -// generateRandomHexCredentials generates a random username and password, each 10 bytes, output as hex strings. +const DefaultPasswordLength = 8 // 8 bytes will give us 16 hex characters, which is a common length for usernames/passwords + +// generateRandomHexCredentials generates a random username and password, each 8 bytes, output as hex strings. func generateRandomHexCredentials() (username string, password string, err error) { - const length = 8 // 8 bytes will give us 16 hex characters, which is a common length for usernames/passwords // Use crypto/rand for cryptographically secure random bytes // If you want longer usernames/passwords, adjust the length accordingly. // For example, 10 bytes would give 20 hex characters. - buf := make([]byte, length) + buf := make([]byte, DefaultPasswordLength) if _, err := rand.Read(buf); err != nil { return "", "", fmt.Errorf("failed to generate random username: %w", err) } @@ -310,14 +308,14 @@ func (ps *ProxySession) Close() { ps.Conn.Close() // Close the network connection ps.Conn = nil // Clear the connection to avoid dangling pointer } - ps.Authed = false // Mark session as unauthenticated - ps.Username = "" // Clear username to avoid dangling pointer - ps.Password = "" // Clear username to avoid dangling pointer - ps.ExpireAt = 0 // Clear expiration time + ps.Authed = false // Mark session as unauthenticated + ps.Username = "" // Clear username to avoid dangling pointer + ps.Password = "" // Clear username to avoid dangling pointer + ps.ExpireAt = 0 // Clear expiration time username := ps.Username // Store username for logging - ps.Username = "" // Clear username to avoid dangling pointer - ps.Password = "" // Clear username to avoid dangling pointer - ps.ExpireAt = 0 // Clear expiration time + ps.Username = "" // Clear username to avoid dangling pointer + ps.Password = "" // Clear username to avoid dangling pointer + ps.ExpireAt = 0 // Clear expiration time log.Printf("Closed session for user '%s'", username) } diff --git a/Server.go b/Server.go index de00a85..db1d104 100644 --- a/Server.go +++ b/Server.go @@ -20,11 +20,7 @@ import ( // handleConnection manages a single NNTP proxy client connection. func handleConnection(conn net.Conn) { //log.Printf("Handling connection from %s", conn.RemoteAddr()) - - // Use textproto.Reader and textproto.Writer - tpReader := textproto.NewReader(bufio.NewReader(conn)) - tpWriter := textproto.NewWriter(bufio.NewWriter(conn)) - + cliTp := textproto.NewConn(conn) var currentUser string // Stores username after successful AUTHINFO USER authenticated := false now := time.Now() // Get the current time for session initialization @@ -70,14 +66,14 @@ func handleConnection(conn net.Conn) { // 200 service available, posting allowed // 201 service available, posting prohibited time.Sleep(time.Duration(mrand.Intn(128)) * time.Millisecond) // Random delay to simulate server startup - tpWriter.PrintfLine("%d %s", welcomeCode, nntpWelcomeMessage) + cliTp.PrintfLine("%d %s", welcomeCode, nntpWelcomeMessage) // incoming client connection is captured in this for loop until QUIT command or error occurs - tpWriter.W.Flush() // Ensure the welcome message is sent immediately + //cliTp.W.Flush() // Ensure the welcome message is sent immediately forever: for { // Read commands from the client in a loop - line, err := tpReader.ReadLine() // Use ReadLine from textproto + line, err := cliTp.ReadLine() // Use ReadLine from textproto if err != nil { if err != io.EOF { // Check for common textproto errors, like malformed lines or read errors @@ -97,7 +93,7 @@ forever: if len(line) > 128 { // only command lines are captured here // reading a line longer than 128 characters is not allowed by RFC 3977 // Line is too long, send error response - tpWriter.PrintfLine("501 Syntax error: cmd line too long") + cliTp.PrintfLine("501 Syntax error: cmd line too long") return } line = strings.TrimSpace(line) // TrimSpace is still useful @@ -130,48 +126,46 @@ forever: switch command { case "CAPABILITIES": - printCapabilities(tpWriter) + ps.printCapabilities() case "AUTHINFO": if authenticated { - tpWriter.PrintfLine("502 Already authenticated") + cliTp.PrintfLine("502 Already authenticated") return } if len(parts) < 2 { - tpWriter.PrintfLine("501 Syntax error in AUTHINFO command") + cliTp.PrintfLine("501 Syntax error in AUTHINFO command") return } authCmd := strings.ToUpper(parts[1]) switch authCmd { case "USER": - if len(parts) < 3 { - tpWriter.PrintfLine("501 Syntax error: AUTHINFO USER ") + cliTp.PrintfLine("501 Syntax error: AUTHINFO USER ") return } currentUser = parts[2] // RFC 3977 suggests 381 if user is valid, otherwise 481/502 or proceed and fail at PASS. // To avoid user enumeration, some servers always respond 381. time.Sleep(time.Duration(mrand.Intn(128)) * time.Millisecond) // Random delay - tpWriter.PrintfLine("381 Password required") + cliTp.PrintfLine("381 Password required") continue forever // Continue to handle further commands after AUTHINFO USER case "PASS": - if currentUser == "" { - tpWriter.PrintfLine("482 Authentication commands out of sequence (AUTHINFO USER first)") + cliTp.PrintfLine("482 Authentication commands out of sequence (AUTHINFO USER first)") return } if len(parts) < 3 { - tpWriter.PrintfLine("501 Syntax error: AUTHINFO PASS ") + cliTp.PrintfLine("501 Syntax error: AUTHINFO PASS ") return } time.Sleep(time.Duration(mrand.Intn(128)) * time.Millisecond) // Random delay passwordToVerify := parts[2] if !verifyPassword(currentUser, passwordToVerify) { - tpWriter.PrintfLine("481 Authentication failed") + cliTp.PrintfLine("481 Authentication failed") log.Printf("Failed authentication attempt for user '%s' from %s", currentUser, conn.RemoteAddr()) return } @@ -188,7 +182,7 @@ forever: // Unlock before sleeping time.Sleep(time.Duration(mrand.Intn(1000)) * time.Millisecond) // Random delay log.Printf("Failed to reload passwd file: %v", err) - tpWriter.PrintfLine("481 Authentication failed (passwd file reload error)") + cliTp.PrintfLine("481 Authentication failed (passwd file reload error)") return } } @@ -200,7 +194,7 @@ forever: // This case should ideally not be reached if verifyPassword relies on passwdMap proxyMutex.RUnlock() time.Sleep(time.Duration(mrand.Intn(1000)) * time.Millisecond) // Random delay - tpWriter.PrintfLine("481 Authentication failed (user data inconsistency)") + cliTp.PrintfLine("481 Authentication failed (user data inconsistency)") log.Printf("User data not found for '%s' in passwdMap after successful verifyPassword. Potential data inconsistency.", currentUser) return } @@ -209,7 +203,7 @@ forever: if userData.ExpireAt > 0 && time.Now().Unix() > userData.ExpireAt { proxyMutex.RUnlock() time.Sleep(time.Duration(mrand.Intn(1000)) * time.Millisecond) // Random delay - tpWriter.PrintfLine("481 Authentication failed (account expired)") + cliTp.PrintfLine("481 Authentication failed (account expired)") log.Printf("Authentication failed for user '%s' from %s: account expired (ExpireAt: %d, Current: %d)", currentUser, conn.RemoteAddr(), userData.ExpireAt, time.Now().Unix()) return } @@ -220,7 +214,7 @@ forever: if userData.MaxConns > 0 && CountConns[currentUser] >= userData.MaxConns { proxyMutex.Unlock() time.Sleep(time.Duration(mrand.Intn(1000)) * time.Millisecond) // Random delay - tpWriter.PrintfLine("452 Too many connections for this user. Please try again later.") + cliTp.PrintfLine("452 Too many connections for this user. Please try again later.") log.Printf("Connection denied for user '%s' from %s: too many connections (current: %d, max: %d)", currentUser, conn.RemoteAddr(), CountConns[currentUser], userData.MaxConns) // Do not set authenticated to true, connection is rejected before full authentication. return @@ -235,19 +229,17 @@ forever: ps.id = CID // Assign a unique session ID ps.Authed = true ps.Username = currentUser - ps.Password = passwordToVerify // Store the hashed password in the session so we can check every now and then if password has changed and close the session - ps.ExpireAt = userData.ExpireAt // Set session expiration time from user data - ps.Conn = conn // Store the connection in the session - ps.CliTp = textproto.NewConn(conn) // Create a textproto connection for easier command handling - ps.Writer = bufio.NewWriter(conn) // Create a bufio writer for the client connection - ps.tpReader = tpReader // Store the textproto reader in the session - ps.tpWriter = tpWriter // Store the textproto writer in the session - ps.Cron = ps.ConnectedAt // Initialize cron time for periodic tasks + ps.Password = passwordToVerify // Store the hashed password in the session so we can check every now and then if password has changed and close the session + ps.ExpireAt = userData.ExpireAt // Set session expiration time from user data + ps.Conn = conn // Store the connection in the session + ps.CliTp = cliTp // Create a textproto connection for easier command handling + ps.Writer = bufio.NewWriter(conn) // Create a bufio writer for the client connection + ps.Cron = ps.ConnectedAt // Initialize cron time for periodic tasks // Store the session in the global ProxySessions map ProxySessions[currentUser] = ps - tpWriter.PrintfLine("281 Welcome to NZBreX Proxy! Your conns: %d/%d. Exp: '%v'", + cliTp.PrintfLine("281 Welcome to NZBreX Proxy! Your conns: %d/%d. Exp: '%v'", CountConns[currentUser], userData.MaxConns, time.Unix(userData.ExpireAt, 0).Format(time.RFC1123Z)) dlog(cfg.opt.Debug, "User '%s' authenticated. Active connections for user: %d/%d", currentUser, CountConns[currentUser], userData.MaxConns) @@ -257,34 +249,33 @@ forever: continue forever // Continue to handle further commands after successful authentication default: - tpWriter.PrintfLine("501 Unknown AUTHINFO subcommand: %s", authCmd) + cliTp.PrintfLine("501 Unknown AUTHINFO subcommand: %s", authCmd) return } case "MODE": if len(parts) < 2 { - tpWriter.PrintfLine("501 Syntax error in MODE command") + cliTp.PrintfLine("501 Syntax error in MODE command") return } switch strings.ToUpper(parts[1]) { case "READER": - tpWriter.PrintfLine("201 Posting prohibited") + cliTp.PrintfLine("201 Posting prohibited") case "STREAM": if !authenticated { - tpWriter.PrintfLine("480 Authentication required for MODE STREAM") + cliTp.PrintfLine("480 Authentication required for MODE STREAM") return } - tpWriter.PrintfLine("200 Switching to STREAM mode") + cliTp.PrintfLine("200 Switching to STREAM mode") default: - tpWriter.PrintfLine("501 Unknown mode") + cliTp.PrintfLine("501 Unknown mode") return } continue forever // Continue to handle further commands after MODE command case "QUIT": - - tpWriter.PrintfLine("205 Closing connection - goodbye.") + cliTp.PrintfLine("205 Closing connection - goodbye.") log.Printf("Client %s issued QUIT.", conn.RemoteAddr()) return @@ -292,10 +283,10 @@ forever: if authenticated { // Handle other NNTP commands for authenticated users here // e.g., GROUP, ARTICLE, LIST, NEXT, LAST, DATE, etc. - tpWriter.PrintfLine("500 Unknown command: %s (authenticated)", command) + cliTp.PrintfLine("500 Unknown command: %s (authenticated)", command) } else { // RFC 3977: "480 Authentication required" for most commands if not authenticated - tpWriter.PrintfLine("480 Authentication required") + cliTp.PrintfLine("480 Authentication required") } } // end switch command } // end for forever @@ -349,7 +340,7 @@ func (ps *ProxySession) handleRequest(command string, args []string) error { } else if num > 0 && ps.Group == "" { // TODO select newsgroup - ps.tpWriter.PrintfLine("412 No newsgroup selected to read messageid: %d", num) + ps.CliTp.PrintfLine("412 No newsgroup selected to read messageid: %d", num) return retry } else if isvalid { @@ -373,11 +364,11 @@ func (ps *ProxySession) handleRequest(command string, args []string) error { } case "CAPABILITIES": - printCapabilities(ps.tpWriter) + ps.printCapabilities() return nil // No error, capabilities printed case "DATE": - ps.tpWriter.PrintfLine("111 %s", time.Now().Format(time.RFC1123Z)) + ps.CliTp.PrintfLine("111 %s", time.Now().Format(time.RFC1123Z)) return nil // No error, date printed case "LIST": @@ -399,17 +390,17 @@ func (ps *ProxySession) handleRequest(command string, args []string) error { return ps.handleNextOrLastCommand(false) case "QUIT": - ps.tpWriter.PrintfLine("205 Closing connection - goodbye. uploaded=%d downloaded=%d connected='%v'", ps.RXBytes, ps.TXBytes, time.Since(ps.ConnectedAt)) + ps.CliTp.PrintfLine("205 Closing connection - goodbye. uploaded=%d downloaded=%d connected='%v'", ps.RXBytes, ps.TXBytes, time.Since(ps.ConnectedAt)) log.Printf(" %s | quit", ps.Username) return fmt.Errorf("client quit") default: - ps.tpWriter.PrintfLine("502 Unknown command") + ps.CliTp.PrintfLine("502 Unknown command") return fmt.Errorf("unknown command: %s", command) } // end switch command1 if !pass { - ps.tpWriter.PrintfLine("501 Syntax error: command %s requires a valid message ID", command) + ps.CliTp.PrintfLine("501 Syntax error: command %s requires a valid message ID", command) return fmt.Errorf("syntax error: command %s requires a valid message ID", command) } // Now we have a valid command and messageId (if applicable), proceed to handle the request @@ -537,10 +528,10 @@ loopProvider: } // end for loopProvider if response != "0" { if response != "" { - ps.tpWriter.PrintfLine("%s", response) + ps.CliTp.PrintfLine("%s", response) } else { dlog(always, " %s | ERROR response to client empty", ps.Username) - ps.tpWriter.PrintfLine("500 Unknown error occurred while processing command %s", command) + ps.CliTp.PrintfLine("500 Unknown error occurred while processing command %s", command) } } return nil // Return nil to indicate the command was handled successfully. an error will disconnect the user From 4c2c19bd1bc57ae08124225e81cd699f50f5eac6 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 12:51:59 +0000 Subject: [PATCH 086/152] compiler --- local_crossbuild_windows-amd64.sh | 4 ++-- rapidyenc/build_rapidyenc_linux-amd64.sh | 3 ++- rapidyenc/build_rapidyenc_linux-arm64.sh | 2 +- rapidyenc/crossbuild_rapidyenc_darwin-amd64.sh | 2 +- rapidyenc/crossbuild_rapidyenc_windows-amd64.sh | 3 ++- rapidyenc/decoder.go | 10 ++++------ rapidyenc/{encoder.go => encoder.go.off} | 2 +- rapidyenc/{encoder_test.go => encoder_test.go.off} | 0 rapidyenc/librapidyenc_darwin.a | 1 - rapidyenc/librapidyenc_linux_amd64.a | 1 - rapidyenc/librapidyenc_linux_arm64.a | 1 - rapidyenc/librapidyenc_windows_amd64.a | 1 - 12 files changed, 13 insertions(+), 17 deletions(-) rename rapidyenc/{encoder.go => encoder.go.off} (98%) rename rapidyenc/{encoder_test.go => encoder_test.go.off} (100%) delete mode 120000 rapidyenc/librapidyenc_darwin.a delete mode 120000 rapidyenc/librapidyenc_linux_amd64.a delete mode 120000 rapidyenc/librapidyenc_linux_arm64.a delete mode 120000 rapidyenc/librapidyenc_windows_amd64.a diff --git a/local_crossbuild_windows-amd64.sh b/local_crossbuild_windows-amd64.sh index a3ad878..04e3f67 100755 --- a/local_crossbuild_windows-amd64.sh +++ b/local_crossbuild_windows-amd64.sh @@ -34,8 +34,8 @@ fi cd ../ # Check if rapidyenc library was built successfully -if [ ! -f "rapidyenc/librapidyenc.a" ]; then - echo "ERROR: rapidyenc static library not found at rapidyenc/librapidyenc.a" +if [ ! -f "rapidyenc/librapidyenc_windows_amd64.a" ]; then + echo "ERROR: rapidyenc static library not found at rapidyenc/librapidyenc_windows_amd64.a" exit 1 fi diff --git a/rapidyenc/build_rapidyenc_linux-amd64.sh b/rapidyenc/build_rapidyenc_linux-amd64.sh index 234b8d4..8232b79 100755 --- a/rapidyenc/build_rapidyenc_linux-amd64.sh +++ b/rapidyenc/build_rapidyenc_linux-amd64.sh @@ -7,5 +7,6 @@ cmake .. || exit 4 cmake --build . --config Release || exit 5 ls . rapidyenc_static/ cd ../../ || exit 6 -cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a . || exit 7 +cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a ./librapidyenc_linux_amd64.a || exit 7 +ln -sfv ./librapidyenc_linux_amd64.a ./librapidyenc.a #rm -rf rapidyenc/build diff --git a/rapidyenc/build_rapidyenc_linux-arm64.sh b/rapidyenc/build_rapidyenc_linux-arm64.sh index 19a453f..bda6c03 100755 --- a/rapidyenc/build_rapidyenc_linux-arm64.sh +++ b/rapidyenc/build_rapidyenc_linux-arm64.sh @@ -7,5 +7,5 @@ cmake .. -DCMAKE_TOOLCHAIN_FILE=../toolchain-linux-arm64.cmake || exit 4 cmake --build . --config Release || exit 5 ls . rapidyenc_static/ cd ../../ || exit 6 -cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a . || exit 7 +cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a ./librapidyenc_linux_arm64.a || exit 7 #rm -rf rapidyenc/build diff --git a/rapidyenc/crossbuild_rapidyenc_darwin-amd64.sh b/rapidyenc/crossbuild_rapidyenc_darwin-amd64.sh index 42a6b30..ee39767 100755 --- a/rapidyenc/crossbuild_rapidyenc_darwin-amd64.sh +++ b/rapidyenc/crossbuild_rapidyenc_darwin-amd64.sh @@ -7,6 +7,6 @@ cmake .. -DCMAKE_TOOLCHAIN_FILE=../../toolchain-darwin.cmake || exit 4 cmake --build . --config Release || exit 5 ls . rapidyenc_static/ cd ../../ || exit 6 -cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a . || exit 7 +cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a ./librapidyenc_darwin.a || exit 7 cp -v rapidyenc/build/librapidyenc.dylib . || exit 8 #rm -rf rapidyenc/build diff --git a/rapidyenc/crossbuild_rapidyenc_windows-amd64.sh b/rapidyenc/crossbuild_rapidyenc_windows-amd64.sh index 286bf0c..9150026 100755 --- a/rapidyenc/crossbuild_rapidyenc_windows-amd64.sh +++ b/rapidyenc/crossbuild_rapidyenc_windows-amd64.sh @@ -7,6 +7,7 @@ cmake .. -DCMAKE_TOOLCHAIN_FILE=../../toolchain-mingw64.cmake || exit 4 cmake --build . --config Release || exit 5 ls . rapidyenc_static/ cd ../../ || exit 6 -cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a . || exit 7 +cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a ./librapidyenc_windows_amd64.a || exit 7 +ln -sfv ./librapidyenc_windows_amd64.a ./librapidyenc.a cp -v rapidyenc/build/librapidyenc.dll . || exit 8 #rm -rf rapidyenc/build diff --git a/rapidyenc/decoder.go b/rapidyenc/decoder.go index 26cf40e..ac72073 100644 --- a/rapidyenc/decoder.go +++ b/rapidyenc/decoder.go @@ -3,9 +3,7 @@ package rapidyenc /* #cgo CFLAGS: -I${SRCDIR}/src #cgo darwin LDFLAGS: ${SRCDIR}/librapidyenc_darwin.a -lstdc++ -#cgo windows,amd64 LDFLAGS: ${SRCDIR}/librapidyenc.a -static-libstdc++ -static-libgcc -#cgo windows,386 LDFLAGS: ${SRCDIR}/librapidyenc.a -static-libstdc++ -static-libgcc -#cgo windows,arm LDFLAGS: ${SRCDIR}/librapidyenc.a -static-libstdc++ -static-libgcc +#cgo windows,amd64 LDFLAGS: ${SRCDIR}/librapidyenc_windows_amd64.a -lstdc++ -static-libstdc++ -static-libgcc #cgo linux,amd64 LDFLAGS: ${SRCDIR}/librapidyenc_linux_amd64.a -lstdc++ #cgo linux,arm64 LDFLAGS: ${SRCDIR}/librapidyenc_linux_arm64.a -lstdc++ #include "rapidyenc.h" @@ -257,7 +255,7 @@ func (d *Decoder) Read(p []byte) (int, error) { // It then incrementally decodes chunks of yEnc encoded data before returning to line-by-line processing // for the footer and EOF pattern (.\r\n) func (d *Decoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { -transform: +loop: if d.body && d.format == FormatYenc { nd, ns, end, _ := DecodeIncremental(dst[nDst:], src[nSrc:], &d.State) if nd > 0 { @@ -311,7 +309,7 @@ transform: } } else if d.format == FormatYenc && ((!d.part && d.m.Size != d.endSize) || (d.endSize != d.actualSize)) { err = fmt.Errorf("[rapidyenc] expected size %d but got %d: %w", d.m.Size, d.actualSize, ErrDataCorruption) - } else if d.crc && d.expectedCrc != d.m.Hash { + } else if d.format == FormatYenc && d.crc && d.expectedCrc != d.m.Hash { // If we have a segment ID, use it for debugging otherwise use an empty string. err = fmt.Errorf("[rapidyenc] ERROR CRC32 expected hash '%#08x' but got '%#08x'! seg.Id='%s' err: %w", d.expectedCrc, d.m.Hash, *d.segId, ErrCrcMismatch) } else { @@ -335,7 +333,7 @@ transform: switch d.format { case FormatYenc: d.processYenc(line) - goto transform + goto loop case FormatUU: d.processYenc(line) // Process UU headers (begin/end) if d.body && !bytes.HasPrefix(line, []byte("begin ")) && !bytes.HasPrefix(line, []byte("end")) { diff --git a/rapidyenc/encoder.go b/rapidyenc/encoder.go.off similarity index 98% rename from rapidyenc/encoder.go rename to rapidyenc/encoder.go.off index 7554221..159e7e8 100644 --- a/rapidyenc/encoder.go +++ b/rapidyenc/encoder.go.off @@ -3,7 +3,7 @@ package rapidyenc /* #cgo CFLAGS: -I${SRCDIR}/src #cgo darwin LDFLAGS: ${SRCDIR}/librapidyenc_darwin.a -lstdc++ -#cgo windows,amd64 LDFLAGS: ${SRCDIR}/librapidyenc_windows_amd64.a -lstdc++ +#cgo windows,amd64 LDFLAGS: ${SRCDIR}/librapidyenc_windows_amd64.a -static-libstdc++ -static-libgcc #cgo linux,amd64 LDFLAGS: ${SRCDIR}/librapidyenc_linux_amd64.a -lstdc++ #cgo linux,arm64 LDFLAGS: ${SRCDIR}/librapidyenc_linux_arm64.a -lstdc++ #include "rapidyenc.h" diff --git a/rapidyenc/encoder_test.go b/rapidyenc/encoder_test.go.off similarity index 100% rename from rapidyenc/encoder_test.go rename to rapidyenc/encoder_test.go.off diff --git a/rapidyenc/librapidyenc_darwin.a b/rapidyenc/librapidyenc_darwin.a deleted file mode 120000 index 6ae5162..0000000 --- a/rapidyenc/librapidyenc_darwin.a +++ /dev/null @@ -1 +0,0 @@ -librapidyenc.a \ No newline at end of file diff --git a/rapidyenc/librapidyenc_linux_amd64.a b/rapidyenc/librapidyenc_linux_amd64.a deleted file mode 120000 index 6ae5162..0000000 --- a/rapidyenc/librapidyenc_linux_amd64.a +++ /dev/null @@ -1 +0,0 @@ -librapidyenc.a \ No newline at end of file diff --git a/rapidyenc/librapidyenc_linux_arm64.a b/rapidyenc/librapidyenc_linux_arm64.a deleted file mode 120000 index 6ae5162..0000000 --- a/rapidyenc/librapidyenc_linux_arm64.a +++ /dev/null @@ -1 +0,0 @@ -librapidyenc.a \ No newline at end of file diff --git a/rapidyenc/librapidyenc_windows_amd64.a b/rapidyenc/librapidyenc_windows_amd64.a deleted file mode 120000 index 6ae5162..0000000 --- a/rapidyenc/librapidyenc_windows_amd64.a +++ /dev/null @@ -1 +0,0 @@ -librapidyenc.a \ No newline at end of file From 50da338bee033e7ee1532c1e081628ab26cc4d0a Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 13:01:52 +0000 Subject: [PATCH 087/152] test build --- rapidyenc/{encoder.go.off => encoder.go} | 2 +- rapidyenc/{encoder_test.go.off => encoder_test.go} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename rapidyenc/{encoder.go.off => encoder.go} (97%) rename rapidyenc/{encoder_test.go.off => encoder_test.go} (100%) diff --git a/rapidyenc/encoder.go.off b/rapidyenc/encoder.go similarity index 97% rename from rapidyenc/encoder.go.off rename to rapidyenc/encoder.go index 159e7e8..1e29278 100644 --- a/rapidyenc/encoder.go.off +++ b/rapidyenc/encoder.go @@ -3,7 +3,7 @@ package rapidyenc /* #cgo CFLAGS: -I${SRCDIR}/src #cgo darwin LDFLAGS: ${SRCDIR}/librapidyenc_darwin.a -lstdc++ -#cgo windows,amd64 LDFLAGS: ${SRCDIR}/librapidyenc_windows_amd64.a -static-libstdc++ -static-libgcc +#cgo windows,amd64 LDFLAGS: ${SRCDIR}/librapidyenc_windows_amd64.a -lstdc++ -static-libstdc++ -static-libgcc #cgo linux,amd64 LDFLAGS: ${SRCDIR}/librapidyenc_linux_amd64.a -lstdc++ #cgo linux,arm64 LDFLAGS: ${SRCDIR}/librapidyenc_linux_arm64.a -lstdc++ #include "rapidyenc.h" diff --git a/rapidyenc/encoder_test.go.off b/rapidyenc/encoder_test.go similarity index 100% rename from rapidyenc/encoder_test.go.off rename to rapidyenc/encoder_test.go From 9533397089751332ada700baa49ac2c9bcf37782 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 13:16:28 +0000 Subject: [PATCH 088/152] test mac arm --- .github/workflows/build_and_publish_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 971047e..92e3fbd 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -302,7 +302,7 @@ jobs: ./crossbuild_rapidyenc_darwin-amd64.sh || ./build_rapidyenc_linux-amd64.sh else # For arm64, we might need to build natively or cross-compile - ./build_rapidyenc_linux-amd64.sh + ./build_rapidyenc_linux-arm64.sh fi - name: "Build rapidyenc (Alpine/musl)" @@ -386,7 +386,7 @@ jobs: binary_name=NZBreX fi echo "BINARY=$binary_name" >> $GITHUB_ENV - GOARCH=${{ matrix.arch }} GOOS=${{ matrix.os }} go build -ldflags="-s -w -X main.appVersion=${{ env.VERSION }}" -o builds/${{ matrix.name }}/usr/bin/$binary_name + GOARCH=${{ matrix.arch }} GOOS=${{ matrix.os }} go build -ldflags="-X main.appVersion=${{ env.VERSION }}" -o builds/${{ matrix.name }}/usr/bin/$binary_name if [ ! -f cleanHeaders.txt ]; then echo "cleanHeaders.txt not found! Build cannot continue." >&2 exit 1 From 98b1c6f4dd228da849bac4f5b0ee39a6d9b17d08 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 13:24:00 +0000 Subject: [PATCH 089/152] retry mac build --- .github/workflows/build_and_publish_test.yml | 2 +- rapidyenc/build_rapidyenc_linux-amd64.sh | 9 +++++++-- rapidyenc/build_rapidyenc_linux-arm64.sh | 8 +++++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 92e3fbd..ae13b97 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -302,7 +302,7 @@ jobs: ./crossbuild_rapidyenc_darwin-amd64.sh || ./build_rapidyenc_linux-amd64.sh else # For arm64, we might need to build natively or cross-compile - ./build_rapidyenc_linux-arm64.sh + ./build_rapidyenc_linux-arm64.sh darwin fi - name: "Build rapidyenc (Alpine/musl)" diff --git a/rapidyenc/build_rapidyenc_linux-amd64.sh b/rapidyenc/build_rapidyenc_linux-amd64.sh index 8232b79..f68cd88 100755 --- a/rapidyenc/build_rapidyenc_linux-amd64.sh +++ b/rapidyenc/build_rapidyenc_linux-amd64.sh @@ -7,6 +7,11 @@ cmake .. || exit 4 cmake --build . --config Release || exit 5 ls . rapidyenc_static/ cd ../../ || exit 6 -cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a ./librapidyenc_linux_amd64.a || exit 7 -ln -sfv ./librapidyenc_linux_amd64.a ./librapidyenc.a +if [ "$1" = "darwin" ]; then + cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a ./librapidyenc_darwin_amd64.a || exit 7 + ln -sfv ./librapidyenc_darwin_amd64.a ./librapidyenc.a +else + cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a ./librapidyenc_linux_amd64.a || exit 7 + ln -sfv ./librapidyenc_linux_amd64.a ./librapidyenc.a +fi #rm -rf rapidyenc/build diff --git a/rapidyenc/build_rapidyenc_linux-arm64.sh b/rapidyenc/build_rapidyenc_linux-arm64.sh index bda6c03..2fc9bf7 100755 --- a/rapidyenc/build_rapidyenc_linux-arm64.sh +++ b/rapidyenc/build_rapidyenc_linux-arm64.sh @@ -7,5 +7,11 @@ cmake .. -DCMAKE_TOOLCHAIN_FILE=../toolchain-linux-arm64.cmake || exit 4 cmake --build . --config Release || exit 5 ls . rapidyenc_static/ cd ../../ || exit 6 -cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a ./librapidyenc_linux_arm64.a || exit 7 +if [ "$1" = "darwin" ]; then + cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a ./librapidyenc_darwin_arm64.a || exit 7 + ln -sfv ./librapidyenc_darwin_arm64.a ./librapidyenc.a +else + cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a ./librapidyenc_linux_arm64.a || exit 7 + ln -sfv ./librapidyenc_linux_arm64.a ./librapidyenc.a +fi #rm -rf rapidyenc/build From a53498afeedc64807372e913929a546d0e316478 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 13:35:11 +0000 Subject: [PATCH 090/152] test build --- .github/workflows/build_and_publish_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index ae13b97..2be455d 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -386,7 +386,7 @@ jobs: binary_name=NZBreX fi echo "BINARY=$binary_name" >> $GITHUB_ENV - GOARCH=${{ matrix.arch }} GOOS=${{ matrix.os }} go build -ldflags="-X main.appVersion=${{ env.VERSION }}" -o builds/${{ matrix.name }}/usr/bin/$binary_name + GOARCH=${{ matrix.arch }} GOOS=${{ matrix.os }} go build -ldflags="-s -w -X main.appVersion=${{ env.VERSION }}" -o builds/${{ matrix.name }}/usr/bin/$binary_name if [ ! -f cleanHeaders.txt ]; then echo "cleanHeaders.txt not found! Build cannot continue." >&2 exit 1 From 0721cfbe95011a7366552eaefeebcc5cad55f9bf Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 13:42:56 +0000 Subject: [PATCH 091/152] test build --- .github/workflows/build_and_publish_test.yml | 24 ++------------------ 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 2be455d..1867082 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -198,18 +198,6 @@ jobs: echo "CGO_ENABLED=1" >> $GITHUB_ENV echo "GOOS=windows" >> $GITHUB_ENV - - name: "Build rapidyenc (macOS)" - if: matrix.os == 'darwin' - timeout-minutes: 5 - run: | - cd rapidyenc - if [ "${{ matrix.arch }}" = "amd64" ]; then - ./crossbuild_rapidyenc_darwin-amd64.sh || ./build_rapidyenc_linux-amd64.sh - else - # For arm64, we might need to build natively or cross-compile - ./build_rapidyenc_linux-amd64.sh - fi - - name: "Build rapidyenc (Alpine/musl)" if: matrix.libc == 'musl' timeout-minutes: 5 @@ -299,19 +287,11 @@ jobs: run: | cd rapidyenc if [ "${{ matrix.arch }}" = "amd64" ]; then - ./crossbuild_rapidyenc_darwin-amd64.sh || ./build_rapidyenc_linux-amd64.sh - else - # For arm64, we might need to build natively or cross-compile + ./crossbuild_rapidyenc_darwin-amd64.sh || ./build_rapidyenc_linux-amd64.sh darwin + else if [ "${{ matrix.arch }}" = "arm64" ]; then ./build_rapidyenc_linux-arm64.sh darwin fi - - name: "Build rapidyenc (Alpine/musl)" - if: matrix.libc == 'musl' - timeout-minutes: 5 - run: | - cd rapidyenc && ./build_rapidyenc_linux-amd64.sh - # Note: May need Alpine-specific build script - # Run tests with race detector for supported platforms - name: "Run race detector tests (amd64/linux only)" if: matrix.os == 'linux' && matrix.arch == 'amd64' && !matrix.cross_compile From d56c03e93703114ee6ecbe534e08f3225f9ec75c Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 13:51:06 +0000 Subject: [PATCH 092/152] test build --- .github/workflows/build_and_publish_test.yml | 93 +------------------- 1 file changed, 3 insertions(+), 90 deletions(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 1867082..0c7c155 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -227,93 +227,6 @@ jobs: # Build first if not cross-compiling go build -o NZBreX -tags other . ./NZBreX -testrapidyenc -# - name: Clean Go module cache -# run: | -# rm -rf ~/.cache/go-build ~/go/pkg/mod -# - name: Restore Go modules cache -# # your cache restore step here - - #- name: Cache Go modules - # uses: actions/cache@v4 - # with: - # path: | - # ~/.cache/go-build - # ~/go/pkg/mod - # key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - # restore-keys: | - # ${{ runner.os }}-go- - - - name: "clone rapidyenc" - run: | - cd rapidyenc - if [ ! -e rapidyenc ]; then - ./clone_rapidyenc.sh - if [ ! -e rapidyenc/.git ]; then - echo "rapidyenc/ src not found, exiting" - exit 1 - fi - else - echo "rapidyenc/ src exists, skipping clone" - fi - - # Build rapidyenc for different platforms - - name: "Build rapidyenc (Linux amd64)" - if: matrix.os == 'linux' && matrix.arch == 'amd64' && !matrix.cross_compile - timeout-minutes: 5 - run: cd rapidyenc && ./build_rapidyenc_linux-amd64.sh - - - name: "Build rapidyenc (Linux arm64 cross-compile)" - if: matrix.os == 'linux' && matrix.arch == 'arm64' && matrix.cross_compile - timeout-minutes: 5 - run: | - cd rapidyenc && ./build_rapidyenc_linux-arm64.sh - echo "CC=aarch64-linux-gnu-gcc" >> $GITHUB_ENV - echo "CXX=aarch64-linux-gnu-g++" >> $GITHUB_ENV - echo "CGO_ENABLED=1" >> $GITHUB_ENV - echo "GOARCH=arm64" >> $GITHUB_ENV - - - name: "Build rapidyenc (Windows amd64 cross-compile)" - if: matrix.os == 'windows' && matrix.arch == 'amd64' - timeout-minutes: 5 - run: | - cd rapidyenc && ./crossbuild_rapidyenc_windows-amd64.sh - echo "CC=x86_64-w64-mingw32-gcc" >> $GITHUB_ENV - echo "CGO_ENABLED=1" >> $GITHUB_ENV - echo "GOOS=windows" >> $GITHUB_ENV - - - name: "Build rapidyenc (macOS)" - if: matrix.os == 'darwin' - timeout-minutes: 5 - run: | - cd rapidyenc - if [ "${{ matrix.arch }}" = "amd64" ]; then - ./crossbuild_rapidyenc_darwin-amd64.sh || ./build_rapidyenc_linux-amd64.sh darwin - else if [ "${{ matrix.arch }}" = "arm64" ]; then - ./build_rapidyenc_linux-arm64.sh darwin - fi - - # Run tests with race detector for supported platforms - - name: "Run race detector tests (amd64/linux only)" - if: matrix.os == 'linux' && matrix.arch == 'amd64' && !matrix.cross_compile - timeout-minutes: 5 - run: go test -race ./rapidyenc/ - - - name: "Run normal tests (non-cross-compile platforms)" - if: ${{ !(matrix.cross_compile) && !(matrix.os == 'linux' && matrix.arch == 'amd64') }} - timeout-minutes: 5 - run: go test ./rapidyenc/ - - - name: "Skip tests for cross-compiled platforms" - if: matrix.cross_compile - run: echo "Skipping tests for cross-compiled platform ${{ matrix.name }} (requires emulation or native hardware)" - - - name: "Test rapidyenc integration" - if: ${{ !(matrix.cross_compile) }} - timeout-minutes: 2 - run: | - # Build first if not cross-compiling - go build -o NZBreX -tags other . - ./NZBreX -testrapidyenc - name: "Set GaRuS Variables" id: vars @@ -392,7 +305,7 @@ jobs: cat builds/${{ matrix.name }}/usr/bin/$binary_name.sha512sum fi # - # packing the build + # packing the build (only create zip and deb) # # .zip pwd && ls -lha && echo ".zip Packing builds/${{ matrix.name }}/usr/bin/$binary_name" @@ -532,8 +445,8 @@ jobs: prerelease: ${{ contains(env.VERSION, 'test') || contains(env.VERSION, 'beta') || contains(env.VERSION, 'alpha') }} files: | NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.deb - NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.tgz - NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.tar.xz + #NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.tgz + #NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.tar.xz NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.zip NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha256sum NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha512sum From 70f510f21391a79e5f0ee16ce3ac1a2b9eb92e97 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 13:54:19 +0000 Subject: [PATCH 093/152] test build --- .github/workflows/build_and_publish_test.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 0c7c155..72b315d 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -205,6 +205,17 @@ jobs: cd rapidyenc && ./build_rapidyenc_linux-amd64.sh # Note: May need Alpine-specific build script + - name: "Build rapidyenc (macOS)" + if: matrix.os == 'darwin' + timeout-minutes: 5 + run: | + cd rapidyenc + if [ "${{ matrix.arch }}" = "amd64" ]; then + ./crossbuild_rapidyenc_darwin-amd64.sh || ./build_rapidyenc_linux-amd64.sh darwin + else if [ "${{ matrix.arch }}" = "arm64" ]; then + ./build_rapidyenc_linux-arm64.sh darwin + fi + # Run tests with race detector for supported platforms - name: "Run race detector tests (amd64/linux only)" if: matrix.os == 'linux' && matrix.arch == 'amd64' && !matrix.cross_compile @@ -305,7 +316,7 @@ jobs: cat builds/${{ matrix.name }}/usr/bin/$binary_name.sha512sum fi # - # packing the build (only create zip and deb) + # packing the build (only create zip and deb for now) # # .zip pwd && ls -lha && echo ".zip Packing builds/${{ matrix.name }}/usr/bin/$binary_name" From 793ace7e8d7bdb90ee884db68b07ce1f1ad667e8 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 14:00:25 +0000 Subject: [PATCH 094/152] test build --- .github/workflows/build_and_publish_test.yml | 3 ++- rapidyenc/build_rapidyenc_linux-amd64.sh | 1 + rapidyenc/build_rapidyenc_linux-arm64.sh | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 72b315d..cfc6595 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -212,7 +212,8 @@ jobs: cd rapidyenc if [ "${{ matrix.arch }}" = "amd64" ]; then ./crossbuild_rapidyenc_darwin-amd64.sh || ./build_rapidyenc_linux-amd64.sh darwin - else if [ "${{ matrix.arch }}" = "arm64" ]; then + fi + if [ "${{ matrix.arch }}" = "arm64" ]; then ./build_rapidyenc_linux-arm64.sh darwin fi diff --git a/rapidyenc/build_rapidyenc_linux-amd64.sh b/rapidyenc/build_rapidyenc_linux-amd64.sh index f68cd88..c115337 100755 --- a/rapidyenc/build_rapidyenc_linux-amd64.sh +++ b/rapidyenc/build_rapidyenc_linux-amd64.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +echo "run: $0 $1" cd rapidyenc || exit 2 rm -rf build mkdir -p build diff --git a/rapidyenc/build_rapidyenc_linux-arm64.sh b/rapidyenc/build_rapidyenc_linux-arm64.sh index 2fc9bf7..14d6968 100755 --- a/rapidyenc/build_rapidyenc_linux-arm64.sh +++ b/rapidyenc/build_rapidyenc_linux-arm64.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +echo "run: $0 $1" cd rapidyenc || exit 2 rm -rf build mkdir -p build From e9930cc0390227f8336da935be03062f579a080f Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 14:03:35 +0000 Subject: [PATCH 095/152] test mac --- rapidyenc/decoder.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rapidyenc/decoder.go b/rapidyenc/decoder.go index ac72073..a2a3fb1 100644 --- a/rapidyenc/decoder.go +++ b/rapidyenc/decoder.go @@ -26,6 +26,8 @@ import ( "golang.org/x/text/transform" ) +/* compile test ... github actions is using cached files... */ + const constBufSize = 4096 // const buffer size for Decoder instances var ( From d9d986813dc837656a5d66e22fefd1bba292cc2b Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 14:07:38 +0000 Subject: [PATCH 096/152] will darwin arm64 compile with amd64? --- .github/workflows/build_and_publish_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index cfc6595..0182e2f 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -214,7 +214,7 @@ jobs: ./crossbuild_rapidyenc_darwin-amd64.sh || ./build_rapidyenc_linux-amd64.sh darwin fi if [ "${{ matrix.arch }}" = "arm64" ]; then - ./build_rapidyenc_linux-arm64.sh darwin + ./build_rapidyenc_linux-arm64.sh darwin || ./build_rapidyenc_linux-amd64.sh darwin fi # Run tests with race detector for supported platforms From 6090a5d30e681875ce2a4205da9c1f612f361224 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 14:09:53 +0000 Subject: [PATCH 097/152] try fix filenames --- rapidyenc/build_rapidyenc_linux-amd64.sh | 4 ++-- rapidyenc/build_rapidyenc_linux-arm64.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rapidyenc/build_rapidyenc_linux-amd64.sh b/rapidyenc/build_rapidyenc_linux-amd64.sh index c115337..cc46140 100755 --- a/rapidyenc/build_rapidyenc_linux-amd64.sh +++ b/rapidyenc/build_rapidyenc_linux-amd64.sh @@ -9,8 +9,8 @@ cmake --build . --config Release || exit 5 ls . rapidyenc_static/ cd ../../ || exit 6 if [ "$1" = "darwin" ]; then - cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a ./librapidyenc_darwin_amd64.a || exit 7 - ln -sfv ./librapidyenc_darwin_amd64.a ./librapidyenc.a + cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a ./librapidyenc_darwin.a || exit 7 + ln -sfv ./librapidyenc_darwin.a ./librapidyenc.a else cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a ./librapidyenc_linux_amd64.a || exit 7 ln -sfv ./librapidyenc_linux_amd64.a ./librapidyenc.a diff --git a/rapidyenc/build_rapidyenc_linux-arm64.sh b/rapidyenc/build_rapidyenc_linux-arm64.sh index 14d6968..d9b03f7 100755 --- a/rapidyenc/build_rapidyenc_linux-arm64.sh +++ b/rapidyenc/build_rapidyenc_linux-arm64.sh @@ -9,8 +9,8 @@ cmake --build . --config Release || exit 5 ls . rapidyenc_static/ cd ../../ || exit 6 if [ "$1" = "darwin" ]; then - cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a ./librapidyenc_darwin_arm64.a || exit 7 - ln -sfv ./librapidyenc_darwin_arm64.a ./librapidyenc.a + cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a ./librapidyenc_darwin.a || exit 7 + ln -sfv ./librapidyenc_darwin.a ./librapidyenc.a else cp -v rapidyenc/build/rapidyenc_static/librapidyenc.a ./librapidyenc_linux_arm64.a || exit 7 ln -sfv ./librapidyenc_linux_arm64.a ./librapidyenc.a From 4b3ca150a344f7e6c5c7665560fdc32ec229bb8c Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 14:12:56 +0000 Subject: [PATCH 098/152] explain error --- .github/workflows/build_and_publish_test.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 0182e2f..560aea2 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -210,12 +210,9 @@ jobs: timeout-minutes: 5 run: | cd rapidyenc - if [ "${{ matrix.arch }}" = "amd64" ]; then - ./crossbuild_rapidyenc_darwin-amd64.sh || ./build_rapidyenc_linux-amd64.sh darwin - fi - if [ "${{ matrix.arch }}" = "arm64" ]; then - ./build_rapidyenc_linux-arm64.sh darwin || ./build_rapidyenc_linux-amd64.sh darwin - fi + # macOS builds natively - the build_rapidyenc_linux-amd64.sh script + # is actually a native build script (despite the misleading name) + ./build_rapidyenc_linux-amd64.sh darwin # Run tests with race detector for supported platforms - name: "Run race detector tests (amd64/linux only)" From fda4b9a6253c1380b83fa1877a7c2c0411c5b9a5 Mon Sep 17 00:00:00 2001 From: go-while Date: Wed, 15 Oct 2025 16:49:34 +0000 Subject: [PATCH 099/152] fail quick on errors --- Workers.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Workers.go b/Workers.go index 134e247..cca0642 100644 --- a/Workers.go +++ b/Workers.go @@ -197,10 +197,8 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro code, err := s.GoCheckRoutine(wid, provider, item, sharedCC) item.PrintItemFlags(cfg.opt.DebugFlags, true, fmt.Sprintf("post-GoCheckRoutine: code=%d", code)) if err != nil { // re-queue? - dlog(always, "ERROR in GoCheckRoutine err='%v' flag retry item", err) - go func(item *segmentChanItem) { - segCC <- item // re-queue the item for checking - }(item) + dlog(always, "ERROR in GoCheckRoutine err='%v'", err) + item.FlagError(provider.id) } case true: log.Fatal("you should not be here! Quitting...") // FIXME TODO: remove this fatal error From 5844e8c4d9d54e196f429dede69d109b5e66dd5c Mon Sep 17 00:00:00 2001 From: go-while Date: Thu, 16 Oct 2025 00:33:15 +0000 Subject: [PATCH 100/152] fix connection problems --- Config.go | 6 ++-- ConnPool.go | 87 ++++++++++++++++++----------------------------------- Session.go | 13 +++++--- Workers.go | 77 ++++++++++++++++++----------------------------- 4 files changed, 69 insertions(+), 114 deletions(-) diff --git a/Config.go b/Config.go index edbd10a..98b28c1 100644 --- a/Config.go +++ b/Config.go @@ -7,7 +7,7 @@ import ( "github.com/go-while/go-loggedrwmutex" ) -const UseSharedCC = false // experimental devel flag, to test sharedConn between routines +const UseSharedCC = true // experimental devel flag, to test sharedConn between routines const UseReadDeadConn = false // experimental devel flag, to test reading from dead connections const UseNoDeadline = true // experimental devel flag, to test reading from dead connections @@ -39,9 +39,9 @@ const ( // DefaultConnectTimeout defines the timeout for connecting to a server DefaultConnectTimeout = 9 * time.Second // DefaultConnectErrSleep defines the time to wait before retrying a connection after an error - DefaultConnectErrSleep = 9 * time.Second + DefaultConnectErrSleep = 3 * time.Second // DefaultRequeueDelay defines the delay before requeuing an item in the segment channel - DefaultRequeueDelay = 9 * time.Second + //DefaultRequeueDelay = 9 * time.Second ) type Config struct { diff --git a/ConnPool.go b/ConnPool.go index 43c0a97..9bf3764 100644 --- a/ConnPool.go +++ b/ConnPool.go @@ -208,8 +208,7 @@ func (c *ConnPool) connect() (connitem *ConnItem, err error) { if isNetworkUnreachable(err) { return nil, fmt.Errorf("error connect Unreachable network! '%s' @ '%s' err='%v'", c.provider.Host, c.provider.Name, err) } - dlog(always, "ERROR connect Dial '%s' retry in %.0fs wants_ssl=%t err='%v'", c.provider.Name, DefaultConnectErrSleep.Seconds(), c.provider.SSL, err) - //time.Sleep(DefaultConnectErrSleep) + dlog(always, "ERROR connect Dial '%s' no retry err='%v'", c.provider.Name, err) return nil, err } case true: @@ -225,8 +224,7 @@ func (c *ConnPool) connect() (connitem *ConnItem, err error) { if isNetworkUnreachable(err) { return nil, fmt.Errorf("error connect Unreachable network! '%s' @ '%s' err='%v'", c.provider.Host, c.provider.Name, err) } - dlog(always, "ERROR connect Dial '%s' retry in %.0fs wants_ssl=%t err='%v'", c.provider.Name, DefaultConnectErrSleep.Seconds(), c.provider.SSL, err) - //time.Sleep(DefaultConnectErrSleep) + dlog(always, "ERROR connect Dial '%s' no-retry err='%v'", c.provider.Name, err) return nil, err } } // end switch @@ -244,8 +242,7 @@ func (c *ConnPool) connect() (connitem *ConnItem, err error) { if conn != nil { conn.Close() } - dlog(always, "ERROR connect welcome '%s' retry in %.0fs code=%d msg='%s' err='%v'", c.provider.Name, DefaultConnectErrSleep.Seconds(), code, msg, err) - //time.Sleep(DefaultConnectErrSleep) + dlog(always, "ERROR in connect '%s' welcome banner code=%d err='%v'", c.provider.Name, code, err) return nil, err } dlog(cfg.opt.DebugConnPool, "ConnPool connect welcome '%s' time0=(%d ms) time00=(%d ms) time000=(%d ms)", c.provider.Name, time.Since(time0).Milliseconds(), time.Since(time00).Milliseconds(), time.Since(time000).Milliseconds()) @@ -256,8 +253,8 @@ func (c *ConnPool) connect() (connitem *ConnItem, err error) { } // end func connect func (c *ConnPool) auth(srvtp *textproto.Conn, conn net.Conn, start time.Time) (connitem *ConnItem, err error) { - var code int - var msg string + //var code int + //var msg string time1 := time.Now() // send auth user sequence id, err := srvtp.Cmd("AUTHINFO USER %s", c.provider.Username) @@ -265,21 +262,19 @@ func (c *ConnPool) auth(srvtp *textproto.Conn, conn net.Conn, start time.Time) ( if conn != nil { conn.Close() } - dlog(always, "ERROR AUTH#1 Cmd(AUTHINFO USER ...) '%s' retry in %.0fs err='%v' ", c.provider.Name, DefaultConnectErrSleep.Seconds(), err) - time.Sleep(DefaultConnectErrSleep) + //dlog(always, "ERROR AUTH#1 Cmd(AUTHINFO USER ...) '%s' no retry err='%v' ", c.provider.Name, err) return nil, err } time2 := time.Now() // await response from server srvtp.StartResponse(id) - code, msg, err = srvtp.ReadCodeLine(nntpMoreInfoCode) // 381 is the code for "more information required" + _, _, err = srvtp.ReadCodeLine(nntpMoreInfoCode) // 381 is the code for "more information required" srvtp.EndResponse(id) if err != nil { if conn != nil { conn.Close() } - dlog(always, "ERROR AUTH#2 ReadCodeLine(381) step#2 '%s' retry in %.0fs code=%d msg='%s' err='%v'", c.provider.Name, DefaultConnectErrSleep.Seconds(), code, msg, err) - time.Sleep(DefaultConnectErrSleep) + //dlog(always, "ERROR AUTH#2 ReadCodeLine(381) step#2 '%s' code=%d msg='%s' err='%v'", c.provider.Name, code, msg, err) return nil, err } time3 := time.Now() @@ -289,21 +284,19 @@ func (c *ConnPool) auth(srvtp *textproto.Conn, conn net.Conn, start time.Time) ( if conn != nil { conn.Close() } - dlog(always, "ERROR AUTH#3 Cmd(AUTHINFO PASS ...) '%s' retry in %.0fs err='%v'", c.provider.Name, DefaultConnectErrSleep.Seconds(), err) - time.Sleep(DefaultConnectErrSleep) + //dlog(always, "ERROR AUTH#3 Cmd(AUTHINFO PASS ...) '%s' no retry err='%v'", c.provider.Name, err) return nil, err } time4 := time.Now() // await response from server srvtp.StartResponse(id) - code, msg, err = srvtp.ReadCodeLine(nntpAuthSuccess) // 281 is the code for "authentication successful" + _, _, err = srvtp.ReadCodeLine(nntpAuthSuccess) // 281 is the code for "authentication successful" srvtp.EndResponse(id) if err != nil { if conn != nil { conn.Close() } - dlog(always, "ERROR AUTH#4 ReadCodeLine(281) '%s' retry in %.0fs code=%d msg='%s' err='%v'", c.provider.Name, DefaultConnectErrSleep.Seconds(), code, msg, err) - time.Sleep(DefaultConnectErrSleep) + //dlog(always, "ERROR AUTH#4 ReadCodeLine(281) '%s' no retry err='%v'", c.provider.Name, err) return nil, err } time5 := time.Now() @@ -329,38 +322,20 @@ func (c *ConnPool) NewConn() (connitem *ConnItem, err error) { c.mux.Unlock() // mutex c459a start := time.Now() // connect to the provider - if c.provider.MaxConnErrors <= 0 { - for { - connitem, err = c.connect() - if err != nil || connitem == nil || connitem.conn == nil { - continue // retry connect - } - c.mux.RLock() // mutex c461 - dlog(cfg.opt.DebugConnPool, "ConnPool NewConn: got new connid=%d '%s' openConns after=%d/%d before=%d connectTook=(%d ms) err='%v", connitem.connid, c.provider.Name, c.openConns, c.provider.MaxConns, openConnsBefore, time.Since(start).Milliseconds(), err) - c.mux.RUnlock() // mutex c461 - // we have a new connection! - GCounter.Incr("TOTAL_NewConns") - return // established new connection and returns connitem - } - } else { - for retried := 0; retried < c.provider.MaxConnErrors; retried++ { - connitem, err = c.connect() - if err != nil || connitem == nil || connitem.conn == nil { - continue // retry connect - } - c.mux.RLock() // mutex c461 - dlog(cfg.opt.DebugConnPool, "ConnPool NewConn: got new connid=%d '%s' openConns after=%d/%d before=%d connectTook=(%d ms) err='%v", connitem.connid, c.provider.Name, c.openConns, c.provider.MaxConns, openConnsBefore, time.Since(start).Milliseconds(), err) - c.mux.RUnlock() // mutex c461 - // we have a new connection! - GCounter.Incr("TOTAL_NewConns") - return connitem, nil // established new connection and returns connitem - } + connitem, err = c.connect() + if err != nil || connitem == nil || connitem.conn == nil { + c.mux.Lock() // mutex c460 + c.openConns-- + dlog(always, "ERROR in ConnPool NewConn: connect failed '%s' openConns=%d connitem='%v' err='%v'", c.provider.Name, c.openConns, connitem, err) + c.mux.Unlock() // mutex c460 + return nil, fmt.Errorf("error in ConnPool NewConn: connect failed '%s' err='%v'", c.provider.Name, err) } - c.mux.Lock() // mutex c460 - c.openConns-- - dlog(always, "ERROR in ConnPool NewConn: connect failed '%s' openConns=%d connitem='%v' err='%v'", c.provider.Name, c.openConns, connitem, err) - c.mux.Unlock() // mutex c460 - return nil, fmt.Errorf("error in ConnPool NewConn: connect failed '%s' err='%v'", c.provider.Name, err) + c.mux.RLock() // mutex c461 + dlog(cfg.opt.DebugConnPool, "ConnPool NewConn: got new connid=%d '%s' openConns after=%d/%d before=%d connectTook=(%d ms) err='%v", connitem.connid, c.provider.Name, c.openConns, c.provider.MaxConns, openConnsBefore, time.Since(start).Milliseconds(), err) + c.mux.RUnlock() // mutex c461 + // we have a new connection! + GCounter.Incr("TOTAL_NewConns") + return connitem, nil // established new connection and returns connitem } c.mux.Unlock() // mutex c459a return nil, fmt.Errorf("error in ConnPool NewConn: openConns=%d > provider.MaxConns", c.openConns) @@ -419,7 +394,7 @@ getConnFromPool: dlog(always, "INFO ConnPool GetConn: got long idle=(%d ms) '%s', close and get NewConn", time.Since(connitem.parktime).Milliseconds(), c.provider.Name) c.CloseConn(connitem, nil) connitem, err = c.NewConn() - if connitem == nil || err != nil { + if err != nil || connitem == nil || connitem.conn == nil { continue getConnFromPool } // extend the read deadline of the new connection @@ -466,16 +441,13 @@ getConnFromPool: // try to open a new connection newconnitem, aerr := c.NewConn() - if newconnitem == nil || aerr != nil { + if aerr != nil || newconnitem == nil || newconnitem.conn == nil { dlog(cfg.opt.DebugConnPool, "WARN in ConnPool GetConn: NewConn failed '%s' connitem='%v' aerr='%v'", c.provider.Name, newconnitem, aerr) time.Sleep(DefaultConnectErrSleep) // wait a bit before retrying retried++ if retried >= c.provider.MaxConnErrors { // we have retried too often - dlog(always, "ERROR in ConnPool GetConn: retried %d times to get a new conn '%s' giving up! aerr='%v'", retried, c.provider.Name, aerr) - //c.mux.Lock() // mutex c457 - //c.openConns-- // decrease open conns as we failed to get a new one - //c.mux.Unlock() // mutex c457 + dlog(cfg.opt.DebugConnPool, "ERROR in ConnPool GetConn: retried %d times to get a new conn '%s' giving up! aerr='%v'", retried, c.provider.Name, aerr) err = aerr break getConnFromPool // break out of the for loop } @@ -729,13 +701,12 @@ func isNetworkUnreachable(err error) bool { // The sharedCC channel is used to share connections between (anonymous) goroutines // which will work on the same item. func GetNewSharedConnChannel(wid int, provider *Provider) (sharedCC chan *ConnItem, err error) { - sharedCC = make(chan *ConnItem, 1) // buffered channel with size 1 connitem, err := provider.ConnPool.GetConn() // get an idle or new connection from the pool if err != nil { - dlog(always, "ERROR a GoWorker (%d) failed to connect '%s' err='%v'", wid, provider.Name, err) + dlog(cfg.opt.DebugWorker, "ERROR in GoWorker (%d) GetNewSharedConnChannel failed to connect '%s' err='%v'", wid, provider.Name, err) return nil, err } - //sharedCC <- nil // put a nil item into the channel to signal that no connection is available yet + sharedCC = make(chan *ConnItem, 1) // shares 1 connection between goroutines sharedCC <- connitem return sharedCC, nil } diff --git a/Session.go b/Session.go index 8464550..b2a7d10 100644 --- a/Session.go +++ b/Session.go @@ -367,17 +367,20 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s s.segmentCheckStartTime = time.Now() // booting work divider go s.GoWorkDivider(&waitDivider, &waitDividerDone) - dlog(cfg.opt.Debug, "sess: waitDividerDone.Wait()") + dlog(always, "sess: waiting on waitDividerDone.Wait()") waitDividerDone.Wait() - dlog(cfg.opt.Debug, "sess: waitDividerDone.Wait() released") + dlog(always, "sess: released waitDividerDone.Wait()") s.StopRoutines() - dlog(cfg.opt.Debug, "sess: waitWorker.Wait()") + dlog(always, "sess: waiting on waitWorker.Wait()") waitWorker.Wait() - dlog(cfg.opt.Debug, "sess: waitWorker.Wait() released, waiting on waitPool.Wait()") + dlog(always, "sess: released waitWorker.Wait(), waiting on waitPool.Wait()") waitPool.Wait() - dlog(cfg.opt.Debug, "sess: waitPool.Wait() released") + dlog(always, "sess: released waitPool.Wait() closing all provider connections") + for _, provider := range s.providerList { + KillConnPool(provider) // close the connection pool for this provider + } result, runtime_info := s.Results(s.preparationStartTime) diff --git a/Workers.go b/Workers.go index cca0642..6e3e5a7 100644 --- a/Workers.go +++ b/Workers.go @@ -146,7 +146,7 @@ func (s *SESSION) GoBootWorkers(waitDivider *sync.WaitGroup, workerWGconnReady * dlog(cfg.opt.Debug, "GoBootWorkers PreBoot Provider '%s' launch wid=%d/%d", provider.Name, wid, provider.MaxConns) } // GoWorker connecting.... - go s.GoWorker(wid, provider, waitWorker, workerWGconnReady, waitPool) + go s.GoWorker(wid, provider, waitWorker, waitPool) time.Sleep(time.Duration(rand.Intn(50*provider.MaxConns)) * time.Millisecond) // random sleep to avoid all workers connecting at the same time } @@ -158,7 +158,7 @@ func (s *SESSION) GoBootWorkers(waitDivider *sync.WaitGroup, workerWGconnReady * dlog(cfg.opt.DebugWorker, "GoBootWorkers: all workers connected (or died if no conn could be established)") } // end func GoBootWorkers -func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGroup, workerWGconnReady *sync.WaitGroup, waitPool *sync.WaitGroup) { +func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGroup, waitPool *sync.WaitGroup) { dlog(cfg.opt.DebugWorker, "GoWorker (%d) launching routines '%s'", wid, provider.Name) // Obtain a connection for this worker and share it among the check, download, and reupload routines. @@ -166,10 +166,17 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro var sharedCC chan *ConnItem = nil var err error if UseSharedCC { - // Get a shared connection channel for the provider} + dlog(cfg.opt.DebugWorker, "GoWorker (%d) trying to get shared connection channel for '%s'", wid, provider.Name) + // Get a shared connection channel for the provider pre filled with an established connection sharedCC, err = GetNewSharedConnChannel(wid, provider) if err != nil || len(sharedCC) == 0 { - dlog(always, "ERROR GoWorker (%d) failed to get shared connection channel for '%s' err='%v'", wid, provider.Name, err) + // release for the 3 routines we won't start + waitWorker.Done() + waitWorker.Done() + waitWorker.Done() + // release this GoWorker from the pool + waitPool.Done() + dlog(cfg.opt.DebugWorker, "ERROR GoWorker (%d) failed to get shared connection channel for '%s' err='%v'", wid, provider.Name, err) return } } @@ -185,9 +192,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro return // channel is closed, exit the routine } if item == nil { - if cfg.opt.DebugWorker { - log.Print("CheckRoutine received a nil pointer to quit") - } + dlog(cfg.opt.DebugWorker, "CheckRoutine received a nil pointer to quit") segCC <- nil // refill the nil so others will die too break forGoCheckRoutine } @@ -202,20 +207,6 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro } case true: log.Fatal("you should not be here! Quitting...") // FIXME TODO: remove this fatal error - /* - item.mux.Lock() - item.flaginDL = true - - //if !item.pushedDL { - IncreaseDLQueueCnt() // cfg.opt.ByPassSTAT - //GCounter.IncrMax("dlQueueCnt", uint64(len(s.segmentList)), "CheckRoutine:ByPassSTAT") // cfg.opt.ByPassSTAT - //GCounter.IncrMax("TOTAL_dlQueueCnt", uint64(len(s.segmentList)), "CheckRoutine:ByPassSTAT") //cfg.opt.ByPassSTAT - //} - item.pushedDL++ // mark as pushed to download queue ByPassSTAT - item.mux.Unlock() - // ! TODO FIXME : use s.WorkDividerChan ? - s.segmentChansDowns[provider.Group] <- item // bypass STAT: GoCheckRoutine push to download queue - */ } continue forGoCheckRoutine } // end forGoCheckRoutine @@ -234,9 +225,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro return // channel is closed, exit the routine } if item == nil { - if cfg.opt.DebugWorker { - log.Print("DownsRoutine received a nil pointer to quit") - } + dlog(cfg.opt.DebugWorker, "DownsRoutine received a nil pointer to quit") segCD <- nil // refill the nil so others will die too break forGoDownsRoutine } @@ -294,9 +283,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro return // channel is closed, exit the routine } if item == nil { - if cfg.opt.DebugWorker { - log.Print("ReupsRoutine received a nil pointer to quit") - } + dlog(cfg.opt.DebugWorker, "ReupsRoutine received a nil pointer to quit") s.segmentChansReups[provider.Group] <- nil // refill the nil so others will die too break forGoReupsRoutine } @@ -337,7 +324,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro dlog(cfg.opt.DebugWorker, "GoWorker (%d) waitWorker.Wait for routines to complete '%s'", wid, provider.Name) waitWorker.Wait() // wait for all 3 routines to finish - dlog(cfg.opt.BUG, "GoWorker (%d) closing @ '%s'", wid, provider.Name) + dlog(cfg.opt.DebugWorker, "GoWorker (%d) closing @ '%s'", wid, provider.Name) if sharedCC != nil { select { @@ -353,8 +340,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro } } - KillConnPool(provider) // close the connection pool for this provider - waitPool.Done() // release this GoWorker from the pool + waitPool.Done() // release this GoWorker from the pool dlog(cfg.opt.BUG, "GoWorker (%d) quit @ '%s'", wid, provider.Name) } // end func GoWorker @@ -595,15 +581,10 @@ type WrappedItem struct { const testing = false // set to true to test the worker loop // GoWorkDivider is the main worker loop that processes segments func (s *SESSION) GoWorkDivider(waitDivider *sync.WaitGroup, waitDividerDone *sync.WaitGroup) { - if cfg.opt.DebugWorker { - log.Print("go GoWorkDivider() waitDivider.Wait()") - } + dlog(cfg.opt.DebugWorker, "go GoWorkDivider() waiting on waitDivider.Wait()") waitDivider.Wait() // waits for all conns to establish defer waitDividerDone.Done() - - if cfg.opt.DebugWorker { - log.Print("go GoWorkDivider() Starting!") - } + dlog(cfg.opt.DebugWorker, "go GoWorkDivider() Starting!") //segcheckdone := false closeWait, closeCase := 1, "" @@ -857,7 +838,7 @@ forever: logstring = log00 + log01 + log02 + log03 + log04 + log05 + log06 + log07 + log08 + log09 + log10 + log11 + log99 if cfg.opt.Verbose && cfg.opt.PrintStats >= 0 && logstring != "" && nextLogPrint < time.Now().Unix() { nextLogPrint = time.Now().Unix() + cfg.opt.PrintStats - log.Print(logstring) + dlog(always, "%s", logstring) } } // print some stats ends here @@ -873,7 +854,7 @@ forever: s.segmentCheckTook = took s.segcheckdone = true s.mux.Unlock() - dlog(cfg.opt.Verbose, " | [DV] | Segment Check Done: took='%.1f sec'", took.Seconds()) + dlog(cfg.opt.Verbose, " | Segment Check Done: took='%.1f sec'", took.Seconds()) } // continue as long as any of this triggers because stuff is still in queues and processing @@ -905,35 +886,35 @@ forever: if closeCase0 { closeWait-- - closeCase = closeCase + "|Debug#0@" + fmt.Sprintf("%d", time.Now().Unix()) + closeCase = closeCase + "|Debug#0" } else if closeCase1 { closeWait-- - closeCase = closeCase + "|Debug#1@" + fmt.Sprintf("%d", time.Now().Unix()) + closeCase = closeCase + "|Debug#1" } else if closeCase2 { closeWait-- - closeCase = closeCase + "|Debug#2@" + fmt.Sprintf("%d", time.Now().Unix()) + closeCase = closeCase + "|Debug#2" } else if closeCase3 { closeWait-- - closeCase = closeCase + "|Debug#3@" + fmt.Sprintf("%d", time.Now().Unix()) + closeCase = closeCase + "|Debug#3" } else if closeCase4 { closeWait-- - closeCase = closeCase + "|Debug#4@" + fmt.Sprintf("%d", time.Now().Unix()) + closeCase = closeCase + "|Debug#4" } else if closeCase5 { closeWait-- - closeCase = closeCase + "|Debug#5@" + fmt.Sprintf("%d", time.Now().Unix()) + closeCase = closeCase + "|Debug#5" } else if closeCase6 { closeWait-- - closeCase = closeCase + "|Debug#6@" + fmt.Sprintf("%d", time.Now().Unix()) + closeCase = closeCase + "|Debug#6" } else if closeCase7 { closeWait-- - closeCase = closeCase + "|Debug#7@" + fmt.Sprintf("%d", time.Now().Unix()) + closeCase = closeCase + "|Debug#7" } else { if closeWait >= 10 { @@ -944,7 +925,7 @@ forever: } closeWait++ if closeWait >= 15 { - log.Print("... force quit ...") + dlog(always, "... force quit ...") return } closeCase = "" From 44c2ef70b38dbc17c7e987d0043836342dcb6240 Mon Sep 17 00:00:00 2001 From: go-while Date: Thu, 16 Oct 2025 00:50:28 +0000 Subject: [PATCH 101/152] test windows compilation --- .github/workflows/build_and_publish_test.yml | 5 ++++- local_crossbuild_windows-amd64.sh | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 560aea2..4eb6522 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -284,11 +284,14 @@ jobs: mkdir -p builds/${{ matrix.name }}/usr/bin if [ "${{ matrix.os }}" == "windows" ]; then binary_name=NZBreX.exe + # For Windows: use static linking to avoid DLL dependencies + LDFLAGS="-X main.appVersion=${{ env.VERSION }} -linkmode external -extldflags '-static -static-libgcc -static-libstdc++'" else binary_name=NZBreX + LDFLAGS="-X main.appVersion=${{ env.VERSION }}" fi echo "BINARY=$binary_name" >> $GITHUB_ENV - GOARCH=${{ matrix.arch }} GOOS=${{ matrix.os }} go build -ldflags="-s -w -X main.appVersion=${{ env.VERSION }}" -o builds/${{ matrix.name }}/usr/bin/$binary_name + GOARCH=${{ matrix.arch }} GOOS=${{ matrix.os }} go build -ldflags="$LDFLAGS" -o builds/${{ matrix.name }}/usr/bin/$binary_name if [ ! -f cleanHeaders.txt ]; then echo "cleanHeaders.txt not found! Build cannot continue." >&2 exit 1 diff --git a/local_crossbuild_windows-amd64.sh b/local_crossbuild_windows-amd64.sh index 04e3f67..5116874 100755 --- a/local_crossbuild_windows-amd64.sh +++ b/local_crossbuild_windows-amd64.sh @@ -49,7 +49,10 @@ export CC=x86_64-w64-mingw32-gcc echo "Cross-compiling Go application for Windows..." # Build with static linking to avoid dependency on MinGW runtime DLLs -if ! go build -o NZBreX.exe -ldflags "-linkmode external -extldflags '-static'" -tags "windows rapidyenc" .; then +# -static: Link all libraries statically (including libgcc, libstdc++, winpthread) +# -static-libgcc: Static link libgcc only +# -static-libstdc++: Static link libstdc++ only +if ! go build -o NZBreX.exe -ldflags "-linkmode external -extldflags '-static -static-libgcc -static-libstdc++'" -tags "windows rapidyenc" .; then echo "ERROR: Failed to build Windows executable" exit 1 fi From 8cb4933ca9eaccf395b9469dea1c8e54a4eed570 Mon Sep 17 00:00:00 2001 From: go-while Date: Fri, 17 Oct 2025 01:40:26 +0000 Subject: [PATCH 102/152] flag errors --- Routines.go | 12 +++++++----- Workers.go | 3 --- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Routines.go b/Routines.go index 248e1b2..00b268c 100644 --- a/Routines.go +++ b/Routines.go @@ -175,10 +175,10 @@ func (s *SESSION) GoDownsRoutine(wid int, provider *Provider, item *segmentChanI startArticle := time.Now() code, msg, rxb, err := CMD(connitem, item, cmdARTICLE) if err != nil { - dlog(always, "ERROR in GoDownsRoutine: CMD_ARTICLE seg.Id='%s' @ '%s'#'%s' err='%v'", item.segment.Id, provider.Name, provider.Group, err) - // handle connection problem / closed connection + // connection problem, closed? item.FlagError(provider.id) provider.ConnPool.CloseConn(connitem, sharedCC) // close conn on error + dlog(always, "ERROR in GoDownsRoutine: CMD_ARTICLE seg.Id='%s' @ '%s'#'%s' err='%v'", item.segment.Id, provider.Name, provider.Group, err) return 0, fmt.Errorf("error in GoDownsRoutine: CMD_ARTICLE seg.Id='%s' @ '%s'#'%s' err='%v'", item.segment.Id, provider.Name, provider.Group, err) } @@ -351,9 +351,10 @@ func (s *SESSION) GoReupsRoutine(wid int, provider *Provider, item *segmentChanI } else if provider.capabilities.ihave { cmd = 2 } else { - //provider.mux.RUnlock() // FIXME TODO #b8bd287b: + // connection problem, closed? + item.FlagError(provider.id) provider.ConnPool.CloseConn(connitem, sharedCC) // close conn on error - return 0, fmt.Errorf("WARN selecting upload mode failed '%s' caps='%#v'", provider.Name, provider.capabilities) + return 0, fmt.Errorf("selecting upload mode failed '%s' caps='%#v'", provider.Name, provider.capabilities) } //provider.mux.RUnlock() // FIXME TODO #b8bd287b: @@ -499,9 +500,10 @@ func (s *SESSION) GoReupsRoutine(wid int, provider *Provider, item *segmentChanI } if err != nil { - dlog(always, "ERROR in GoReupsRoutine: seg.Id='%s' @ '%s'#'%s' err='%v'", item.segment.Id, provider.Name, provider.Group, err) // handle connection problem / closed connection + item.FlagError(provider.id) provider.ConnPool.CloseConn(connitem, sharedCC) // close conn on error + dlog(always, "ERROR in GoReupsRoutine: seg.Id='%s' @ '%s'#'%s' err='%v'", item.segment.Id, provider.Name, provider.Group, err) return 0, fmt.Errorf("error in GoReupsRoutine: seg.Id='%s' @ '%s'#'%s' err='%v'", item.segment.Id, provider.Name, provider.Group, err) } diff --git a/Workers.go b/Workers.go index 6e3e5a7..58d3df6 100644 --- a/Workers.go +++ b/Workers.go @@ -203,7 +203,6 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro item.PrintItemFlags(cfg.opt.DebugFlags, true, fmt.Sprintf("post-GoCheckRoutine: code=%d", code)) if err != nil { // re-queue? dlog(always, "ERROR in GoCheckRoutine err='%v'", err) - item.FlagError(provider.id) } case true: log.Fatal("you should not be here! Quitting...") // FIXME TODO: remove this fatal error @@ -246,7 +245,6 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro // 430 is a normal error code for GoDownsRoutine, so we don't log it as an error errStr = fmt.Sprintf("ERROR in GoDownsRoutine code='%d' err='%v'. no retry", code, err) dlog(always, "%s", errStr) - item.FlagError(provider.id) // mark item with error for this provider } memlim.MemReturn("MemRetOnERR: "+errStr, item) // memfree GoDownsRoutine on error continue forGoDownsRoutine @@ -310,7 +308,6 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro errStr := fmt.Sprintf("ERROR in GoReupsRoutine code='%d' err='%v' no retry", code, err) dlog(always, "%s", errStr) memlim.MemReturn("MemRetOnERR: "+errStr, item) // memfree GoReupsRoutine on error - item.FlagError(provider.id) // mark item with error for this provider continue forGoReupsRoutine } speedInKBytes := (float64(item.size) / 1024) / float64(time.Since(StartReUps).Seconds()) From ed093db5eca2fb313b60b3303004b8424b362248 Mon Sep 17 00:00:00 2001 From: go-while Date: Fri, 17 Oct 2025 04:07:00 +0000 Subject: [PATCH 103/152] conn & memlim --- MemLimit.go | 21 ++++++++++-- Routines.go | 23 +++++++++---- Session.go | 2 +- Workers.go | 94 +++++++++++++++++++++++++++++++++-------------------- 4 files changed, 95 insertions(+), 45 deletions(-) diff --git a/MemLimit.go b/MemLimit.go index 90450a0..d09bf4b 100644 --- a/MemLimit.go +++ b/MemLimit.go @@ -71,6 +71,13 @@ func (m *MemLimiter) MemAvail() (retbool bool) { } func (m *MemLimiter) MemLockWait(item *segmentChanItem, who string) { + item.mux.Lock() + if item.memlocked > 0 { + item.mux.Unlock() + dlog(always, "MemLockWait called on already memlocked item seg.Id='%s' who='%s'", item.segment.Id, who) + return + } + item.mux.Unlock() GCounter.Incr("MemLockWait") defer GCounter.Decr("MemLockWait") @@ -102,6 +109,9 @@ func (m *MemLimiter) MemLockWait(item *segmentChanItem, who string) { } // end for waithere */ <-m.memchan // infinite wait to get a slot from chan + item.mux.Lock() + item.memlocked++ + item.mux.Unlock() m.mux.Lock() m.waiting-- @@ -115,7 +125,12 @@ func (m *MemLimiter) MemLockWait(item *segmentChanItem, who string) { func (m *MemLimiter) MemReturn(who string, item *segmentChanItem) { //dlog(cfg.opt.DebugMemlim, "MemReturn free seg.Id='%s' who='%s'", item.segment.Id, who) defer GCounter.Incr("TOTAL_MemReturned") - + item.mux.RLock() + if item.memlocked == 0 { + dlog(always, "MemReturn called on non-memlocked item seg.Id='%s' who='%s'", item.segment.Id, who) + return // not memlocked, nothing to do + } + item.mux.RUnlock() // remove map entry from mem m.mux.Lock() delete(m.memdata, item) @@ -130,6 +145,8 @@ func (m *MemLimiter) MemReturn(who string, item *segmentChanItem) { dlog(always, "ERROR on MemReturn chan is full seg.Id='%s' who='%s'", item.segment.Id, who) os.Exit(1) // this is a bug! we should never return a slot to a full chan! } - + item.mux.Lock() + item.memlocked-- + item.mux.Unlock() dlog(cfg.opt.DebugMemlim, "MemReturned seg.Id='%s' who='%s'", item.segment.Id, who) } // end func memlim.MemReturn diff --git a/Routines.go b/Routines.go index 00b268c..6dbdbac 100644 --- a/Routines.go +++ b/Routines.go @@ -40,10 +40,10 @@ func (s *SESSION) GoCheckRoutine(wid int, provider *Provider, item *segmentChanI connitem, err = provider.ConnPool.GetConn() } if err != nil { - return 0, fmt.Errorf("ERROR in GoCheckRoutine: ConnGet '%s' connitem='%v' sharedCC='%v' err='%v'", provider.Name, connitem, sharedCC, err) + return 0, fmt.Errorf("error in GoCheckRoutine: ConnGet '%s' connitem='%v' sharedCC='%v' err='%v'", provider.Name, connitem, sharedCC, err) } if connitem == nil || connitem.conn == nil { - return 0, fmt.Errorf("ERROR in GoCheckRoutine: ConnGet got nil item or conn '%s' connitem='%v' sharedCC='%v' err='%v'", provider.Name, connitem, sharedCC, err) + return 0, fmt.Errorf("error in GoCheckRoutine: ConnGet got nil item or conn '%s' connitem='%v' sharedCC='%v' err='%v'", provider.Name, connitem, sharedCC, err) } code, msg, err := CMD_STAT(connitem, item) @@ -165,10 +165,10 @@ func (s *SESSION) GoDownsRoutine(wid int, provider *Provider, item *segmentChanI connitem, err = provider.ConnPool.GetConn() } if err != nil { - return 0, fmt.Errorf("ERROR in GoDownsRoutine: ConnGet '%s' connitem='%v' sharedCC='%v' err='%v'", provider.Name, connitem, sharedCC, err) + return 0, fmt.Errorf("error in GoDownsRoutine: ConnGet '%s' connitem='%v' sharedCC='%v' err='%v'", provider.Name, connitem, sharedCC, err) } if connitem == nil || connitem.conn == nil { - return 0, fmt.Errorf("ERROR in GoDownsRoutine: ConnGet got nil item or conn '%s' connitem='%v' sharedCC='%v' err='%v'", provider.Name, connitem, sharedCC, err) + return 0, fmt.Errorf("error in GoDownsRoutine: ConnGet got nil item or conn '%s' connitem='%v' sharedCC='%v' err='%v'", provider.Name, connitem, sharedCC, err) } dlog(cfg.opt.DebugWorker, "GoDownsRoutine got connitem='%v' sharedCC='%v' --> CMD_ARTICLE seg.Id='%s'", connitem, sharedCC, item.segment.Id) @@ -326,6 +326,15 @@ func (s *SESSION) GoReupsRoutine(wid int, provider *Provider, item *segmentChanI defer GCounter.Decr("GoReupsRoutines") //who := fmt.Sprintf("UR=%d@'%s' seg.Id='%s'", wid, provider.Name, item.segment.Id) // DISABLED MEMRETURN + item.mux.Lock() + memlocked := item.memlocked > 0 + item.mux.Unlock() + + if !memlocked { + dlog(cfg.opt.DebugWorker, "GoReupsRoutine: item not memlocked seg.Id='%s' @ '%s'#'%s'", item.segment.Id, provider.Name, provider.Group) + memlim.MemLockWait(item, "GoUR") // gets memlock here + } + var err error var connitem *ConnItem if sharedCC != nil { @@ -334,10 +343,10 @@ func (s *SESSION) GoReupsRoutine(wid int, provider *Provider, item *segmentChanI connitem, err = provider.ConnPool.GetConn() } if err != nil { - return 0, fmt.Errorf("ERROR in GoReupsRoutine: ConnGet '%s' connitem='%v' sharedCC='%v' err='%v'", provider.Name, connitem, sharedCC, err) + return 0, fmt.Errorf("error in GoReupsRoutine: ConnGet '%s' connitem='%v' sharedCC='%v' err='%v'", provider.Name, connitem, sharedCC, err) } if connitem == nil || connitem.conn == nil || connitem.srvtp == nil { - return 0, fmt.Errorf("ERROR in GoReupsRoutine: ConnGet got nil item or conn '%s' connitem='%v' sharedCC='%v' err='%v'", provider.Name, connitem, sharedCC, err) + return 0, fmt.Errorf("error in GoReupsRoutine: ConnGet got nil item or conn '%s' connitem='%v' sharedCC='%v' err='%v'", provider.Name, connitem, sharedCC, err) } var uploaded, unwanted, retry bool @@ -523,7 +532,7 @@ func (s *SESSION) StopRoutines() { } // pushing nil into the segment chans will stop the routines for _, provider := range s.providerList { - closeSegmentChannel(s.segmentChansCheck[provider.Group]) + //closeSegmentChannel(s.segmentChansCheck[provider.Group]) closeSegmentChannel(s.segmentChansDowns[provider.Group]) closeSegmentChannel(s.segmentChansReups[provider.Group]) diff --git a/Session.go b/Session.go index b2a7d10..27706ba 100644 --- a/Session.go +++ b/Session.go @@ -258,7 +258,7 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s make(map[int]bool, len(s.providerList)), []string{}, []string{}, []string{}, // []string fields false, false, false, false, false, false, false, false, false, // bool fields - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} // int fields + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} // int fields if len(segment.Id) < 3 { // a@a dlog(always, "ERROR LaunchSession: segment.Id='%s' is too short! file='%s'", segment.Id, file.Filename) continue diff --git a/Workers.go b/Workers.go index 58d3df6..c5b6885 100644 --- a/Workers.go +++ b/Workers.go @@ -104,6 +104,7 @@ func (s *SESSION) GoBootWorkers(waitDivider *sync.WaitGroup, workerWGconnReady * // the check routine may still be activly checking! s.checkFeedDone = true s.mux.Unlock() + close(segmentChanCheck) // close the chan to signal no more items will come }(s.segmentChansCheck[provider.Group]) } globalmux.Unlock() @@ -185,6 +186,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro segCC := s.segmentChansCheck[provider.Group] go func(wid int, provider *Provider, waitWorker *sync.WaitGroup, sharedCC chan *ConnItem, segCC chan *segmentChanItem) { defer waitWorker.Done() + defer dlog(always, "CheckRoutine: wid=%d provider='%s' exiting", wid, provider.Name) forGoCheckRoutine: for { item, ok := <-segCC @@ -216,6 +218,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro segCD := s.segmentChansDowns[provider.Group] go func(wid int, provider *Provider, waitWorker *sync.WaitGroup, sharedCC chan *ConnItem, segCD chan *segmentChanItem) { defer waitWorker.Done() + defer dlog(always, "GoDownsRoutine: wid=%d provider='%s' exiting", wid, provider.Name) forGoDownsRoutine: for { dlog(cfg.opt.DebugWorker, "GoDownsRoutine: wid=%d provider='%s' wait on segCD len=%d", wid, provider.Name, len(segCD)) @@ -232,7 +235,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro start := time.Now() who := fmt.Sprintf("DR=%d@'%s'#'%s' seg.Id='%s'", wid, provider.Name, provider.Group, item.segment.Id) - memlim.MemLockWait(item, who) + memlim.MemLockWait(item, who) // gets memlock here dlog(cfg.opt.DebugWorker && cfg.opt.DebugMemlim, "GoDownsRoutine got MemCheckWait who='%s' waited=(%d ms)", who, time.Since(start).Milliseconds()) errStr := "" StartDowns := time.Now() @@ -274,6 +277,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro segCR := s.segmentChansReups[provider.Group] go func(wid int, provider *Provider, waitWorker *sync.WaitGroup, sharedCC chan *ConnItem, segCR chan *segmentChanItem) { defer waitWorker.Done() + defer dlog(always, "ReupsRoutine: wid=%d provider='%s' exiting", wid, provider.Name) forGoReupsRoutine: for { item, ok := <-segCR @@ -304,7 +308,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro item.PrintItemFlags(cfg.opt.DebugFlags, true, fmt.Sprintf("post-GoReupsRoutine: code=%d", code)) DecreaseUPQueueCnt() - if err != nil { + if err != nil || code == 0 { errStr := fmt.Sprintf("ERROR in GoReupsRoutine code='%d' err='%v' no retry", code, err) dlog(always, "%s", errStr) memlim.MemReturn("MemRetOnERR: "+errStr, item) // memfree GoReupsRoutine on error @@ -362,9 +366,17 @@ func matchThisDL(item *segmentChanItem) bool { } // end func matchThisDL func (item *segmentChanItem) FlagError(providerId int) { + log.Printf("FlagError called for providerId=%d on segment.Id='%s' waiting to lock item mutex", providerId, item.segment.Id) item.mux.Lock() defer item.mux.Unlock() + item.ignoreDlOn[providerId] = true + item.ignoreUlOn[providerId] = true item.errorOn[providerId] = true + item.flaginUP = false + item.flaginDL = false + item.fails++ + delete(item.availableOn, providerId) + dlog(always, "Flagged item error, will not retry '%s' on providerId=%d", item.segment.Id, providerId) } func (item *segmentChanItem) FlagDLFailed(providerList []*Provider, providerGroup string) { @@ -377,6 +389,7 @@ func (item *segmentChanItem) FlagDLFailed(providerList []*Provider, providerGrou item.ignoreDlOn[id] = true item.missingOn[id] = true item.errorOn[id] = true + delete(item.availableOn, id) } } @@ -418,7 +431,7 @@ func (s *SESSION) pushDL(allowDl bool, item *segmentChanItem) (pushed bool, nodl return false, 1, nil // not a match, item is already in DL or UP or has article } if !memlim.MemAvail() { - return + return false, 1, nil // not enough memory available to download } // if we are here, we are allowed to push the item to download queue // loop over the availableOn map and check if we can push the item to download @@ -436,6 +449,10 @@ providerDl: dlog(cfg.opt.DebugWorker, " | [DV-pushDL] (nodl) item missingOn but should be avail!? seg.Id='%s' @ #'%s'", item.segment.Id, s.providerList[pid].Group) continue providerDl } + if item.errorOn[pid] { + dlog(cfg.opt.DebugWorker, " | [DV-pushDL] (nodl) item errorOn seg.Id='%s' @ #'%s'", item.segment.Id, s.providerList[pid].Group) + continue providerDl + } if s.providerList[pid].NoDownload { nodl++ item.ignoreDlOn[pid] = true @@ -464,7 +481,7 @@ providerDl: item.pushedDL++ // mark as pushed to download queue (in pushDL) IncreaseDLQueueCnt() dlog(cfg.opt.DebugWorker, " | [DV-pushDL] pushed to dlchan seg.Id='%s' @ #'%s'", item.segment.Id, s.providerList[pid].Group) - return // return after 1st push! + return pushed, 0, nil // return after 1st push! default: dlog(cfg.opt.BUG, "DEBUG SPAM pushDL: chan is full @ #'%s', retry next", s.providerList[pid].Group) // chan is full means we cannot push the item to the download queue to this provider group @@ -475,19 +492,19 @@ providerDl: } // end select //} } // end for providerDl - return + return false, 1, nil } // end func pushDL func (s *SESSION) pushUP(allowUp bool, item *segmentChanItem) (pushed bool, noup uint64, inretry uint64, err error) { if !allowUp { - return + return false, 1, 0, fmt.Errorf("pushUP not allowed") } s.mux.RLock() segcheckdone := s.segcheckdone // get the segcheckdone state s.mux.RUnlock() if cfg.opt.CheckFirst && !segcheckdone { - return + return false, 1, 0, nil // still checking, do not push to upload yet } item.mux.Lock() // LOCK item for the duration of this function @@ -514,6 +531,11 @@ providerUp: dlog(cfg.opt.DebugWorker, " | [DV-pushUP] (noup) flagNoUp seg.Id='%s' @ #'%s'", item.segment.Id, s.providerList[pid].Group) continue providerUp } + if item.ignoreUlOn[pid] { + noup++ + dlog(cfg.opt.DebugWorker, " | [DV-pushUP] (noup) ignoreUlOn seg.Id='%s' @ #'%s'", item.segment.Id, s.providerList[pid].Group) + continue providerUp + } if item.uploadedTo[pid] { noup++ dlog(cfg.opt.DebugWorker, " | [DV-pushUP] (noup) uploadedTo seg.Id='%s' @ #'%s'", item.segment.Id, s.providerList[pid].Group) @@ -534,6 +556,11 @@ providerUp: dlog(cfg.opt.DebugWorker, " | [DV-pushUP] (noup) dmcaOn seg.Id='%s' @ #'%s'", item.segment.Id, s.providerList[pid].Group) continue providerUp } + if item.errorOn[pid] { + noup++ + dlog(cfg.opt.DebugWorker, " | [DV-pushUP] (noup) errorOn seg.Id='%s' @ #'%s'", item.segment.Id, s.providerList[pid].Group) + continue providerUp + } if item.retryOn[pid] { if item.retryIn > time.Now().Unix() { inretry++ @@ -557,7 +584,7 @@ providerUp: IncreaseUPQueueCnt() // increment upQueueCnt counter //GCounter.IncrMax("upQueueCnt", uint64(len(s.segmentList)), "pushUP") //GCounter.IncrMax("TOTAL_upQueueCnt", uint64(len(s.segmentList)), "pushUP") - dlog(cfg.opt.DebugWorker, " | pushUP: in chan seg.Id='%s' @ #'%s'", item.segment.Id, s.providerList[pid].Group) + dlog(always, " | pushUP: in chan seg.Id='%s' @ #'%s'", item.segment.Id, s.providerList[pid].Group) return true, 0, 0, nil // return after 1st push to a group! default: //dlog(cfg.opt.BUG "DEBUG SPAM pushUP: chan is full @ '%s'", s.providerList[pid].Name) @@ -692,36 +719,33 @@ forever: } item.mux.RUnlock() // RUNLOCKS HERE #824d - if !testing { - pushedUp, nNoUp, nInRetry, a1err := s.pushUP(allowUp, item) - if a1err != nil { - dlog(always, "ERROR pushUP err='%v' (seg.Id='%s')", a1err, item.segment.Id) + pushedUp, nNoUp, nInRetry, a1err := s.pushUP(allowUp, item) + if a1err != nil { + dlog(always, "ERROR pushUP err='%v' (seg.Id='%s')", a1err, item.segment.Id) + continue forsegmentList + } + if pushedUp { + inup++ + dlog(cfg.opt.DebugWorker, " | [DV] PUSHEDup seg.Id='%s' pushedUp=%t inup=%d", item.segment.Id, pushedUp, inup) + } + noup += nNoUp + inretry += nInRetry + + if !pushedUp && allowDl { + pushedDl, nNoDl, b1err := s.pushDL(allowDl, item) + if b1err != nil { + dlog(always, "ERROR pushDL err='%v' (seg.Id='%s')", b1err, item.segment.Id) continue forsegmentList } - if pushedUp { - inup++ - dlog(cfg.opt.DebugWorker, " | [DV] PUSHEDup seg.Id='%s' pushedUp=%t inup=%d", item.segment.Id, pushedUp, inup) + nodl += nNoDl + Tnodl += uint64(len(item.ignoreDlOn)) + if pushedDl { + indl++ + //if cfg.opt.BUG { + dlog(cfg.opt.DebugWorker, " | [DV] PUSHEDdl seg.Id='%s' pushedDl=%t indl=%d", item.segment.Id, pushedDl, indl) + //} } - noup += nNoUp - //Tnoup += len(item.ignoreDlOn) - inretry += nInRetry - - if !pushedUp && allowDl { - pushedDl, nNoDl, b1err := s.pushDL(allowDl, item) - if b1err != nil { - dlog(always, "ERROR pushDL err='%v' (seg.Id='%s')", b1err, item.segment.Id) - continue forsegmentList - } - nodl += nNoDl - Tnodl += uint64(len(item.ignoreDlOn)) - if pushedDl { - indl++ - //if cfg.opt.BUG { - dlog(cfg.opt.DebugWorker, " | [DV] PUSHEDdl seg.Id='%s' pushedDl=%t indl=%d", item.segment.Id, pushedDl, indl) - //} - } - } // end pushDL - } // if !testing { + } // end pushDL } // end for forsegmentList //dlog(cfg.opt.DebugWorker, " | [DV] lastRunTook='%d ms' '%v", lastRunTook.Milliseconds(), lastRunTook) From fa95a52aff690f87c9471426a13eeff535e78213 Mon Sep 17 00:00:00 2001 From: go-while Date: Fri, 17 Oct 2025 04:11:55 +0000 Subject: [PATCH 104/152] Update Config.go --- Config.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Config.go b/Config.go index 98b28c1..788e4d9 100644 --- a/Config.go +++ b/Config.go @@ -156,6 +156,12 @@ type Provider struct { refreshed uint64 // number of articles refreshed verified uint64 // number of articles verified } + // speed tracking (updated by Speedmeter every PrintStats seconds) + speed struct { + downloadSpeed float64 // bytes/sec (download) + uploadSpeed float64 // bytes/sec (upload) + lastUpdated int64 // unix timestamp of last update + } } // end Provider struct type segmentChanItem struct { @@ -204,6 +210,7 @@ type segmentChanItem struct { flaginYenc bool // if true, item is in writing to yenc cache flagisYenc bool // if true, item has been written to yenc cache flagCache bool // if true, item is cached + memlocked int // counts up if item is memlocked checkedOn int // counts up if item has been checked on a provider pushedDL int // a counter for debugging pushedUP int // a counter for debugging From cf32eeba26b96b925fae4f37bdc541ffb103cf79 Mon Sep 17 00:00:00 2001 From: go-while Date: Fri, 17 Oct 2025 04:24:28 +0000 Subject: [PATCH 105/152] debugs & mutex --- MemLimit.go | 1 + Workers.go | 11 ++++------- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/MemLimit.go b/MemLimit.go index d09bf4b..ced8653 100644 --- a/MemLimit.go +++ b/MemLimit.go @@ -128,6 +128,7 @@ func (m *MemLimiter) MemReturn(who string, item *segmentChanItem) { item.mux.RLock() if item.memlocked == 0 { dlog(always, "MemReturn called on non-memlocked item seg.Id='%s' who='%s'", item.segment.Id, who) + item.mux.RUnlock() return // not memlocked, nothing to do } item.mux.RUnlock() diff --git a/Workers.go b/Workers.go index c5b6885..eda6dd6 100644 --- a/Workers.go +++ b/Workers.go @@ -186,7 +186,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro segCC := s.segmentChansCheck[provider.Group] go func(wid int, provider *Provider, waitWorker *sync.WaitGroup, sharedCC chan *ConnItem, segCC chan *segmentChanItem) { defer waitWorker.Done() - defer dlog(always, "CheckRoutine: wid=%d provider='%s' exiting", wid, provider.Name) + defer dlog(always, "CheckRoutine exiting wid=%d provider='%s'", wid, provider.Name) forGoCheckRoutine: for { item, ok := <-segCC @@ -194,7 +194,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro return // channel is closed, exit the routine } if item == nil { - dlog(cfg.opt.DebugWorker, "CheckRoutine received a nil pointer to quit") + dlog(always, "CheckRoutine received a nil pointer to quit") segCC <- nil // refill the nil so others will die too break forGoCheckRoutine } @@ -218,7 +218,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro segCD := s.segmentChansDowns[provider.Group] go func(wid int, provider *Provider, waitWorker *sync.WaitGroup, sharedCC chan *ConnItem, segCD chan *segmentChanItem) { defer waitWorker.Done() - defer dlog(always, "GoDownsRoutine: wid=%d provider='%s' exiting", wid, provider.Name) + defer dlog(always, "GoDownsRoutine exiting wid=%d provider='%s'", wid, provider.Name) forGoDownsRoutine: for { dlog(cfg.opt.DebugWorker, "GoDownsRoutine: wid=%d provider='%s' wait on segCD len=%d", wid, provider.Name, len(segCD)) @@ -270,14 +270,13 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro // back to top } // end forGoDownsRoutine - dlog(always, "GoDownsRoutine: wid=%d provider='%s' exiting", wid, provider.Name) }(wid, provider, waitWorker, sharedCC, segCD) // end go func() /* new worker code ReupsRoutine */ segCR := s.segmentChansReups[provider.Group] go func(wid int, provider *Provider, waitWorker *sync.WaitGroup, sharedCC chan *ConnItem, segCR chan *segmentChanItem) { defer waitWorker.Done() - defer dlog(always, "ReupsRoutine: wid=%d provider='%s' exiting", wid, provider.Name) + defer dlog(always, "ReupsRoutine exiting: wid=%d provider='%s'", wid, provider.Name) forGoReupsRoutine: for { item, ok := <-segCR @@ -302,7 +301,6 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro dlog(cfg.opt.DebugWorker && cfg.opt.BUG, "WorkerReup: unlocked (wid=%d) (waited=%d µs), process seg.Id='%s' @ '%s'", wid, time.Since(start).Microseconds(), item.segment.Id, provider.Name) } */ - // TODO handle memlim freemem here StartReUps := time.Now() code, err := s.GoReupsRoutine(wid, provider, item, sharedCC) item.PrintItemFlags(cfg.opt.DebugFlags, true, fmt.Sprintf("post-GoReupsRoutine: code=%d", code)) @@ -320,7 +318,6 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro memlim.MemReturn("UR", item) // memfree GoReupsRoutine on success // back to top } // end forGoReupsRoutine - dlog(always, "ReupsRoutine: wid=%d provider='%s' exiting", wid, provider.Name) }(wid, provider, waitWorker, sharedCC, segCR) // end go func() dlog(cfg.opt.DebugWorker, "GoWorker (%d) waitWorker.Wait for routines to complete '%s'", wid, provider.Name) From 793f153fe58f68dde0a7de132ece18a70defcebf Mon Sep 17 00:00:00 2001 From: go-while Date: Fri, 17 Oct 2025 04:26:03 +0000 Subject: [PATCH 106/152] counter --- MemLimit.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/MemLimit.go b/MemLimit.go index ced8653..a1ebbd6 100644 --- a/MemLimit.go +++ b/MemLimit.go @@ -124,7 +124,6 @@ func (m *MemLimiter) MemLockWait(item *segmentChanItem, who string) { // it is also called if no upload and is written to cache func (m *MemLimiter) MemReturn(who string, item *segmentChanItem) { //dlog(cfg.opt.DebugMemlim, "MemReturn free seg.Id='%s' who='%s'", item.segment.Id, who) - defer GCounter.Incr("TOTAL_MemReturned") item.mux.RLock() if item.memlocked == 0 { dlog(always, "MemReturn called on non-memlocked item seg.Id='%s' who='%s'", item.segment.Id, who) @@ -132,6 +131,8 @@ func (m *MemLimiter) MemReturn(who string, item *segmentChanItem) { return // not memlocked, nothing to do } item.mux.RUnlock() + defer GCounter.Incr("TOTAL_MemReturned") + // remove map entry from mem m.mux.Lock() delete(m.memdata, item) From ba3538c5386e1f52dce09fac9c52b759de0f656e Mon Sep 17 00:00:00 2001 From: go-while Date: Fri, 17 Oct 2025 15:43:08 +0000 Subject: [PATCH 107/152] conn timeouts --- Config.go | 13 ++ MemLimit.go | 8 +- Routines.go | 1 + Session.go | 1 + Workers.go | 384 ++++++++++++++++++++++++------------- local_build_linux-amd64.sh | 11 +- 6 files changed, 281 insertions(+), 137 deletions(-) diff --git a/Config.go b/Config.go index 788e4d9..3aa6d4b 100644 --- a/Config.go +++ b/Config.go @@ -132,6 +132,7 @@ type Provider struct { Username string // username for authentication Password string // password for authentication MaxConns int // maximum number of connections to the provider + OpenConns int // current number of open connections TCPMode string // TCP mode to use (tcp, tcp4, tcp6) PreferIHAVE bool // if true, prefer IHAVE over POST method MaxConnErrors int // maximum number of errors before giving up on a connection @@ -164,6 +165,18 @@ type Provider struct { } } // end Provider struct +func (p *Provider) IncrementOpenConns() { + p.mux.Lock() + p.OpenConns++ + p.mux.Unlock() +} + +func (p *Provider) DecrementOpenConns() { + p.mux.Lock() + p.OpenConns-- + p.mux.Unlock() +} + type segmentChanItem struct { // segmentChanItem is used to store information about a segment/article // that is being processed in the segment channel. diff --git a/MemLimit.go b/MemLimit.go index a1ebbd6..696ae8e 100644 --- a/MemLimit.go +++ b/MemLimit.go @@ -15,6 +15,7 @@ package main */ import ( + "log" "os" "github.com/go-while/go-loggedrwmutex" @@ -131,7 +132,6 @@ func (m *MemLimiter) MemReturn(who string, item *segmentChanItem) { return // not memlocked, nothing to do } item.mux.RUnlock() - defer GCounter.Incr("TOTAL_MemReturned") // remove map entry from mem m.mux.Lock() @@ -141,11 +141,11 @@ func (m *MemLimiter) MemReturn(who string, item *segmentChanItem) { // return the slot select { case m.memchan <- struct{}{}: // return mem slot into chan - //pass + GCounter.Incr("TOTAL_MemReturned") default: // wtf chan is full?? that's a bug! - dlog(always, "ERROR on MemReturn chan is full seg.Id='%s' who='%s'", item.segment.Id, who) - os.Exit(1) // this is a bug! we should never return a slot to a full chan! + // this is a bug! we should never return a slot to a full chan! + log.Fatalf("ERROR on MemReturn chan is full seg.Id='%s' who='%s'", item.segment.Id, who) } item.mux.Lock() item.memlocked-- diff --git a/Routines.go b/Routines.go index 6dbdbac..5acd747 100644 --- a/Routines.go +++ b/Routines.go @@ -447,6 +447,7 @@ func (s *SESSION) GoReupsRoutine(wid int, provider *Provider, item *segmentChanI } item.flaginUP = false item.flagisUP = true + item.fails = 0 item.mux.Unlock() // update provider statistics provider.mux.Lock() // mutex #87c9 articles.refreshed++ diff --git a/Session.go b/Session.go index 27706ba..2a80637 100644 --- a/Session.go +++ b/Session.go @@ -83,6 +83,7 @@ type SESSION struct { checkFeedDone bool // checkDone is set to true when the segment feeder has finished feeding to channel, check may still be activly running! segcheckdone bool // segcheckdone is set to true when the segment check is done proxy bool // flag to signal proxy is used in LaunchSession + bootedWorkers int // counts how many workers have been booted } // end type SESSION struct func (p *PROCESSOR) NewProcessor() error { diff --git a/Workers.go b/Workers.go index eda6dd6..4e2612c 100644 --- a/Workers.go +++ b/Workers.go @@ -118,6 +118,13 @@ func (s *SESSION) GoBootWorkers(waitDivider *sync.WaitGroup, workerWGconnReady * connitem, err := provider.ConnPool.GetConn() if err != nil { dlog(always, "ERROR Boot Provider '%s' err='%v'", provider.Name, err) + // Release waitWorker and waitPool counters for all workers that won't be started + for i := 0; i < provider.MaxConns; i++ { + waitWorker.Done() // 3 times per worker + waitWorker.Done() + waitWorker.Done() + waitPool.Done() // 1 time per worker + } return } // check of capabilities @@ -161,7 +168,8 @@ func (s *SESSION) GoBootWorkers(waitDivider *sync.WaitGroup, workerWGconnReady * func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGroup, waitPool *sync.WaitGroup) { dlog(cfg.opt.DebugWorker, "GoWorker (%d) launching routines '%s'", wid, provider.Name) - + var thisWorkerWG sync.WaitGroup + thisWorkerWG.Add(3) // we have 3 routines per worker // Obtain a connection for this worker and share it among the check, download, and reupload routines. // The connection is not returned to the pool until all three routines have finished. var sharedCC chan *ConnItem = nil @@ -177,152 +185,240 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro waitWorker.Done() // release this GoWorker from the pool waitPool.Done() - dlog(cfg.opt.DebugWorker, "ERROR GoWorker (%d) failed to get shared connection channel for '%s' err='%v'", wid, provider.Name, err) + dlog(always, "ERROR GoWorker (%d) failed to get shared conn for '%s' err='%v' close worker", wid, provider.Name, err) return } } - + s.mux.Lock() + s.bootedWorkers++ + s.mux.Unlock() /* new worker code CheckRoutine */ segCC := s.segmentChansCheck[provider.Group] - go func(wid int, provider *Provider, waitWorker *sync.WaitGroup, sharedCC chan *ConnItem, segCC chan *segmentChanItem) { + go func(wid int, provider *Provider, waitWorker *sync.WaitGroup, thisWorkerWG *sync.WaitGroup, sharedCC chan *ConnItem, segCC chan *segmentChanItem) { defer waitWorker.Done() + defer thisWorkerWG.Done() defer dlog(always, "CheckRoutine exiting wid=%d provider='%s'", wid, provider.Name) + timeOut := time.Now().Add(15 * time.Second) + lastGood := time.Now() + toChan := make(chan struct{}, 1) + go func() { + for { + time.Sleep(3 * time.Second) + select { + case toChan <- struct{}{}: + default: + } + } + }() forGoCheckRoutine: for { - item, ok := <-segCC - if !ok { - return // channel is closed, exit the routine - } - if item == nil { - dlog(always, "CheckRoutine received a nil pointer to quit") - segCC <- nil // refill the nil so others will die too - break forGoCheckRoutine - } - - switch cfg.opt.ByPassSTAT { - case false: - code, err := s.GoCheckRoutine(wid, provider, item, sharedCC) - item.PrintItemFlags(cfg.opt.DebugFlags, true, fmt.Sprintf("post-GoCheckRoutine: code=%d", code)) - if err != nil { // re-queue? - dlog(always, "ERROR in GoCheckRoutine err='%v'", err) + select { + case <-toChan: + if time.Now().After(timeOut) { + dlog(always, "CheckRoutine: wid=%d provider='%s' timeout reached, exiting. lastGood: %v", wid, provider.Name, time.Since(lastGood)) + return + } + dlog(always, "CheckRoutine: wid=%d provider='%s' still alive, waiting on segCC len=%d timeout in: %v (lastGood: %v)", wid, provider.Name, len(segCC), time.Until(timeOut), time.Since(lastGood)) + case item, ok := <-segCC: + if !ok { + dlog(always, "CheckRoutine: channel closed wid=%d provider='%s'", wid, provider.Name) + return // channel is closed, exit the routine + } + if item == nil { + dlog(always, "CheckRoutine received a nil pointer to quit") + segCC <- nil // refill the nil so others will die too + break forGoCheckRoutine + } + timeOut = time.Now().Add(15 * time.Second) + lastGood = time.Now() + switch cfg.opt.ByPassSTAT { + case false: + code, err := s.GoCheckRoutine(wid, provider, item, sharedCC) + item.PrintItemFlags(cfg.opt.DebugFlags, true, fmt.Sprintf("post-GoCheckRoutine: code=%d", code)) + if err != nil { // re-queue? + dlog(always, "ERROR in GoCheckRoutine err='%v'", err) + } + case true: + log.Fatal("you should not be here! Quitting...") // FIXME TODO: remove this fatal error } - case true: - log.Fatal("you should not be here! Quitting...") // FIXME TODO: remove this fatal error + continue forGoCheckRoutine } - continue forGoCheckRoutine } // end forGoCheckRoutine dlog(always, "CheckRoutine: wid=%d provider='%s' exiting", wid, provider.Name) - }(wid, provider, waitWorker, sharedCC, segCC) // end go func() + }(wid, provider, waitWorker, &thisWorkerWG, sharedCC, segCC) // end go func() /* new worker code DownsRoutine */ segCD := s.segmentChansDowns[provider.Group] - go func(wid int, provider *Provider, waitWorker *sync.WaitGroup, sharedCC chan *ConnItem, segCD chan *segmentChanItem) { + go func(wid int, provider *Provider, waitWorker *sync.WaitGroup, thisWorkerWG *sync.WaitGroup, sharedCC chan *ConnItem, segCD chan *segmentChanItem) { defer waitWorker.Done() + defer thisWorkerWG.Done() defer dlog(always, "GoDownsRoutine exiting wid=%d provider='%s'", wid, provider.Name) + timeOut := time.Now().Add(15 * time.Second) + lastGood := time.Now() + toChan := make(chan struct{}, 1) + go func() { + for { + time.Sleep(3 * time.Second) + select { + case toChan <- struct{}{}: + default: + } + } + }() forGoDownsRoutine: for { dlog(cfg.opt.DebugWorker, "GoDownsRoutine: wid=%d provider='%s' wait on segCD len=%d", wid, provider.Name, len(segCD)) - item, ok := <-segCD - if !ok { - return // channel is closed, exit the routine - } - if item == nil { - dlog(cfg.opt.DebugWorker, "DownsRoutine received a nil pointer to quit") - segCD <- nil // refill the nil so others will die too - break forGoDownsRoutine - } - dlog(cfg.opt.DebugWorker, "GoDownsRoutine: wid=%d provider='%s' start seg.Id='%s'", wid, provider.Name, item.segment.Id) - - start := time.Now() - who := fmt.Sprintf("DR=%d@'%s'#'%s' seg.Id='%s'", wid, provider.Name, provider.Group, item.segment.Id) - memlim.MemLockWait(item, who) // gets memlock here - dlog(cfg.opt.DebugWorker && cfg.opt.DebugMemlim, "GoDownsRoutine got MemCheckWait who='%s' waited=(%d ms)", who, time.Since(start).Milliseconds()) - errStr := "" - StartDowns := time.Now() - code, err := s.GoDownsRoutine(wid, provider, item, sharedCC) - item.PrintItemFlags(cfg.opt.DebugFlags, true, fmt.Sprintf("post-GoDownsRoutine: code=%d", code)) - - DecreaseDLQueueCnt() - if err != nil || (code != 220 && code != 920) { - if code != 430 { - // 430 is a normal error code for GoDownsRoutine, so we don't log it as an error - errStr = fmt.Sprintf("ERROR in GoDownsRoutine code='%d' err='%v'. no retry", code, err) - dlog(always, "%s", errStr) + select { + case <-toChan: + s.mux.RLock() + segcheckdone := s.segcheckdone + s.mux.RUnlock() + if !segcheckdone { + timeOut = time.Now().Add(15 * time.Second) + lastGood = time.Now() // reset lastGood if segcheck is not done + continue forGoDownsRoutine } - memlim.MemReturn("MemRetOnERR: "+errStr, item) // memfree GoDownsRoutine on error - continue forGoDownsRoutine - } - var speedInKBytes float64 - mode := "downloaded" - if item.size > 0 { - speedInKBytes = (float64(item.size) / 1024) / float64(time.Since(StartDowns).Seconds()) - } - switch code { - case 220: - // pass + if segcheckdone && time.Now().After(timeOut) { + dlog(always, "GoDownsRoutine: wid=%d provider='%s' timeout reached, exiting. lastGood: %v", wid, provider.Name, time.Since(lastGood)) + return + } + dlog(always && segcheckdone, "GoDownsRoutine: wid=%d provider='%s' still alive, waiting on segCD len=%d timeout in: %v (lastGood: %v)", wid, provider.Name, len(segCD), time.Until(timeOut), time.Since(lastGood)) + case item, ok := <-segCD: + if !ok { + dlog(always, "GoDownsRoutine: channel closed wid=%d provider='%s'", wid, provider.Name) + return // channel is closed, exit the routine + } + if item == nil { + dlog(cfg.opt.DebugWorker, "DownsRoutine received a nil pointer to quit") + segCD <- nil // refill the nil so others will die too + break forGoDownsRoutine + } + timeOut = time.Now().Add(15 * time.Second) + lastGood = time.Now() + dlog(cfg.opt.DebugWorker, "GoDownsRoutine: wid=%d provider='%s' start seg.Id='%s'", wid, provider.Name, item.segment.Id) - case 920: // 920 is a special code for GoDownsRoutine to indicate that the item has been read from cache - mode = "cache read" - // Memory was already returned in GoDownsRoutine for cache reads - } + start := time.Now() + who := fmt.Sprintf("DR=%d@'%s'#'%s' seg.Id='%s'", wid, provider.Name, provider.Group, item.segment.Id) + memlim.MemLockWait(item, who) // gets memlock here + dlog(cfg.opt.DebugWorker && cfg.opt.DebugMemlim, "GoDownsRoutine got MemCheckWait who='%s' waited=(%d ms)", who, time.Since(start).Milliseconds()) + errStr := "" + StartDowns := time.Now() + code, err := s.GoDownsRoutine(wid, provider, item, sharedCC) + item.PrintItemFlags(cfg.opt.DebugFlags, true, fmt.Sprintf("post-GoDownsRoutine: code=%d", code)) + + DecreaseDLQueueCnt() + if err != nil || (code != 220 && code != 920) { + if code != 430 { + // 430 is a normal error code for GoDownsRoutine, so we don't log it as an error + errStr = fmt.Sprintf("ERROR in GoDownsRoutine code='%d' err='%v'. no retry", code, err) + dlog(always, "%s", errStr) + } + memlim.MemReturn("MemRetOnERR: "+errStr, item) // memfree GoDownsRoutine on error + continue forGoDownsRoutine + } + var speedInKBytes float64 + mode := "downloaded" + if item.size > 0 { + speedInKBytes = (float64(item.size) / 1024) / float64(time.Since(StartDowns).Seconds()) + } + switch code { + case 220: + // pass - dlog(cfg.opt.DebugWorker && cfg.opt.BUG, "GoDownsRoutine: %s (wid=%d) seg.Id='%s' @ '%s' took='%v' speedInKBytes=%.2f", mode, wid, item.segment.Id, provider.Name, time.Since(StartDowns), speedInKBytes) + case 920: // 920 is a special code for GoDownsRoutine to indicate that the item has been read from cache + mode = "cache read" + // Memory was already returned in GoDownsRoutine for cache reads + } + + dlog(cfg.opt.DebugWorker && cfg.opt.BUG, "GoDownsRoutine: %s (wid=%d) seg.Id='%s' @ '%s' took='%v' speedInKBytes=%.2f", mode, wid, item.segment.Id, provider.Name, time.Since(StartDowns), speedInKBytes) - // back to top + // back to top + } // end select } // end forGoDownsRoutine - }(wid, provider, waitWorker, sharedCC, segCD) // end go func() + }(wid, provider, waitWorker, &thisWorkerWG, sharedCC, segCD) // end go func() /* new worker code ReupsRoutine */ segCR := s.segmentChansReups[provider.Group] - go func(wid int, provider *Provider, waitWorker *sync.WaitGroup, sharedCC chan *ConnItem, segCR chan *segmentChanItem) { + go func(wid int, provider *Provider, waitWorker *sync.WaitGroup, thisWorkerWG *sync.WaitGroup, sharedCC chan *ConnItem, segCR chan *segmentChanItem) { defer waitWorker.Done() + defer thisWorkerWG.Done() defer dlog(always, "ReupsRoutine exiting: wid=%d provider='%s'", wid, provider.Name) + timeOut := time.Now().Add(15 * time.Second) + lastGood := time.Now() + toChan := make(chan struct{}, 1) + go func() { + for { + time.Sleep(3 * time.Second) + select { + case toChan <- struct{}{}: + default: + } + } + }() forGoReupsRoutine: for { - item, ok := <-segCR - if !ok { - return // channel is closed, exit the routine - } - if item == nil { - dlog(cfg.opt.DebugWorker, "ReupsRoutine received a nil pointer to quit") - s.segmentChansReups[provider.Group] <- nil // refill the nil so others will die too - break forGoReupsRoutine - } - - // we might get an item still locked for setting flags, so we lock too and wait for upper layer to release first. - /* - start := time.Now() - item.mux.Lock() - if cfg.opt.DebugWorker { - dlog(cfg.opt.DebugWorker && cfg.opt.BUG, "WorkerReup: got lock (wid=%d) seg.Id='%s' @ '%s'", wid, item.segment.Id, provider.Name) + select { + case <-toChan: + s.mux.RLock() + segcheckdone := s.segcheckdone // get the segcheckdone state + s.mux.RUnlock() + if !segcheckdone { + timeOut = time.Now().Add(15 * time.Second) + lastGood = time.Now() // reset lastGood if segcheck is not done + continue forGoReupsRoutine } - item.mux.Unlock() - if cfg.opt.DebugWorker { - dlog(cfg.opt.DebugWorker && cfg.opt.BUG, "WorkerReup: unlocked (wid=%d) (waited=%d µs), process seg.Id='%s' @ '%s'", wid, time.Since(start).Microseconds(), item.segment.Id, provider.Name) + if segcheckdone && time.Now().After(timeOut) { + dlog(always, "ReupsRoutine: wid=%d provider='%s' timeout reached, exiting. lastGood: %v", wid, provider.Name, time.Since(lastGood)) + return } - */ - StartReUps := time.Now() - code, err := s.GoReupsRoutine(wid, provider, item, sharedCC) - item.PrintItemFlags(cfg.opt.DebugFlags, true, fmt.Sprintf("post-GoReupsRoutine: code=%d", code)) - - DecreaseUPQueueCnt() - if err != nil || code == 0 { - errStr := fmt.Sprintf("ERROR in GoReupsRoutine code='%d' err='%v' no retry", code, err) - dlog(always, "%s", errStr) - memlim.MemReturn("MemRetOnERR: "+errStr, item) // memfree GoReupsRoutine on error - continue forGoReupsRoutine - } - speedInKBytes := (float64(item.size) / 1024) / float64(time.Since(StartReUps).Seconds()) - dlog(cfg.opt.DebugWorker && cfg.opt.BUG, "ReupsRoutine: finished item (wid=%d) seg.Id='%s' @ '%s' took='%v' speedInKBytes=%.2f", wid, item.segment.Id, provider.Name, time.Since(StartReUps), speedInKBytes) + dlog(always, "ReupsRoutine: wid=%d provider='%s' still alive, waiting on segCR len=%d timeout in: %v (lastgood: %v)", wid, provider.Name, len(segCR), time.Until(timeOut), time.Since(lastGood)) + case item, ok := <-segCR: + if !ok { + dlog(always, "ReupsRoutine: channel closed wid=%d provider='%s'", wid, provider.Name) + return // channel is closed, exit the routine + } + if item == nil { + dlog(cfg.opt.DebugWorker, "ReupsRoutine received a nil pointer to quit") + s.segmentChansReups[provider.Group] <- nil // refill the nil so others will die too + break forGoReupsRoutine + } + timeOut = time.Now().Add(15 * time.Second) + lastGood = time.Now() + // we might get an item still locked for setting flags, so we lock too and wait for upper layer to release first. + /* + start := time.Now() + item.mux.Lock() + if cfg.opt.DebugWorker { + dlog(cfg.opt.DebugWorker && cfg.opt.BUG, "WorkerReup: got lock (wid=%d) seg.Id='%s' @ '%s'", wid, item.segment.Id, provider.Name) + } + item.mux.Unlock() + if cfg.opt.DebugWorker { + dlog(cfg.opt.DebugWorker && cfg.opt.BUG, "WorkerReup: unlocked (wid=%d) (waited=%d µs), process seg.Id='%s' @ '%s'", wid, time.Since(start).Microseconds(), item.segment.Id, provider.Name) + } + */ + StartReUps := time.Now() + code, err := s.GoReupsRoutine(wid, provider, item, sharedCC) + item.PrintItemFlags(cfg.opt.DebugFlags, true, fmt.Sprintf("post-GoReupsRoutine: code=%d", code)) + + DecreaseUPQueueCnt() + if err != nil || code == 0 { + errStr := fmt.Sprintf("ERROR in GoReupsRoutine code='%d' err='%v' no retry", code, err) + dlog(always, "%s", errStr) + memlim.MemReturn("MemRetOnERR: "+errStr, item) // memfree GoReupsRoutine on error + continue forGoReupsRoutine + } + speedInKBytes := (float64(item.size) / 1024) / float64(time.Since(StartReUps).Seconds()) + dlog(cfg.opt.DebugWorker && cfg.opt.BUG, "ReupsRoutine: finished item (wid=%d) seg.Id='%s' @ '%s' took='%v' speedInKBytes=%.2f", wid, item.segment.Id, provider.Name, time.Since(StartReUps), speedInKBytes) - memlim.MemReturn("UR", item) // memfree GoReupsRoutine on success - // back to top + memlim.MemReturn("UR", item) // memfree GoReupsRoutine on success + // back to top + } // end select } // end forGoReupsRoutine - }(wid, provider, waitWorker, sharedCC, segCR) // end go func() + }(wid, provider, waitWorker, &thisWorkerWG, sharedCC, segCR) // end go func() - dlog(cfg.opt.DebugWorker, "GoWorker (%d) waitWorker.Wait for routines to complete '%s'", wid, provider.Name) - waitWorker.Wait() // wait for all 3 routines to finish - dlog(cfg.opt.DebugWorker, "GoWorker (%d) closing @ '%s'", wid, provider.Name) + dlog(always, "GoWorker (%d) waitWorker.Wait for routines to complete '%s'", wid, provider.Name) + thisWorkerWG.Wait() // wait for all 3 routines to finish + dlog(always, "GoWorker (%d) closing @ '%s'", wid, provider.Name) if sharedCC != nil { select { @@ -337,10 +433,13 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro dlog(cfg.opt.DebugWorker, "GoWorker (%d) no sharedConn there...? @ '%s' .. ciao", wid, provider.Name) } } - + s.mux.Lock() + s.bootedWorkers-- + s.mux.Unlock() + dlog(always, "GoWorker (%d) done with routines, releasing pool @ '%s'", wid, provider.Name) waitPool.Done() // release this GoWorker from the pool - - dlog(cfg.opt.BUG, "GoWorker (%d) quit @ '%s'", wid, provider.Name) + waitWorker.Wait() + dlog(always, "GoWorker (%d) quit @ '%s'", wid, provider.Name) } // end func GoWorker // matchThisDL checks if the item is a candidate for download @@ -366,10 +465,16 @@ func (item *segmentChanItem) FlagError(providerId int) { log.Printf("FlagError called for providerId=%d on segment.Id='%s' waiting to lock item mutex", providerId, item.segment.Id) item.mux.Lock() defer item.mux.Unlock() - item.ignoreDlOn[providerId] = true - item.ignoreUlOn[providerId] = true - item.errorOn[providerId] = true - item.flaginUP = false + //item.ignoreDlOn[providerId] = true + //item.ignoreUlOn[providerId] = true + //item.errorOn[providerId] = true + if item.flaginUP { + if item.fails < 3 { + item.retryIn = int64(time.Now().Add(5 * time.Second).Second()) + } + item.flaginUP = false + } + item.flaginDL = false item.fails++ delete(item.availableOn, providerId) @@ -622,14 +727,16 @@ func (s *SESSION) GoWorkDivider(waitDivider *sync.WaitGroup, waitDividerDone *sy var logstring, log00, log01, log02, log03, log04, log05, log06, log07, log08, log09, log10, log11, log99 string // loops forever over the s.segmentList and checks if there is anything to do for an item - var minsleep int64 = 10 // 0.01 second in milliseconds - var baseline int64 = 1000 - var maxsleep int64 = 6000 // 5 seconds in milliseconds + var minsleep int64 = 1000 + var baseline int64 = 1500 + var maxsleep int64 = 6000 microsleep := baseline fetchedtoDL, fetchedtoUP, backlogDL, refillUP, pushedDL, pushedUP := uint64(0), uint64(0), uint64(0), uint64(0), uint64(0), uint64(0) startLoop := time.Now() + deadWorkersDeadline := 9 forever: for { + dvlastRunTook := time.Since(startLoop) pushed := pushedDL + pushedUP @@ -639,8 +746,24 @@ forever: } time.Sleep(time.Duration((dvlastRunTook.Milliseconds() * 2)) + (time.Duration(microsleep) * time.Millisecond)) + if deadWorkersDeadline <= 0 { + dlog(always, "GoWorkDivider: no booted workers for too long, exiting") + closeCase = "noworkers" + break forever + } + s.mux.RLock() + dlog(always, "GoWorkDivider: booted workers: %d", s.bootedWorkers) + + if s.bootedWorkers == 0 { + deadWorkersDeadline-- + dlog(always, "GoWorkDivider: no booted workers, deadWorkersDeadline=%d", deadWorkersDeadline) + } else { + deadWorkersDeadline = 9 // reset + } + s.mux.RUnlock() + // uint64 - var segm, allOk, done, dead, isdl, indl, inup, isup, checked, dmca, nodl, noup, cached, inretry, inyenc, isyenc, dlQ, upQ, yeQ uint64 + var segm, allOk, done, fails, dead, isdl, indl, inup, isup, checked, dmca, nodl, noup, cached, inretry, inyenc, isyenc, dlQ, upQ, yeQ uint64 // Tnodl, Tnoup counts segments that not have been downloaded // or uploaded because provider has config flat NoDownload or NoUpload set @@ -688,6 +811,9 @@ forever: inup++ doContinue = true } + if item.fails >= 3 { + fails++ + } if len(item.availableOn) > 0 { segm++ // counts overall availability of segments @@ -810,8 +936,8 @@ forever: if dmca > 0 { log06 = fmt.Sprintf(" | DMCA:[%03.3f%%] (%"+s.D+"d)", dmca_perc, dmca) } - if inretry > 0 { - log07 = fmt.Sprintf(" | ERRS:(%"+s.D+"d)", inretry) + if fails > 0 || inretry > 0 { + log07 = fmt.Sprintf(" | ERRS:[%d] (inRetry: %"+s.D+"d)", fails, inretry) } if indl > 0 || isdl > 0 || dlQ > 0 || TdlQ > 0 { log08 = fmt.Sprintf(" | DL:[%03.3f%%] (%"+s.D+"d / %"+s.D+"d Q:%d=%d)", isdl_perc, isdl, TdlQ, dlQ, indl) @@ -861,18 +987,18 @@ forever: } // print some stats ends here s.mux.RLock() - checkDone := s.checkFeedDone // read value of s.checkDone - segcheckdone := s.segcheckdone // read value of s.segcheckdone + checkFeedDone := s.checkFeedDone // read value of s.checkDone + segcheckdone := s.segcheckdone // read value of s.segcheckdone s.mux.RUnlock() - if !segcheckdone && (checked == todo && checkDone) { + if !segcheckdone && ((checked == todo || cached == todo) && checkFeedDone) { s.mux.Lock() s.segmentCheckEndTime = time.Now() took := time.Since(s.segmentCheckStartTime) s.segmentCheckTook = took s.segcheckdone = true s.mux.Unlock() - dlog(cfg.opt.Verbose, " | Segment Check Done: took='%.1f sec'", took.Seconds()) + dlog(always, " | Segment Check Done: took='%.1f sec'", took.Seconds()) } // continue as long as any of this triggers because stuff is still in queues and processing @@ -892,13 +1018,13 @@ forever: // figure out if all jobs are done globalmux.RLock() - closeCase0 := (done == todo) + closeCase0 := (done == todo || fails == todo || fails+done == todo) closeCase1 := (cfg.opt.CheckOnly) closeCase2 := (cacheON && (GCounter.GetValue("postProviders") == 0 && cached == todo)) closeCase3 := (cacheON && (dead+cached == todo && dead+isup == todo)) closeCase4 := (isup == todo) - closeCase5 := (dead+isup == todo) - closeCase6 := (dead+done == todo) + closeCase5 := (dead+isup == todo || dead+isup+fails == todo) + closeCase6 := (dead+done == todo || dead+done+fails == todo) closeCase7 := false // placeholder globalmux.RUnlock() diff --git a/local_build_linux-amd64.sh b/local_build_linux-amd64.sh index dae80ff..1878474 100755 --- a/local_build_linux-amd64.sh +++ b/local_build_linux-amd64.sh @@ -1,7 +1,10 @@ -rm -rf rapidyenc/rapidyenc/build -mkdir -p rapidyenc/rapidyenc/build -cd rapidyenc && ./build_rapidyenc_linux-amd64.sh && cd ../ +#!/bin/bash +if [ "$1" != "quick" ]; then + rm -rf rapidyenc/rapidyenc/build + mkdir -p rapidyenc/rapidyenc/build + cd rapidyenc && ./build_rapidyenc_linux-amd64.sh && cd ../ +fi export GOOS=linux export GOARCH=amd64 -go build -o NZBreX -tags other . +go build -race -o NZBreX -tags other . && echo "built ok" exit $? From 8d7ad8b3cc224fcaf10b5b95dd4b35b9e9b79454 Mon Sep 17 00:00:00 2001 From: go-while Date: Fri, 17 Oct 2025 15:46:12 +0000 Subject: [PATCH 108/152] Update Workers.go --- Workers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Workers.go b/Workers.go index 4e2612c..00a4862 100644 --- a/Workers.go +++ b/Workers.go @@ -991,7 +991,7 @@ forever: segcheckdone := s.segcheckdone // read value of s.segcheckdone s.mux.RUnlock() - if !segcheckdone && ((checked == todo || cached == todo) && checkFeedDone) { + if !segcheckdone && checked == todo && checkFeedDone { s.mux.Lock() s.segmentCheckEndTime = time.Now() took := time.Since(s.segmentCheckStartTime) From 5c0f29537731f5262e095ddef7aaf48a8fc50890 Mon Sep 17 00:00:00 2001 From: go-while Date: Fri, 17 Oct 2025 18:38:36 +0000 Subject: [PATCH 109/152] testing --- Workers.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Workers.go b/Workers.go index 00a4862..7548acf 100644 --- a/Workers.go +++ b/Workers.go @@ -271,7 +271,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro select { case <-toChan: s.mux.RLock() - segcheckdone := s.segcheckdone + segcheckdone := s.segcheckdone // get the segcheckdone state s.mux.RUnlock() if !segcheckdone { timeOut = time.Now().Add(15 * time.Second) @@ -357,6 +357,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro }() forGoReupsRoutine: for { + dlog(cfg.opt.DebugWorker, "ReupsRoutine: wid=%d provider='%s' wait on segCD len=%d", wid, provider.Name, len(segCD)) select { case <-toChan: s.mux.RLock() @@ -751,7 +752,7 @@ forever: closeCase = "noworkers" break forever } - s.mux.RLock() + s.mux.Lock() dlog(always, "GoWorkDivider: booted workers: %d", s.bootedWorkers) if s.bootedWorkers == 0 { @@ -760,7 +761,7 @@ forever: } else { deadWorkersDeadline = 9 // reset } - s.mux.RUnlock() + s.mux.Unlock() // uint64 var segm, allOk, done, fails, dead, isdl, indl, inup, isup, checked, dmca, nodl, noup, cached, inretry, inyenc, isyenc, dlQ, upQ, yeQ uint64 From 4596aa1a20640ba71f125abc4cc145617c98ebdd Mon Sep 17 00:00:00 2001 From: go-while Date: Sat, 18 Oct 2025 04:55:39 +0000 Subject: [PATCH 110/152] adds .zip support --- Utils.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/Utils.go b/Utils.go index 54f8eaa..4035634 100644 --- a/Utils.go +++ b/Utils.go @@ -3,6 +3,7 @@ package main import ( //"bufio" //"bytes" + "archive/zip" "bufio" "compress/gzip" "crypto/sha256" @@ -79,6 +80,38 @@ func loadNzbFile(path string) (*nzbparser.Nzb, error) { } defer gzReader.Close() parsedReader = gzReader + } else if strings.HasSuffix(strings.ToLower(path), ".zip") { + // ZIP requires random access, can't stream directly + // Need to read file info first + fi, err := f.Stat() + if err != nil { + return nil, err + } + zipReader, err := zip.NewReader(f, fi.Size()) + if err != nil { + return nil, err + } + // Find first .nzb file in ZIP + if len(zipReader.File) == 0 { + return nil, fmt.Errorf("zip file is empty") + } + var nzbFile *zip.File + for _, zf := range zipReader.File { + if strings.HasSuffix(strings.ToLower(zf.Name), ".nzb") { + nzbFile = zf + dlog(always, "loadNzbFile: found nzb file in zip: '%s'", zf.Name) + break + } + } + if nzbFile == nil { + return nil, fmt.Errorf("no .nzb file found in zip archive") + } + rc, err := nzbFile.Open() + if err != nil { + return nil, err + } + defer rc.Close() + parsedReader = rc } else { parsedReader = f } From 17b143657b36084188960528059cd924f599429c Mon Sep 17 00:00:00 2001 From: go-while Date: Sat, 18 Oct 2025 05:15:40 +0000 Subject: [PATCH 111/152] debugs --- Workers.go | 62 +++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/Workers.go b/Workers.go index 7548acf..d5fd2b5 100644 --- a/Workers.go +++ b/Workers.go @@ -218,7 +218,9 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro dlog(always, "CheckRoutine: wid=%d provider='%s' timeout reached, exiting. lastGood: %v", wid, provider.Name, time.Since(lastGood)) return } - dlog(always, "CheckRoutine: wid=%d provider='%s' still alive, waiting on segCC len=%d timeout in: %v (lastGood: %v)", wid, provider.Name, len(segCC), time.Until(timeOut), time.Since(lastGood)) + if time.Since(lastGood) > 9*time.Second { + dlog(always, "CheckRoutine: wid=%d provider='%s' waiting on segCC len=%d timeout in: %v (lastGood: %v)", wid, provider.Name, len(segCC), time.Until(timeOut), time.Since(lastGood)) + } case item, ok := <-segCC: if !ok { dlog(always, "CheckRoutine: channel closed wid=%d provider='%s'", wid, provider.Name) @@ -282,7 +284,9 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro dlog(always, "GoDownsRoutine: wid=%d provider='%s' timeout reached, exiting. lastGood: %v", wid, provider.Name, time.Since(lastGood)) return } - dlog(always && segcheckdone, "GoDownsRoutine: wid=%d provider='%s' still alive, waiting on segCD len=%d timeout in: %v (lastGood: %v)", wid, provider.Name, len(segCD), time.Until(timeOut), time.Since(lastGood)) + if time.Since(lastGood) > 9*time.Second { + dlog(always && segcheckdone, "GoDownsRoutine: wid=%d provider='%s' waiting on segCD len=%d timeout in: %v (lastGood: %v)", wid, provider.Name, len(segCD), time.Until(timeOut), time.Since(lastGood)) + } case item, ok := <-segCD: if !ok { dlog(always, "GoDownsRoutine: channel closed wid=%d provider='%s'", wid, provider.Name) @@ -372,7 +376,9 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro dlog(always, "ReupsRoutine: wid=%d provider='%s' timeout reached, exiting. lastGood: %v", wid, provider.Name, time.Since(lastGood)) return } - dlog(always, "ReupsRoutine: wid=%d provider='%s' still alive, waiting on segCR len=%d timeout in: %v (lastgood: %v)", wid, provider.Name, len(segCR), time.Until(timeOut), time.Since(lastGood)) + if time.Since(lastGood) > 9*time.Second { + dlog(always, "ReupsRoutine: wid=%d provider='%s' waiting on segCR len=%d timeout in: %v (lastgood: %v)", wid, provider.Name, len(segCR), time.Until(timeOut), time.Since(lastGood)) + } case item, ok := <-segCR: if !ok { dlog(always, "ReupsRoutine: channel closed wid=%d provider='%s'", wid, provider.Name) @@ -687,7 +693,7 @@ providerUp: IncreaseUPQueueCnt() // increment upQueueCnt counter //GCounter.IncrMax("upQueueCnt", uint64(len(s.segmentList)), "pushUP") //GCounter.IncrMax("TOTAL_upQueueCnt", uint64(len(s.segmentList)), "pushUP") - dlog(always, " | pushUP: in chan seg.Id='%s' @ #'%s'", item.segment.Id, s.providerList[pid].Group) + dlog(cfg.opt.DebugWorker, " | pushUP: in chan seg.Id='%s' @ #'%s'", item.segment.Id, s.providerList[pid].Group) return true, 0, 0, nil // return after 1st push to a group! default: //dlog(cfg.opt.BUG "DEBUG SPAM pushUP: chan is full @ '%s'", s.providerList[pid].Name) @@ -735,6 +741,9 @@ func (s *SESSION) GoWorkDivider(waitDivider *sync.WaitGroup, waitDividerDone *sy fetchedtoDL, fetchedtoUP, backlogDL, refillUP, pushedDL, pushedUP := uint64(0), uint64(0), uint64(0), uint64(0), uint64(0), uint64(0) startLoop := time.Now() deadWorkersDeadline := 9 + deadcounter := 10 + var capture_done, capture_segm, capture_isdl, capture_isup float64 + forever: for { @@ -776,6 +785,12 @@ forever: for _, item := range s.segmentList { item.mux.PrintStatus(false) + if deadcounter <= 0 { + dlog(always, "GoWorkDivider: counters not increasing for too long, exiting") + closeCase = "noProgress" + break forever + } + item.mux.RLock() // RLOCKS HERE #824d if item.flagCache { cached++ @@ -873,6 +888,10 @@ forever: } // end for forsegmentList //dlog(cfg.opt.DebugWorker, " | [DV] lastRunTook='%d ms' '%v", lastRunTook.Milliseconds(), lastRunTook) + s.mux.RLock() + checkFeedDone := s.checkFeedDone // read value of s.checkDone + segcheckdone := s.segcheckdone // read value of s.segcheckdone + s.mux.RUnlock() // part to print stats begins here upQ = GCounter.GetValue("upQueueCnt") @@ -893,9 +912,37 @@ forever: isup_perc := float64(isup) / float64(todo) * 100 dmca_perc := float64(dmca) / float64(todo) * 100 yenc_perc := float64(isyenc) / float64(todo) * 100 - used_slots, max_slots := memlim.Usage() + if segm_perc > capture_segm { + capture_segm = segm_perc + deadcounter = 10 // reset deadcounter + } else if segm_perc < 100 && segm_perc == capture_segm { + deadcounter-- + dlog(always, "GoWorkDivider: segm_perc not increasing, deadcounter=%d", deadcounter) + + } else if segcheckdone && done_perc > capture_done { + capture_done = done_perc + deadcounter = 10 // reset deadcounter + } else if segcheckdone && done_perc < 100 && done_perc == capture_done { + deadcounter-- + dlog(always, "GoWorkDivider: done_perc not increasing, deadcounter=%d", deadcounter) + + } else if segcheckdone && isdl_perc > capture_isdl { + capture_isdl = isdl_perc + deadcounter = 10 // reset deadcounter + } else if segcheckdone && isdl_perc < 100 && isdl_perc == capture_isdl { + deadcounter-- + dlog(always, "GoWorkDivider: isdl_perc not increasing, deadcounter=%d", deadcounter) + + } else if segcheckdone && isup_perc > capture_isup { + capture_isup = isup_perc + deadcounter = 10 // reset deadcounter + } else if segcheckdone && isup_perc < 100 && isup_perc == capture_isup { + deadcounter-- + dlog(always, "GoWorkDivider: isup_perc not increasing, deadcounter=%d", deadcounter) + } + //log00 = fmt.Sprintf(" TODO: %d ", todo) if (cfg.opt.CheckFirst || cfg.opt.CheckOnly) || (checked > 0 && checked != segm && checked != done) { @@ -987,11 +1034,6 @@ forever: } } // print some stats ends here - s.mux.RLock() - checkFeedDone := s.checkFeedDone // read value of s.checkDone - segcheckdone := s.segcheckdone // read value of s.segcheckdone - s.mux.RUnlock() - if !segcheckdone && checked == todo && checkFeedDone { s.mux.Lock() s.segmentCheckEndTime = time.Now() From 151e72a4d61d6054485821ea1cc8bdeba2a2ff18 Mon Sep 17 00:00:00 2001 From: go-while Date: Sat, 18 Oct 2025 05:21:43 +0000 Subject: [PATCH 112/152] debugs --- Workers.go | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/Workers.go b/Workers.go index d5fd2b5..eeb61bd 100644 --- a/Workers.go +++ b/Workers.go @@ -195,9 +195,10 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro /* new worker code CheckRoutine */ segCC := s.segmentChansCheck[provider.Group] go func(wid int, provider *Provider, waitWorker *sync.WaitGroup, thisWorkerWG *sync.WaitGroup, sharedCC chan *ConnItem, segCC chan *segmentChanItem) { + var processed uint64 defer waitWorker.Done() defer thisWorkerWG.Done() - defer dlog(always, "CheckRoutine exiting wid=%d provider='%s'", wid, provider.Name) + defer dlog(always, "CheckRoutine exiting (%d@'%s') processed: %d", wid, provider.Name, processed) timeOut := time.Now().Add(15 * time.Second) lastGood := time.Now() toChan := make(chan struct{}, 1) @@ -215,15 +216,15 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro select { case <-toChan: if time.Now().After(timeOut) { - dlog(always, "CheckRoutine: wid=%d provider='%s' timeout reached, exiting. lastGood: %v", wid, provider.Name, time.Since(lastGood)) + dlog(always, "CheckRoutine: (%d@'%s') timeout reached, exiting. lastGood: %v", wid, provider.Name, time.Since(lastGood)) return } if time.Since(lastGood) > 9*time.Second { - dlog(always, "CheckRoutine: wid=%d provider='%s' waiting on segCC len=%d timeout in: %v (lastGood: %v)", wid, provider.Name, len(segCC), time.Until(timeOut), time.Since(lastGood)) + dlog(always, "CheckRoutine: (%d@'%s') waiting on segCC len=%d timeout in: %v (lastGood: %v)", wid, provider.Name, len(segCC), time.Until(timeOut), time.Since(lastGood)) } case item, ok := <-segCC: if !ok { - dlog(always, "CheckRoutine: channel closed wid=%d provider='%s'", wid, provider.Name) + dlog(always, "CheckRoutine: channel closed (%d@'%s')", wid, provider.Name) return // channel is closed, exit the routine } if item == nil { @@ -243,10 +244,10 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro case true: log.Fatal("you should not be here! Quitting...") // FIXME TODO: remove this fatal error } + processed++ continue forGoCheckRoutine } } // end forGoCheckRoutine - dlog(always, "CheckRoutine: wid=%d provider='%s' exiting", wid, provider.Name) }(wid, provider, waitWorker, &thisWorkerWG, sharedCC, segCC) // end go func() /* new worker code DownsRoutine */ @@ -254,7 +255,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro go func(wid int, provider *Provider, waitWorker *sync.WaitGroup, thisWorkerWG *sync.WaitGroup, sharedCC chan *ConnItem, segCD chan *segmentChanItem) { defer waitWorker.Done() defer thisWorkerWG.Done() - defer dlog(always, "GoDownsRoutine exiting wid=%d provider='%s'", wid, provider.Name) + defer dlog(always, "GoDownsRoutine exiting (%d@'%s')", wid, provider.Name) timeOut := time.Now().Add(15 * time.Second) lastGood := time.Now() toChan := make(chan struct{}, 1) @@ -269,7 +270,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro }() forGoDownsRoutine: for { - dlog(cfg.opt.DebugWorker, "GoDownsRoutine: wid=%d provider='%s' wait on segCD len=%d", wid, provider.Name, len(segCD)) + dlog(cfg.opt.DebugWorker, "GoDownsRoutine: (%d@'%s') wait on segCD len=%d", wid, provider.Name, len(segCD)) select { case <-toChan: s.mux.RLock() @@ -281,15 +282,15 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro continue forGoDownsRoutine } if segcheckdone && time.Now().After(timeOut) { - dlog(always, "GoDownsRoutine: wid=%d provider='%s' timeout reached, exiting. lastGood: %v", wid, provider.Name, time.Since(lastGood)) + dlog(always, "GoDownsRoutine: (%d@'%s') timeout reached, exiting. lastGood: %v", wid, provider.Name, time.Since(lastGood)) return } if time.Since(lastGood) > 9*time.Second { - dlog(always && segcheckdone, "GoDownsRoutine: wid=%d provider='%s' waiting on segCD len=%d timeout in: %v (lastGood: %v)", wid, provider.Name, len(segCD), time.Until(timeOut), time.Since(lastGood)) + dlog(always && segcheckdone, "GoDownsRoutine: (%d@'%s') waiting on segCD len=%d timeout in: %v (lastGood: %v)", wid, provider.Name, len(segCD), time.Until(timeOut), time.Since(lastGood)) } case item, ok := <-segCD: if !ok { - dlog(always, "GoDownsRoutine: channel closed wid=%d provider='%s'", wid, provider.Name) + dlog(always, "GoDownsRoutine: channel closed (%d@'%s')", wid, provider.Name) return // channel is closed, exit the routine } if item == nil { @@ -299,7 +300,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro } timeOut = time.Now().Add(15 * time.Second) lastGood = time.Now() - dlog(cfg.opt.DebugWorker, "GoDownsRoutine: wid=%d provider='%s' start seg.Id='%s'", wid, provider.Name, item.segment.Id) + dlog(cfg.opt.DebugWorker, "GoDownsRoutine: (%d@'%s') start seg.Id='%s'", wid, provider.Name, item.segment.Id) start := time.Now() who := fmt.Sprintf("DR=%d@'%s'#'%s' seg.Id='%s'", wid, provider.Name, provider.Group, item.segment.Id) @@ -346,7 +347,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro go func(wid int, provider *Provider, waitWorker *sync.WaitGroup, thisWorkerWG *sync.WaitGroup, sharedCC chan *ConnItem, segCR chan *segmentChanItem) { defer waitWorker.Done() defer thisWorkerWG.Done() - defer dlog(always, "ReupsRoutine exiting: wid=%d provider='%s'", wid, provider.Name) + defer dlog(always, "ReupsRoutine exiting: (%d@'%s')", wid, provider.Name) timeOut := time.Now().Add(15 * time.Second) lastGood := time.Now() toChan := make(chan struct{}, 1) @@ -361,7 +362,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro }() forGoReupsRoutine: for { - dlog(cfg.opt.DebugWorker, "ReupsRoutine: wid=%d provider='%s' wait on segCD len=%d", wid, provider.Name, len(segCD)) + dlog(cfg.opt.DebugWorker, "ReupsRoutine: (%d@'%s') wait on segCD len=%d", wid, provider.Name, len(segCD)) select { case <-toChan: s.mux.RLock() @@ -373,15 +374,15 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro continue forGoReupsRoutine } if segcheckdone && time.Now().After(timeOut) { - dlog(always, "ReupsRoutine: wid=%d provider='%s' timeout reached, exiting. lastGood: %v", wid, provider.Name, time.Since(lastGood)) + dlog(always, "ReupsRoutine: (%d@'%s') timeout reached, exiting. lastGood: %v", wid, provider.Name, time.Since(lastGood)) return } if time.Since(lastGood) > 9*time.Second { - dlog(always, "ReupsRoutine: wid=%d provider='%s' waiting on segCR len=%d timeout in: %v (lastgood: %v)", wid, provider.Name, len(segCR), time.Until(timeOut), time.Since(lastGood)) + dlog(always, "ReupsRoutine: (%d@'%s') waiting on segCR len=%d timeout in: %v (lastgood: %v)", wid, provider.Name, len(segCR), time.Until(timeOut), time.Since(lastGood)) } case item, ok := <-segCR: if !ok { - dlog(always, "ReupsRoutine: channel closed wid=%d provider='%s'", wid, provider.Name) + dlog(always, "ReupsRoutine: channel closed (%d@'%s')", wid, provider.Name) return // channel is closed, exit the routine } if item == nil { From 8ed44644b10b00ad2d71e42f945b2c0784a8863a Mon Sep 17 00:00:00 2001 From: go-while Date: Sat, 18 Oct 2025 05:41:41 +0000 Subject: [PATCH 113/152] debugs --- Workers.go | 76 ++++++++++++++++++++---------------------------------- 1 file changed, 28 insertions(+), 48 deletions(-) diff --git a/Workers.go b/Workers.go index eeb61bd..79a2c80 100644 --- a/Workers.go +++ b/Workers.go @@ -37,13 +37,8 @@ func (s *SESSION) GoBootWorkers(waitDivider *sync.WaitGroup, workerWGconnReady * dlog(always, "Cached: %d/%d", cached, len(s.segmentList)) } - if cfg.opt.ChanSize > 0 { - if len(s.segmentList) < cfg.opt.ChanSize { - cfg.opt.ChanSize = len(s.segmentList) - } - } else { + if cfg.opt.ChanSize <= 0 { cfg.opt.ChanSize = len(s.segmentList) - //cfg.opt.ChanSize = DefaultChanSize } // loop over the providerList and boot up anonymous workers for each provider @@ -198,7 +193,6 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro var processed uint64 defer waitWorker.Done() defer thisWorkerWG.Done() - defer dlog(always, "CheckRoutine exiting (%d@'%s') processed: %d", wid, provider.Name, processed) timeOut := time.Now().Add(15 * time.Second) lastGood := time.Now() toChan := make(chan struct{}, 1) @@ -217,15 +211,13 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro case <-toChan: if time.Now().After(timeOut) { dlog(always, "CheckRoutine: (%d@'%s') timeout reached, exiting. lastGood: %v", wid, provider.Name, time.Since(lastGood)) - return - } - if time.Since(lastGood) > 9*time.Second { - dlog(always, "CheckRoutine: (%d@'%s') waiting on segCC len=%d timeout in: %v (lastGood: %v)", wid, provider.Name, len(segCC), time.Until(timeOut), time.Since(lastGood)) + break forGoCheckRoutine } + dlog(time.Since(lastGood) > 9*time.Second, "CheckRoutine: (%d@'%s') waiting on segCC len=%d timeout in: %v (lastGood: %v)", wid, provider.Name, len(segCC), time.Until(timeOut), time.Since(lastGood)) case item, ok := <-segCC: if !ok { - dlog(always, "CheckRoutine: channel closed (%d@'%s')", wid, provider.Name) - return // channel is closed, exit the routine + dlog(cfg.opt.DebugWorker, "CheckRoutine: channel closed (%d@'%s')", wid, provider.Name) + break forGoCheckRoutine // channel is closed, exit the routine } if item == nil { dlog(always, "CheckRoutine received a nil pointer to quit") @@ -248,14 +240,15 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro continue forGoCheckRoutine } } // end forGoCheckRoutine + dlog(processed > 0, "CheckRoutine exiting (%d@'%s') processed: %d", wid, provider.Name, processed) }(wid, provider, waitWorker, &thisWorkerWG, sharedCC, segCC) // end go func() /* new worker code DownsRoutine */ segCD := s.segmentChansDowns[provider.Group] go func(wid int, provider *Provider, waitWorker *sync.WaitGroup, thisWorkerWG *sync.WaitGroup, sharedCC chan *ConnItem, segCD chan *segmentChanItem) { + var processed uint64 defer waitWorker.Done() defer thisWorkerWG.Done() - defer dlog(always, "GoDownsRoutine exiting (%d@'%s')", wid, provider.Name) timeOut := time.Now().Add(15 * time.Second) lastGood := time.Now() toChan := make(chan struct{}, 1) @@ -283,15 +276,13 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro } if segcheckdone && time.Now().After(timeOut) { dlog(always, "GoDownsRoutine: (%d@'%s') timeout reached, exiting. lastGood: %v", wid, provider.Name, time.Since(lastGood)) - return - } - if time.Since(lastGood) > 9*time.Second { - dlog(always && segcheckdone, "GoDownsRoutine: (%d@'%s') waiting on segCD len=%d timeout in: %v (lastGood: %v)", wid, provider.Name, len(segCD), time.Until(timeOut), time.Since(lastGood)) + break forGoDownsRoutine } + dlog(time.Since(lastGood) > 9*time.Second && always && segcheckdone, "GoDownsRoutine: (%d@'%s') waiting on segCD len=%d timeout in: %v (lastGood: %v)", wid, provider.Name, len(segCD), time.Until(timeOut), time.Since(lastGood)) case item, ok := <-segCD: if !ok { - dlog(always, "GoDownsRoutine: channel closed (%d@'%s')", wid, provider.Name) - return // channel is closed, exit the routine + dlog(cfg.opt.DebugWorker, "GoDownsRoutine: channel closed (%d@'%s')", wid, provider.Name) + break forGoDownsRoutine } if item == nil { dlog(cfg.opt.DebugWorker, "DownsRoutine received a nil pointer to quit") @@ -310,7 +301,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro StartDowns := time.Now() code, err := s.GoDownsRoutine(wid, provider, item, sharedCC) item.PrintItemFlags(cfg.opt.DebugFlags, true, fmt.Sprintf("post-GoDownsRoutine: code=%d", code)) - + processed++ DecreaseDLQueueCnt() if err != nil || (code != 220 && code != 920) { if code != 430 { @@ -335,19 +326,20 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro // Memory was already returned in GoDownsRoutine for cache reads } - dlog(cfg.opt.DebugWorker && cfg.opt.BUG, "GoDownsRoutine: %s (wid=%d) seg.Id='%s' @ '%s' took='%v' speedInKBytes=%.2f", mode, wid, item.segment.Id, provider.Name, time.Since(StartDowns), speedInKBytes) + dlog(cfg.opt.DebugWorker && cfg.opt.BUG, "GoDownsRoutine: segId='%s' (wid=%d) seg.Id='%s' @ '%s' took='%v' speedInKBytes=%.2f", mode, wid, item.segment.Id, provider.Name, time.Since(StartDowns), speedInKBytes) // back to top } // end select } // end forGoDownsRoutine + dlog(processed > 0, "GoDownsRoutine exiting (%d@'%s') processed: %d", wid, provider.Name, processed) }(wid, provider, waitWorker, &thisWorkerWG, sharedCC, segCD) // end go func() /* new worker code ReupsRoutine */ segCR := s.segmentChansReups[provider.Group] go func(wid int, provider *Provider, waitWorker *sync.WaitGroup, thisWorkerWG *sync.WaitGroup, sharedCC chan *ConnItem, segCR chan *segmentChanItem) { + var processed uint64 defer waitWorker.Done() defer thisWorkerWG.Done() - defer dlog(always, "ReupsRoutine exiting: (%d@'%s')", wid, provider.Name) timeOut := time.Now().Add(15 * time.Second) lastGood := time.Now() toChan := make(chan struct{}, 1) @@ -375,15 +367,14 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro } if segcheckdone && time.Now().After(timeOut) { dlog(always, "ReupsRoutine: (%d@'%s') timeout reached, exiting. lastGood: %v", wid, provider.Name, time.Since(lastGood)) - return - } - if time.Since(lastGood) > 9*time.Second { - dlog(always, "ReupsRoutine: (%d@'%s') waiting on segCR len=%d timeout in: %v (lastgood: %v)", wid, provider.Name, len(segCR), time.Until(timeOut), time.Since(lastGood)) + break forGoReupsRoutine } + dlog(time.Since(lastGood) > 9*time.Second, "ReupsRoutine: (%d@'%s') waiting on segCR len=%d timeout in: %v (lastgood: %v)", wid, provider.Name, len(segCR), time.Until(timeOut), time.Since(lastGood)) + case item, ok := <-segCR: if !ok { - dlog(always, "ReupsRoutine: channel closed (%d@'%s')", wid, provider.Name) - return // channel is closed, exit the routine + dlog(cfg.opt.DebugWorker, "ReupsRoutine: channel closed (%d@'%s')", wid, provider.Name) + break forGoReupsRoutine } if item == nil { dlog(cfg.opt.DebugWorker, "ReupsRoutine received a nil pointer to quit") @@ -392,22 +383,10 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro } timeOut = time.Now().Add(15 * time.Second) lastGood = time.Now() - // we might get an item still locked for setting flags, so we lock too and wait for upper layer to release first. - /* - start := time.Now() - item.mux.Lock() - if cfg.opt.DebugWorker { - dlog(cfg.opt.DebugWorker && cfg.opt.BUG, "WorkerReup: got lock (wid=%d) seg.Id='%s' @ '%s'", wid, item.segment.Id, provider.Name) - } - item.mux.Unlock() - if cfg.opt.DebugWorker { - dlog(cfg.opt.DebugWorker && cfg.opt.BUG, "WorkerReup: unlocked (wid=%d) (waited=%d µs), process seg.Id='%s' @ '%s'", wid, time.Since(start).Microseconds(), item.segment.Id, provider.Name) - } - */ - StartReUps := time.Now() + StartReUps := lastGood code, err := s.GoReupsRoutine(wid, provider, item, sharedCC) item.PrintItemFlags(cfg.opt.DebugFlags, true, fmt.Sprintf("post-GoReupsRoutine: code=%d", code)) - + processed++ DecreaseUPQueueCnt() if err != nil || code == 0 { errStr := fmt.Sprintf("ERROR in GoReupsRoutine code='%d' err='%v' no retry", code, err) @@ -422,11 +401,12 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro // back to top } // end select } // end forGoReupsRoutine + dlog(processed > 0, "ReupsRoutine exiting: (%d@'%s') processed: %d", wid, provider.Name, processed) }(wid, provider, waitWorker, &thisWorkerWG, sharedCC, segCR) // end go func() - dlog(always, "GoWorker (%d) waitWorker.Wait for routines to complete '%s'", wid, provider.Name) + dlog(cfg.opt.DebugWorker, "GoWorker (%d) waitWorker.Wait for routines to complete '%s'", wid, provider.Name) thisWorkerWG.Wait() // wait for all 3 routines to finish - dlog(always, "GoWorker (%d) closing @ '%s'", wid, provider.Name) + dlog(cfg.opt.DebugWorker, "GoWorker closing (%d@'%s')", wid, provider.Name) if sharedCC != nil { select { @@ -444,10 +424,10 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro s.mux.Lock() s.bootedWorkers-- s.mux.Unlock() - dlog(always, "GoWorker (%d) done with routines, releasing pool @ '%s'", wid, provider.Name) + dlog(cfg.opt.DebugWorker, "GoWorker (%d@'%s') done with routines, releasing pool", wid, provider.Name) waitPool.Done() // release this GoWorker from the pool waitWorker.Wait() - dlog(always, "GoWorker (%d) quit @ '%s'", wid, provider.Name) + dlog(cfg.opt.DebugWorker, "GoWorker quit (%d@'%s') ", wid, provider.Name) } // end func GoWorker // matchThisDL checks if the item is a candidate for download @@ -763,7 +743,7 @@ forever: break forever } s.mux.Lock() - dlog(always, "GoWorkDivider: booted workers: %d", s.bootedWorkers) + dlog(cfg.opt.DebugWorker, "GoWorkDivider: booted workers: %d", s.bootedWorkers) if s.bootedWorkers == 0 { deadWorkersDeadline-- From 0ac2658aa8ec0ef20583e5a9c1a0049d52bcbe3c Mon Sep 17 00:00:00 2001 From: go-while Date: Sat, 18 Oct 2025 05:41:51 +0000 Subject: [PATCH 114/152] debugs --- ConnPool.go | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/ConnPool.go b/ConnPool.go index 9bf3764..1c9e11c 100644 --- a/ConnPool.go +++ b/ConnPool.go @@ -208,7 +208,7 @@ func (c *ConnPool) connect() (connitem *ConnItem, err error) { if isNetworkUnreachable(err) { return nil, fmt.Errorf("error connect Unreachable network! '%s' @ '%s' err='%v'", c.provider.Host, c.provider.Name, err) } - dlog(always, "ERROR connect Dial '%s' no retry err='%v'", c.provider.Name, err) + dlog(cfg.opt.DebugConnPool, "ERROR connect Dial '%s' no retry err='%v'", c.provider.Name, err) return nil, err } case true: @@ -224,7 +224,7 @@ func (c *ConnPool) connect() (connitem *ConnItem, err error) { if isNetworkUnreachable(err) { return nil, fmt.Errorf("error connect Unreachable network! '%s' @ '%s' err='%v'", c.provider.Host, c.provider.Name, err) } - dlog(always, "ERROR connect Dial '%s' no-retry err='%v'", c.provider.Name, err) + dlog(cfg.opt.DebugConnPool, "ERROR connect Dial '%s' no-retry err='%v'", c.provider.Name, err) return nil, err } } // end switch @@ -326,7 +326,7 @@ func (c *ConnPool) NewConn() (connitem *ConnItem, err error) { if err != nil || connitem == nil || connitem.conn == nil { c.mux.Lock() // mutex c460 c.openConns-- - dlog(always, "ERROR in ConnPool NewConn: connect failed '%s' openConns=%d connitem='%v' err='%v'", c.provider.Name, c.openConns, connitem, err) + dlog(cfg.opt.DebugConnPool, "ERROR in ConnPool NewConn: connect failed '%s' openConns=%d connitem='%v' err='%v'", c.provider.Name, c.openConns, connitem, err) c.mux.Unlock() // mutex c460 return nil, fmt.Errorf("error in ConnPool NewConn: connect failed '%s' err='%v'", c.provider.Name, err) } @@ -335,6 +335,7 @@ func (c *ConnPool) NewConn() (connitem *ConnItem, err error) { c.mux.RUnlock() // mutex c461 // we have a new connection! GCounter.Incr("TOTAL_NewConns") + go c.provider.IncrementOpenConns() return connitem, nil // established new connection and returns connitem } c.mux.Unlock() // mutex c459a @@ -539,6 +540,7 @@ func (c *ConnPool) CloseConn(connitem *ConnItem, sharedConnChan chan *ConnItem) c.openConns-- dlog(cfg.opt.DebugConnPool, "DisConn '%s' inPool=%d open=%d", c.provider.Name, len(c.pool), c.openConns) c.mux.Unlock() + go c.provider.DecrementOpenConns() // if a sharedConnChan is supplied we send a nil to the channel // a nil as connitem signals the routines to get a new conn // mostly because conn was closed by network, protocol error or timeout @@ -737,9 +739,7 @@ func SharedConnGet(sharedCC chan *ConnItem, provider *Provider) (connitem *ConnI } if !needNewConn && aconnitem != nil && aconnitem.conn != nil { // we have a valid connection, use it - if cfg.opt.BUG { - dlog(cfg.opt.DebugConnPool, "SharedConnGet: got shared connection '%s' aconnitem='%#v'", provider.Name, aconnitem) - } + dlog(cfg.opt.DebugConnPool, "SharedConnGet: got shared connection '%s' aconnitem='%#v'", provider.Name, aconnitem) provider.ConnPool.ExtendConn(connitem) return aconnitem, nil } @@ -750,13 +750,13 @@ func SharedConnGet(sharedCC chan *ConnItem, provider *Provider) (connitem *ConnI newconnitem, err := provider.ConnPool.GetConn() if err != nil || newconnitem == nil || newconnitem.conn == nil { // unable to get a new connection, put nil back into sharedCC - dlog(always, "ERROR SharedConnGet connect failed Provider '%s' err='%v'", provider.Name, err) + dlog(cfg.opt.DebugConnPool, "ERROR SharedConnGet connect failed Provider '%s' err='%v'", provider.Name, err) sharedCC <- nil // put nil back into sharedCC - return + return nil, err } connitem = newconnitem // use the new connection item provider.ConnPool.ExtendConn(connitem) - return + return connitem, nil } // end func SharedConnGet // SharedConnReturn puts a connection back into the sharedCC channel. @@ -836,6 +836,16 @@ func Speedmeter(byteSize int64, cp *ConnPool, cnt *Counter_uint64, workerWGconnR txSpeed, txMbps := ConvertSpeed(int64(tmpTX), PrintStats) dlPerc := int(float64(totalRX) / float64(byteSize) * 100) upPerc := int(float64(totalTX) / float64(byteSize) * 100) + + // Store speed values in provider struct for web UI (if this is a provider-specific speedmeter) + if cp != nil && cp.provider != nil { + cp.provider.mux.Lock() + cp.provider.speed.downloadSpeed = float64(rxSpeed) * 1024 // convert KiB/s to bytes/sec + cp.provider.speed.uploadSpeed = float64(txSpeed) * 1024 // convert KiB/s to bytes/sec + cp.provider.speed.lastUpdated = time.Now().Unix() + cp.provider.mux.Unlock() + } + printSpeedTable(dlPerc, upPerc, totalRX, totalTX, rxSpeed, txSpeed, rxMbps, txMbps, name, group) } From 504e002fce850eefdd50d9c5f26e49ab95bf16b8 Mon Sep 17 00:00:00 2001 From: go-while Date: Sat, 18 Oct 2025 22:59:27 +0000 Subject: [PATCH 115/152] debugs --- Workers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Workers.go b/Workers.go index 79a2c80..36dc22f 100644 --- a/Workers.go +++ b/Workers.go @@ -841,7 +841,7 @@ forever: pushedUp, nNoUp, nInRetry, a1err := s.pushUP(allowUp, item) if a1err != nil { - dlog(always, "ERROR pushUP err='%v' (seg.Id='%s')", a1err, item.segment.Id) + dlog(cfg.opt.DebugWorker, "ERROR pushUP err='%v' (seg.Id='%s')", a1err, item.segment.Id) continue forsegmentList } if pushedUp { From 1e9c33cf7291a8eed56775d8e6c9616b69280b8f Mon Sep 17 00:00:00 2001 From: go-while Date: Sat, 19 Sep 2026 23:11:09 +0000 Subject: [PATCH 116/152] Update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 64e335c..8104d56 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,5 @@ oryxBuildBinary # dont ignore gh action yml !.github/workflows/*.yml +rapidyenc/librapidyenc_linux_amd64.a +rapidyenc/librapidyenc_windows_amd64.a From c136df95d798ab1c584937376c3c600336a1583e Mon Sep 17 00:00:00 2001 From: go-while Date: Sat, 17 Oct 2026 12:00:17 +0000 Subject: [PATCH 117/152] rapidyenc: use upstream C library as a submodule Replace the vendored copy of animetosho/rapidyenc in rapidyenc/rapidyenc/ (byte-identical to upstream 47f67f5, 2024-08-06) with a git submodule pinned to upstream d223ecc (v1.1.1-18). This picks up 14 upstream commits, including a scalar encoder overflow fix and an unaligned-load fix in the decoder. rapidyenc.h is unchanged, so the Go wrapper needs no changes. - .gitmodules: drop the stale mnightingale/rapidyenc entry, add rapidyenc-c (ignore = untracked, so build/ does not dirty the tree) - CI: check out submodules in build_and_publish_test and codeql - clone_rapidyenc.sh: init the submodule instead of being a no-op Verified on linux/amd64: go test -race ./rapidyenc/, -testrapidyenc, go vet. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build_and_publish_test.yml | 2 + .github/workflows/codeql.yml | 1 + .gitmodules | 7 +- rapidyenc/clone_rapidyenc.sh | 6 +- rapidyenc/rapidyenc | 1 + rapidyenc/rapidyenc/CMakeLists.txt | 275 --- rapidyenc/rapidyenc/README.md | 69 - rapidyenc/rapidyenc/crcutil-1.0/LICENSE | 202 -- .../rapidyenc/crcutil-1.0/code/base_types.h | 73 - .../rapidyenc/crcutil-1.0/code/crc32c_sse4.cc | 369 --- .../rapidyenc/crcutil-1.0/code/crc32c_sse4.h | 252 -- .../crcutil-1.0/code/crc32c_sse4_intrin.h | 102 - .../rapidyenc/crcutil-1.0/code/crc_casts.h | 68 - .../rapidyenc/crcutil-1.0/code/generic_crc.h | 687 ------ .../rapidyenc/crcutil-1.0/code/gf_util.h | 304 --- .../code/multiword_128_64_gcc_amd64_sse2.cc | 291 --- .../code/multiword_64_64_cl_i386_mmx.cc | 304 --- .../code/multiword_64_64_gcc_amd64_asm.cc | 298 --- .../code/multiword_64_64_gcc_i386_mmx.cc | 284 --- .../multiword_64_64_intrinsic_i386_mmx.cc | 243 -- .../rapidyenc/crcutil-1.0/code/platform.h | 245 -- .../crcutil-1.0/code/protected_crc.h | 61 - .../rapidyenc/crcutil-1.0/code/rolling_crc.h | 106 - .../rapidyenc/crcutil-1.0/code/std_headers.h | 51 - .../rapidyenc/crcutil-1.0/code/uint128_sse2.h | 312 --- .../crcutil-1.0/examples/interface.cc | 307 --- .../crcutil-1.0/examples/interface.h | 204 -- .../crcutil-1.0/tests/aligned_alloc.h | 66 - rapidyenc/rapidyenc/rapidyenc.cc | 126 - rapidyenc/rapidyenc/rapidyenc.h | 225 -- rapidyenc/rapidyenc/src/common.h | 364 --- rapidyenc/rapidyenc/src/crc.cc | 297 --- rapidyenc/rapidyenc/src/crc.h | 91 - rapidyenc/rapidyenc/src/crc_arm.cc | 226 -- rapidyenc/rapidyenc/src/crc_arm_pmull.cc | 215 -- rapidyenc/rapidyenc/src/crc_common.h | 26 - rapidyenc/rapidyenc/src/crc_folding.cc | 513 ----- rapidyenc/rapidyenc/src/crc_folding_256.cc | 223 -- rapidyenc/rapidyenc/src/crc_riscv.cc | 251 -- rapidyenc/rapidyenc/src/decoder.cc | 429 ---- rapidyenc/rapidyenc/src/decoder.h | 53 - rapidyenc/rapidyenc/src/decoder_avx.cc | 18 - rapidyenc/rapidyenc/src/decoder_avx2.cc | 18 - rapidyenc/rapidyenc/src/decoder_avx2_base.h | 618 ----- rapidyenc/rapidyenc/src/decoder_common.h | 220 -- rapidyenc/rapidyenc/src/decoder_neon.cc | 462 ---- rapidyenc/rapidyenc/src/decoder_neon64.cc | 446 ---- rapidyenc/rapidyenc/src/decoder_rvv.cc | 281 --- rapidyenc/rapidyenc/src/decoder_sse2.cc | 38 - rapidyenc/rapidyenc/src/decoder_sse_base.h | 699 ------ rapidyenc/rapidyenc/src/decoder_ssse3.cc | 18 - rapidyenc/rapidyenc/src/decoder_vbmi2.cc | 32 - rapidyenc/rapidyenc/src/encoder.cc | 198 -- rapidyenc/rapidyenc/src/encoder.h | 23 - rapidyenc/rapidyenc/src/encoder_avx.cc | 17 - rapidyenc/rapidyenc/src/encoder_avx2.cc | 17 - rapidyenc/rapidyenc/src/encoder_avx_base.h | 575 ----- rapidyenc/rapidyenc/src/encoder_common.h | 103 - rapidyenc/rapidyenc/src/encoder_neon.cc | 550 ----- rapidyenc/rapidyenc/src/encoder_rvv.cc | 207 -- rapidyenc/rapidyenc/src/encoder_sse2.cc | 15 - rapidyenc/rapidyenc/src/encoder_sse_base.h | 725 ------ rapidyenc/rapidyenc/src/encoder_ssse3.cc | 19 - rapidyenc/rapidyenc/src/encoder_vbmi2.cc | 31 - rapidyenc/rapidyenc/src/hedley.h | 2042 ----------------- rapidyenc/rapidyenc/src/platform.cc | 213 -- rapidyenc/rapidyenc/src/stdint.h | 251 -- rapidyenc/rapidyenc/tool/bench.cc | 130 -- rapidyenc/rapidyenc/tool/cli.c | 157 -- 69 files changed, 13 insertions(+), 16339 deletions(-) create mode 160000 rapidyenc/rapidyenc delete mode 100644 rapidyenc/rapidyenc/CMakeLists.txt delete mode 100644 rapidyenc/rapidyenc/README.md delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/LICENSE delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/code/base_types.h delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/code/crc32c_sse4.cc delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/code/crc32c_sse4.h delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/code/crc32c_sse4_intrin.h delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/code/crc_casts.h delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/code/generic_crc.h delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/code/gf_util.h delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/code/multiword_128_64_gcc_amd64_sse2.cc delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/code/multiword_64_64_cl_i386_mmx.cc delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/code/multiword_64_64_gcc_amd64_asm.cc delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/code/multiword_64_64_gcc_i386_mmx.cc delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/code/multiword_64_64_intrinsic_i386_mmx.cc delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/code/platform.h delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/code/protected_crc.h delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/code/rolling_crc.h delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/code/std_headers.h delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/code/uint128_sse2.h delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/examples/interface.cc delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/examples/interface.h delete mode 100644 rapidyenc/rapidyenc/crcutil-1.0/tests/aligned_alloc.h delete mode 100644 rapidyenc/rapidyenc/rapidyenc.cc delete mode 100644 rapidyenc/rapidyenc/rapidyenc.h delete mode 100644 rapidyenc/rapidyenc/src/common.h delete mode 100644 rapidyenc/rapidyenc/src/crc.cc delete mode 100644 rapidyenc/rapidyenc/src/crc.h delete mode 100644 rapidyenc/rapidyenc/src/crc_arm.cc delete mode 100644 rapidyenc/rapidyenc/src/crc_arm_pmull.cc delete mode 100644 rapidyenc/rapidyenc/src/crc_common.h delete mode 100644 rapidyenc/rapidyenc/src/crc_folding.cc delete mode 100644 rapidyenc/rapidyenc/src/crc_folding_256.cc delete mode 100644 rapidyenc/rapidyenc/src/crc_riscv.cc delete mode 100644 rapidyenc/rapidyenc/src/decoder.cc delete mode 100644 rapidyenc/rapidyenc/src/decoder.h delete mode 100644 rapidyenc/rapidyenc/src/decoder_avx.cc delete mode 100644 rapidyenc/rapidyenc/src/decoder_avx2.cc delete mode 100644 rapidyenc/rapidyenc/src/decoder_avx2_base.h delete mode 100644 rapidyenc/rapidyenc/src/decoder_common.h delete mode 100644 rapidyenc/rapidyenc/src/decoder_neon.cc delete mode 100644 rapidyenc/rapidyenc/src/decoder_neon64.cc delete mode 100644 rapidyenc/rapidyenc/src/decoder_rvv.cc delete mode 100644 rapidyenc/rapidyenc/src/decoder_sse2.cc delete mode 100644 rapidyenc/rapidyenc/src/decoder_sse_base.h delete mode 100644 rapidyenc/rapidyenc/src/decoder_ssse3.cc delete mode 100644 rapidyenc/rapidyenc/src/decoder_vbmi2.cc delete mode 100644 rapidyenc/rapidyenc/src/encoder.cc delete mode 100644 rapidyenc/rapidyenc/src/encoder.h delete mode 100644 rapidyenc/rapidyenc/src/encoder_avx.cc delete mode 100644 rapidyenc/rapidyenc/src/encoder_avx2.cc delete mode 100644 rapidyenc/rapidyenc/src/encoder_avx_base.h delete mode 100644 rapidyenc/rapidyenc/src/encoder_common.h delete mode 100644 rapidyenc/rapidyenc/src/encoder_neon.cc delete mode 100644 rapidyenc/rapidyenc/src/encoder_rvv.cc delete mode 100644 rapidyenc/rapidyenc/src/encoder_sse2.cc delete mode 100644 rapidyenc/rapidyenc/src/encoder_sse_base.h delete mode 100644 rapidyenc/rapidyenc/src/encoder_ssse3.cc delete mode 100644 rapidyenc/rapidyenc/src/encoder_vbmi2.cc delete mode 100644 rapidyenc/rapidyenc/src/hedley.h delete mode 100644 rapidyenc/rapidyenc/src/platform.cc delete mode 100644 rapidyenc/rapidyenc/src/stdint.h delete mode 100644 rapidyenc/rapidyenc/tool/bench.cc delete mode 100644 rapidyenc/rapidyenc/tool/cli.c diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index 4eb6522..f6c4785 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -124,6 +124,8 @@ jobs: - name: "Checkout code" uses: actions/checkout@v4 + with: + submodules: true - name: "Set up Go" uses: actions/setup-go@v5 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 83938b4..54d7929 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -87,6 +87,7 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + submodules: true - name: Install build dependencies if: matrix.language == 'c-cpp' diff --git a/.gitmodules b/.gitmodules index 8856e92..21afd8f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ -[submodule "rapidyenc"] - path = rapidyenc - url = https://github.com/mnightingale/rapidyenc.git +[submodule "rapidyenc-c"] + path = rapidyenc/rapidyenc + url = https://github.com/animetosho/rapidyenc.git + ignore = untracked diff --git a/rapidyenc/clone_rapidyenc.sh b/rapidyenc/clone_rapidyenc.sh index fefd3d5..1feddda 100755 --- a/rapidyenc/clone_rapidyenc.sh +++ b/rapidyenc/clone_rapidyenc.sh @@ -1 +1,5 @@ -# test ! -d rapidyenc && git clone https://github.com/animetosho/rapidyenc.git +#!/usr/bin/env bash +# rapidyenc C sources are a git submodule pinned in .gitmodules (animetosho/rapidyenc). +# To update: git -C rapidyenc/rapidyenc fetch && git -C rapidyenc/rapidyenc checkout && git add rapidyenc/rapidyenc +cd "$(dirname "$0")" || exit 1 +git submodule update --init -- rapidyenc || exit 1 diff --git a/rapidyenc/rapidyenc b/rapidyenc/rapidyenc new file mode 160000 index 0000000..d223ecc --- /dev/null +++ b/rapidyenc/rapidyenc @@ -0,0 +1 @@ +Subproject commit d223ecc92aa20e95e86827f341a7019ba7b62ecb diff --git a/rapidyenc/rapidyenc/CMakeLists.txt b/rapidyenc/rapidyenc/CMakeLists.txt deleted file mode 100644 index 7637eda..0000000 --- a/rapidyenc/rapidyenc/CMakeLists.txt +++ /dev/null @@ -1,275 +0,0 @@ -cmake_minimum_required(VERSION 2.8.9...3.22) -project(rapidyenc) - -option(BUILD_NATIVE "Optimise for and target only the build host's CPU; this build may not be re-distributable" OFF) -option(DISABLE_AVX256 "Disable the use of 256-bit AVX instructions on x86 processors" OFF) -option(DISABLE_CRCUTIL "Disable usage of crcutil library" OFF) -option(DISABLE_ENCODE "Exclude yEnc encoder from build" OFF) -option(DISABLE_DECODE "Exclude yEnc decoder from build" OFF) -option(DISABLE_CRC "Exclude CRC32 functions from build" OFF) - -include(CheckCXXCompilerFlag) -include(CheckIncludeFileCXX) -include(CheckCXXSymbolExists) - -set(CMAKE_CXX_STANDARD 98) - -if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Release) -endif() -if(NOT TARGET_ARCH) - if(CMAKE_GENERATOR_PLATFORM) - set(TARGET_ARCH ${CMAKE_GENERATOR_PLATFORM}) - else() - set(TARGET_ARCH ${CMAKE_SYSTEM_PROCESSOR}) - endif() -endif() - -if (${TARGET_ARCH} MATCHES "i386|i686|x86|x86_64|x64|amd64|AMD64|win32|Win32") - set(IS_X86 TRUE) - if(${TARGET_ARCH} MATCHES "x86_64|x64|amd64|AMD64") - if(NOT MSVC) - CHECK_CXX_SYMBOL_EXISTS(__ILP32__ "" IS_X32) - endif() - endif() -endif() -if (${TARGET_ARCH} MATCHES "arm|ARM|aarch64|arm64|ARM64") - CHECK_CXX_SYMBOL_EXISTS(__aarch64__ "" COMPILER_HAS_AARCH64) - CHECK_CXX_SYMBOL_EXISTS(_M_ARM64 "" COMPILER_HAS_ARM64) - - if(COMPILER_HAS_AARCH64 OR COMPILER_HAS_ARM64) - set(IS_ARM64 TRUE) - else() - set(IS_ARM32 TRUE) - endif() -endif() -if (${TARGET_ARCH} MATCHES "riscv64|rv64") - set(IS_RISCV64 TRUE) -endif() -if (${TARGET_ARCH} MATCHES "riscv32|rv32") - set(IS_RISCV32 TRUE) -endif() - -if(DISABLE_AVX256) - add_compile_definitions(YENC_DISABLE_AVX256=1) -endif() -if(DISABLE_ENCODE) - add_compile_definitions(RAPIDYENC_DISABLE_ENCODE=1) -endif() -if(DISABLE_DECODE) - add_compile_definitions(RAPIDYENC_DISABLE_DECODE=1) -endif() -if(DISABLE_CRC) - add_compile_definitions(RAPIDYENC_DISABLE_CRC=1) -endif() -if(DISABLE_CRCUTIL OR DISABLE_CRC) - add_compile_definitions(YENC_DISABLE_CRCUTIL=1) -endif() - -if(NOT MSVC) - if(BUILD_NATIVE) - CHECK_CXX_COMPILER_FLAG("-march=native" COMPILER_SUPPORTS_NATIVE) - if(COMPILER_SUPPORTS_NATIVE) - add_compile_options(-march=native) - add_compile_definitions(YENC_BUILD_NATIVE=1) - endif() - endif() - if(NOT BUILD_NATIVE AND IS_X86) - CHECK_CXX_COMPILER_FLAG("-mno-avx256-split-unaligned-load -mno-avx256-split-unaligned-store" COMPILER_SUPPORTS_NOSPLIT_AVX256) - if(COMPILER_SUPPORTS_NOSPLIT_AVX256) - add_compile_options(-mno-avx256-split-unaligned-load -mno-avx256-split-unaligned-store) - endif() - endif() -endif() - -if(IS_ARM64) - set(DECODER_NEON_FILE decoder_neon64.cc) -else() - set(DECODER_NEON_FILE decoder_neon.cc) -endif() - -set(SRC_DIR ./src) -set(RAPIDYENC_SOURCES - ${SRC_DIR}/platform.cc -) -if(NOT DISABLE_ENCODE) - set(RAPIDYENC_SOURCES ${RAPIDYENC_SOURCES} - ${SRC_DIR}/encoder.cc - ${SRC_DIR}/encoder_sse2.cc - ${SRC_DIR}/encoder_ssse3.cc - ${SRC_DIR}/encoder_avx.cc - ${SRC_DIR}/encoder_avx2.cc - ${SRC_DIR}/encoder_vbmi2.cc - ${SRC_DIR}/encoder_neon.cc - ${SRC_DIR}/encoder_rvv.cc - ) -endif() -if(NOT DISABLE_DECODE) - set(RAPIDYENC_SOURCES ${RAPIDYENC_SOURCES} - ${SRC_DIR}/decoder.cc - ${SRC_DIR}/decoder_sse2.cc - ${SRC_DIR}/decoder_ssse3.cc - ${SRC_DIR}/decoder_avx.cc - ${SRC_DIR}/decoder_avx2.cc - ${SRC_DIR}/decoder_vbmi2.cc - ${SRC_DIR}/${DECODER_NEON_FILE} - ${SRC_DIR}/decoder_rvv.cc - ) -endif() -if(NOT DISABLE_CRC) - set(RAPIDYENC_SOURCES ${RAPIDYENC_SOURCES} - ${SRC_DIR}/crc.cc - ${SRC_DIR}/crc_folding.cc - ${SRC_DIR}/crc_folding_256.cc - ${SRC_DIR}/crc_arm.cc - ${SRC_DIR}/crc_arm_pmull.cc - ${SRC_DIR}/crc_riscv.cc - ) - - if(NOT DISABLE_CRCUTIL) - set(CRCUTIL_DIR ./crcutil-1.0) - set(RAPIDYENC_SOURCES ${RAPIDYENC_SOURCES} - ${CRCUTIL_DIR}/code/crc32c_sse4.cc - ${CRCUTIL_DIR}/code/multiword_64_64_cl_i386_mmx.cc - ${CRCUTIL_DIR}/code/multiword_64_64_gcc_amd64_asm.cc - ${CRCUTIL_DIR}/code/multiword_64_64_gcc_i386_mmx.cc - ${CRCUTIL_DIR}/code/multiword_64_64_intrinsic_i386_mmx.cc - ${CRCUTIL_DIR}/code/multiword_128_64_gcc_amd64_sse2.cc - ${CRCUTIL_DIR}/examples/interface.cc - ) - - include_directories(${CRCUTIL_DIR}/code ${CRCUTIL_DIR}/examples ${CRCUTIL_DIR}/tests) - add_compile_definitions(CRCUTIL_USE_MM_CRC32=0) - endif() -endif() - - -if(MSVC) - set(RELEASE_COMPILE_FLAGS /GS- /Gy /sdl- /Oy /Oi) - set(RELEASE_LINK_FLAGS /OPT:REF /OPT:ICF) - add_compile_options(/W2 "$<$>:${RELEASE_COMPILE_FLAGS}>") - add_link_options("$<$>:${RELEASE_LINK_FLAGS}>") -else() - add_compile_options(-Wall -Wextra -Wno-unused-function -fno-exceptions -fwrapv -Wno-expansion-to-defined -fvisibility=hidden) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti") - if(NOT ${CMAKE_BUILD_TYPE} MATCHES "Debug") - add_compile_options(-fomit-frame-pointer) - endif() - - # posix_memalign may require _POSIX_C_SOURCE, but MacOS might not like it - include(CheckSymbolExists) - check_symbol_exists(posix_memalign "stdlib.h" HAVE_MEMALIGN) - if(NOT HAVE_MEMALIGN) - add_compile_definitions(_POSIX_C_SOURCE=200112L) - endif() -endif() - -add_library(rapidyenc OBJECT ${RAPIDYENC_SOURCES}) -set_property(TARGET rapidyenc PROPERTY POSITION_INDEPENDENT_CODE 1) # needed for shared build - -if(MSVC) - if(IS_X86) - set_source_files_properties(${SRC_DIR}/encoder_avx.cc PROPERTIES COMPILE_OPTIONS /arch:AVX) - set_source_files_properties(${SRC_DIR}/decoder_avx.cc PROPERTIES COMPILE_OPTIONS /arch:AVX) - set_source_files_properties(${SRC_DIR}/encoder_avx2.cc PROPERTIES COMPILE_OPTIONS /arch:AVX2) - set_source_files_properties(${SRC_DIR}/decoder_avx2.cc PROPERTIES COMPILE_OPTIONS /arch:AVX2) - set_source_files_properties(${SRC_DIR}/encoder_vbmi2.cc PROPERTIES COMPILE_OPTIONS /arch:AVX512) - set_source_files_properties(${SRC_DIR}/decoder_vbmi2.cc PROPERTIES COMPILE_OPTIONS /arch:AVX512) - set_source_files_properties(${SRC_DIR}/crc_folding_256.cc PROPERTIES COMPILE_OPTIONS /arch:AVX2) - endif() -endif() -if(NOT MSVC OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(IS_X86) - set_source_files_properties(${SRC_DIR}/encoder_sse2.cc PROPERTIES COMPILE_OPTIONS -msse2) - set_source_files_properties(${SRC_DIR}/decoder_sse2.cc PROPERTIES COMPILE_OPTIONS -msse2) - set_source_files_properties(${SRC_DIR}/encoder_ssse3.cc PROPERTIES COMPILE_OPTIONS -mssse3) - set_source_files_properties(${SRC_DIR}/decoder_ssse3.cc PROPERTIES COMPILE_OPTIONS -mssse3) - set_source_files_properties(${SRC_DIR}/encoder_avx.cc PROPERTIES COMPILE_OPTIONS "-mavx;-mpopcnt") - set_source_files_properties(${SRC_DIR}/decoder_avx.cc PROPERTIES COMPILE_OPTIONS "-mavx;-mpopcnt") - set_source_files_properties(${SRC_DIR}/encoder_avx2.cc PROPERTIES COMPILE_OPTIONS "-mavx2;-mpopcnt;-mbmi;-mbmi2;-mlzcnt") - set_source_files_properties(${SRC_DIR}/decoder_avx2.cc PROPERTIES COMPILE_OPTIONS "-mavx2;-mpopcnt;-mbmi;-mbmi2;-mlzcnt") - CHECK_CXX_COMPILER_FLAG("-mavx512vbmi2 -mavx512vl -mavx512bw" COMPILER_SUPPORTS_VBMI2) - CHECK_CXX_COMPILER_FLAG("-mavx512vl -mno-evex512" COMPILER_SUPPORTS_AVX10) - if(COMPILER_SUPPORTS_AVX10) - set_source_files_properties(${SRC_DIR}/encoder_vbmi2.cc PROPERTIES COMPILE_OPTIONS "-mavx512vbmi2;-mavx512vl;-mavx512bw;-mpopcnt;-mbmi;-mbmi2;-mlzcnt;-mno-evex512") - set_source_files_properties(${SRC_DIR}/decoder_vbmi2.cc PROPERTIES COMPILE_OPTIONS "-mavx512vbmi2;-mavx512vl;-mavx512bw;-mpopcnt;-mbmi;-mbmi2;-mlzcnt;-mno-evex512") - elseif(COMPILER_SUPPORTS_VBMI2) - set_source_files_properties(${SRC_DIR}/encoder_vbmi2.cc PROPERTIES COMPILE_OPTIONS "-mavx512vbmi2;-mavx512vl;-mavx512bw;-mpopcnt;-mbmi;-mbmi2;-mlzcnt") - set_source_files_properties(${SRC_DIR}/decoder_vbmi2.cc PROPERTIES COMPILE_OPTIONS "-mavx512vbmi2;-mavx512vl;-mavx512bw;-mpopcnt;-mbmi;-mbmi2;-mlzcnt") - endif() - set_source_files_properties(${SRC_DIR}/crc_folding.cc PROPERTIES COMPILE_OPTIONS "-mssse3;-msse4.1;-mpclmul") - CHECK_CXX_COMPILER_FLAG("-mvpclmulqdq" COMPILER_SUPPORTS_VPCLMULQDQ) - if(COMPILER_SUPPORTS_VPCLMULQDQ) - set_source_files_properties(${SRC_DIR}/crc_folding_256.cc PROPERTIES COMPILE_OPTIONS "-mavx2;-mvpclmulqdq;-mpclmul") - endif() - - if(IS_X32) - add_compile_definitions(CRCUTIL_USE_ASM=0) - endif() - endif() - - if(IS_ARM32) - CHECK_CXX_COMPILER_FLAG("-mfpu=neon -march=armv7-a" COMPILER_SUPPORTS_ARM32_NEON) - if(COMPILER_SUPPORTS_ARM32_NEON) - set_source_files_properties(${SRC_DIR}/encoder_neon.cc PROPERTIES COMPILE_OPTIONS "-mfpu=neon;-march=armv7-a") - set_source_files_properties(${SRC_DIR}/${DECODER_NEON_FILE} PROPERTIES COMPILE_OPTIONS "-mfpu=neon;-march=armv7-a") - endif() - CHECK_CXX_COMPILER_FLAG("-march=armv8-a+crc -mfpu=fp-armv8" COMPILER_SUPPORTS_ARM_CRC) - if(COMPILER_SUPPORTS_ARM_CRC) - set_source_files_properties(${SRC_DIR}/crc_arm.cc PROPERTIES COMPILE_OPTIONS "-march=armv8-a+crc;-mfpu=fp-armv8") - endif() - CHECK_CXX_COMPILER_FLAG("-march=armv8-a+crypto+crc -mfpu=neon" COMPILER_SUPPORTS_ARM_CRYPTO) - if(COMPILER_SUPPORTS_ARM_CRYPTO) - set_source_files_properties(${SRC_DIR}/crc_arm_pmull.cc PROPERTIES COMPILE_OPTIONS "-march=armv8-a+crypto+crc;-mfpu=neon") - endif() - endif() - if(IS_ARM64) - CHECK_CXX_COMPILER_FLAG("-march=armv8-a+crc" COMPILER_SUPPORTS_ARM_CRC) - if(COMPILER_SUPPORTS_ARM_CRC) - set_source_files_properties(${SRC_DIR}/crc_arm.cc PROPERTIES COMPILE_OPTIONS -march=armv8-a+crc) - endif() - CHECK_CXX_COMPILER_FLAG("-march=armv8-a+crypto+crc" COMPILER_SUPPORTS_ARM_CRYPTO) - if(COMPILER_SUPPORTS_ARM_CRYPTO) - set_source_files_properties(${SRC_DIR}/crc_arm_pmull.cc PROPERTIES COMPILE_OPTIONS -march=armv8-a+crypto+crc) - endif() - endif() - - if(IS_RISCV64) - CHECK_CXX_COMPILER_FLAG("-march=rv64gcv" COMPILER_SUPPORTS_RVV) - if(COMPILER_SUPPORTS_RVV) - set_source_files_properties(${SRC_DIR}/encoder_rvv.cc PROPERTIES COMPILE_OPTIONS -march=rv64gcv) - set_source_files_properties(${SRC_DIR}/decoder_rvv.cc PROPERTIES COMPILE_OPTIONS -march=rv64gcv) - endif() - CHECK_CXX_COMPILER_FLAG("-march=rv64gc_zbkc" COMPILER_SUPPORTS_ZBKC) - if(COMPILER_SUPPORTS_ZBKC) - set_source_files_properties(${SRC_DIR}/crc_riscv.cc PROPERTIES COMPILE_OPTIONS -march=rv64gc_zbkc) - endif() - endif() - if(IS_RISCV32) - CHECK_CXX_COMPILER_FLAG("-march=rv32gcv" COMPILER_SUPPORTS_RVV) - if(COMPILER_SUPPORTS_RVV) - set_source_files_properties(${SRC_DIR}/encoder_rvv.cc PROPERTIES COMPILE_OPTIONS -march=rv32gcv) - set_source_files_properties(${SRC_DIR}/decoder_rvv.cc PROPERTIES COMPILE_OPTIONS -march=rv32gcv) - endif() - CHECK_CXX_COMPILER_FLAG("-march=rv32gc_zbkc" COMPILER_SUPPORTS_ZBKC) - if(COMPILER_SUPPORTS_ZBKC) - set_source_files_properties(${SRC_DIR}/crc_riscv.cc PROPERTIES COMPILE_OPTIONS -march=rv32gc_zbkc) - endif() - endif() -endif() - - -add_library(rapidyenc_shared SHARED $ ./rapidyenc.cc) -target_compile_definitions(rapidyenc_shared PUBLIC BUILD_SHARED=1) -add_library(rapidyenc_static STATIC $ ./rapidyenc.cc) -# suggestion from https://stackoverflow.com/a/67027398/459150 -set_target_properties(rapidyenc_static PROPERTIES OUTPUT_NAME rapidyenc ARCHIVE_OUTPUT_DIRECTORY rapidyenc_static) -set_target_properties(rapidyenc_shared PROPERTIES OUTPUT_NAME rapidyenc ARCHIVE_OUTPUT_DIRECTORY rapidyenc_shared) - - -# binaries -add_executable(rapidyenc_cli tool/cli.c) -target_link_libraries(rapidyenc_cli rapidyenc_static) -add_executable(rapidyenc_bench tool/bench.cc) -target_link_libraries(rapidyenc_bench rapidyenc_static) -target_compile_features(rapidyenc_bench PUBLIC cxx_std_11) diff --git a/rapidyenc/rapidyenc/README.md b/rapidyenc/rapidyenc/README.md deleted file mode 100644 index fa4d013..0000000 --- a/rapidyenc/rapidyenc/README.md +++ /dev/null @@ -1,69 +0,0 @@ -This C compatible library provides functions for implementing [yEnc](http://www.yenc.org/yenc-draft.1.3.txt) where speed is important. - -Note that it only handles the underlying encoding/decoding routines - yEnc headers aren’t handled. - -Features ---------- - -- implementation uses x86/ARM/RISC-V SIMD capabilities, with support for ARMv7 NEON, ARMv8 ASIMD or the following x86 SIMD extensions: SSE2, SSSE3, AVX, AVX2, AVX512-BW (128/256-bit), AVX512-VBMI2 (or AVX10.1/256) -- CPU detection and dynamic dispatch (i.e. select best implementation for currently running CPU) -- incremental processing, including detection of yEnc/NNTP end sequences in decoder -- raw yEnc encoding with the ability to specify line length. A single thread can achieve \>450MB/s on a Raspberry Pi 3, or \>5GB/s on a Core-i series CPU. -- yEnc decoding, with and without NNTP layer dot unstuffing. A single thread can achieve \>300MB/s on a Raspberry Pi 3, or \>4.5GB/s on a Core-i series CPU. -- CRC32 implementation via [crcutil](https://code.google.com/p/crcutil/) or [PCLMULQDQ instruction](http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf), ARMv8’s CRC instructions, or RISC-V’s Zb(k)c extension (\>1GB/s on a low power Atom/ARM CPU, \>15GB/s on a modern Intel CPU) -- ability to combine two CRC32 hashes into one (useful for amalgamating *pcrc32s* into a *crc32* for yEnc), as well as quickly compute the CRC32 of a sequence of null bytes - -Building -========== - -A build file/project can be created using CMake. - -``` -mkdir build -cd build -cmake .. -cmake --build . --config Release -``` - -After compilation, a shared and static library should be generated, as well as a benchmark and sample CLI application. - -## Build Options - -The following options can be passed into CMake: - -* **BUILD_NATIVE**: Optimise for and target only the build host’s CPU; this build may not be re-distributable -* **DISABLE_AVX256**: Disable the use of 256-bit AVX instructions on x86 processors -* **DISABLE_CRCUTIL**: Disable crcutil usage. crcutil is only ever enabled for x86 builds -* **DISABLE_ENCODE**: Remove yEnc encode functionality from build. `rapidyenc_encode`* functions, except `rapidyenc_encode_max_length`, will be unavailable -* **DISABLE_DECODE**: Remove yEnc decode functionality from build. `rapidyenc_decode`* functions will be unavailable -* **DISABLE_CRC**: Remove CRC32 functionality from build. `rapidyenc_crc`* functions will be unavailable. Implies *DISABLE_CRCUTIL* - -API -=== - -Before any encoding/decoding/CRC functions can be used, the respective `_init` function must be called. These functions set up the necessary state for computation. Note that `_init` functions aren’t thread-safe, but all others are. - -Functions documented in the [header file](rapidyenc.h). - -[cli.c](tool/cli.c) is a simple command-line application which encodes/decodes stdin to stdout. It demonstrates how to do incremental encoding/decoding/CRC32 using this library. - -# Other Language Bindings - -* [node-yencode](https://github.com/animetosho/node-yencode): for NodeJS/Bun -* [go-yencode](https://github.com/mnightingale/go-yencode): for Golang -* [sabctools](https://github.com/sabnzbd/sabctools): for Python (Sabnzbd specific binding) - -Algorithm -========= - -A brief description of how the SIMD yEnc encoding algorithm works [can be found here](https://github.com/animetosho/node-yencode/issues/4#issuecomment-330025192). -I may eventually write up something more detailed, regarding optimizations and such used. - -License -======= - -This module is Public Domain or [CC0](https://creativecommons.org/publicdomain/zero/1.0/legalcode) (or equivalent) if PD isn’t recognised. - -[crcutil](https://code.google.com/p/crcutil/), used for CRC32 calculation, is licensed under the [Apache License 2.0](http://www.apache.org/licenses/LICENSE-2.0) - -[zlib-ng](https://github.com/Dead2/zlib-ng), from where the CRC32 calculation using folding approach was stolen, is under a [zlib license](https://github.com/Dead2/zlib-ng/blob/develop/LICENSE.md) \ No newline at end of file diff --git a/rapidyenc/rapidyenc/crcutil-1.0/LICENSE b/rapidyenc/rapidyenc/crcutil-1.0/LICENSE deleted file mode 100644 index d645695..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/rapidyenc/rapidyenc/crcutil-1.0/code/base_types.h b/rapidyenc/rapidyenc/crcutil-1.0/code/base_types.h deleted file mode 100644 index 5b74364..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/code/base_types.h +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Defines 8/16/32/64-bit integer types. -// -// Either uint64 or uint32 will map to size_t. -// This way, specialized variants of CRC implementation -// parameterized by "size_t" will be reused when -// parameterized by "uint64" or "uint32". -// In their turn, specialized verisons are parameterized -// by "size_t" so that one version of the code is optimal -// both on 32-bit and 64-bit platforms. - -#ifndef CRCUTIL_BASE_TYPES_H_ -#define CRCUTIL_BASE_TYPES_H_ - -#include "std_headers.h" // size_t, ptrdiff_t - -namespace crcutil { - -template class ChooseFirstIfSame { - public: - template class ChooseFirstIfTrue { - public: - typedef AA Type; - }; - template class ChooseFirstIfTrue { - public: - typedef BB Type; - }; - - typedef typename ChooseFirstIfTrue::Type Type; -}; - -typedef unsigned char uint8; -typedef signed char int8; - -typedef unsigned short uint16; -typedef short int16; - -typedef ChooseFirstIfSame::Type uint32; -typedef ChooseFirstIfSame::Type int32; - -#if defined(_MSC_VER) -typedef ChooseFirstIfSame::Type uint64; -typedef ChooseFirstIfSame::Type int64; -#define HAVE_UINT64 1 -#elif defined(__GNUC__) -typedef ChooseFirstIfSame::Type uint64; -typedef ChooseFirstIfSame::Type int64; -#define HAVE_UINT64 1 -#else -// TODO: ensure that everything compiles and works when HAVE_UINT64 is false. -// TODO: remove HAVE_UINT64 and use sizeof(uint64) instead? -#define HAVE_UINT64 0 -typedef uint32 uint64; -typedef int32 int64; -#endif - -} // namespace crcutil - -#endif // CRCUTIL_BASE_TYPES_H_ diff --git a/rapidyenc/rapidyenc/crcutil-1.0/code/crc32c_sse4.cc b/rapidyenc/rapidyenc/crcutil-1.0/code/crc32c_sse4.cc deleted file mode 100644 index 13aa84a..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/code/crc32c_sse4.cc +++ /dev/null @@ -1,369 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Implements CRC32C using Intel's SSE4 crc32 instruction. -// Uses _mm_crc32_u64/32/8 intrinsics if CRCUTIL_USE_MM_CRC32 is not zero, -// emilates intrinsics via CRC_WORD/CRC_BYTE otherwise. - -#include "crc32c_sse4.h" - -#if HAVE_I386 || HAVE_AMD64 - -namespace crcutil { - -#define UPDATE_STRIPE_CRCS(index, block_size, num_stripes) do { \ - CRC_UPDATE_WORD(crc0, \ - reinterpret_cast(src + \ - 0 * CRC32C_SSE4_STRIPE_SIZE(block_size, num_stripes))[index]); \ - CRC_UPDATE_WORD(crc1, \ - reinterpret_cast(src + \ - 1 * CRC32C_SSE4_STRIPE_SIZE(block_size, num_stripes))[index]); \ - CRC_UPDATE_WORD(crc2, \ - reinterpret_cast(src + \ - 2 * CRC32C_SSE4_STRIPE_SIZE(block_size, num_stripes))[index]); \ - if (num_stripes > 3) { \ - CRC_UPDATE_WORD(crc3, \ - reinterpret_cast(src + \ - 3 * CRC32C_SSE4_STRIPE_SIZE(block_size, num_stripes))[index]); \ - } \ -} while (0) - -// Multiplies "crc" by "x**(8 * STRIPE_SIZE(block_size)" -// using appropriate multiplication table(s). -// -#if 0 - -// This variant is for illustration purposes only. -// Actual implementation below: -// 1. Splits the computation into 2 data-independent paths -// by independently multiplying lower and upper halves -// of "crc0" in interleaved manner, and combining the -// results in the end. -// 2. Removing redundant "crc0 = 0" etc. in the beginning. -// 3. Removing redundant shifts of "tmp0" and "tmp1" in the last round. -#define MULTIPLY_CRC(crc0, block_size, num_stripes) do { \ - size_t tmp0 = crc0; \ - crc0 = 0; \ - for (size_t i = 0; i < kNumTables; ++i) { \ - crc0 ^= CRC32C_SSE4_MUL_TABLE(block_size, num_stripes) \ - [i][tmp0 & (kTableEntries - 1)]; \ - tmp0 >>= kTableEntryBits; \ - } \ -} while (0) - -#else - -#define MULTIPLY_CRC(crc0, block_size, num_stripes) do { \ - size_t tmp0 = crc0; \ - size_t tmp1 = crc0 >> (kTableEntryBits * kNumTablesHalfHi); \ - crc0 = CRC32C_SSE4_MUL_TABLE(block_size, num_stripes) \ - [0][tmp0 & (kTableEntries - 1)]; \ - tmp0 >>= kTableEntryBits; \ - size_t crc1 = CRC32C_SSE4_MUL_TABLE(block_size, num_stripes) \ - [kNumTablesHalfHi][tmp1 & (kTableEntries - 1)]; \ - tmp1 >>= kTableEntryBits; \ - for (size_t i = 1; i < kNumTablesHalfLo - 1; ++i) { \ - crc0 ^= CRC32C_SSE4_MUL_TABLE(block_size, num_stripes) \ - [i][tmp0 & (kTableEntries - 1)]; \ - tmp0 >>= kTableEntryBits; \ - crc1 ^= CRC32C_SSE4_MUL_TABLE(block_size, num_stripes) \ - [i + kNumTablesHalfHi][tmp1 & (kTableEntries - 1)]; \ - tmp1 >>= kTableEntryBits; \ - } \ - crc0 ^= CRC32C_SSE4_MUL_TABLE(block_size, num_stripes) \ - [kNumTablesHalfLo - 1][tmp0 & (kTableEntries - 1)]; \ - if (kNumTables & 1) { \ - tmp0 >>= kTableEntryBits; \ - } \ - crc1 ^= CRC32C_SSE4_MUL_TABLE(block_size, num_stripes) \ - [kNumTables - 1][tmp1]; \ - if (kNumTables & 1) { \ - crc0 ^= CRC32C_SSE4_MUL_TABLE(block_size, num_stripes) \ - [kNumTablesHalfLo][tmp0 & (kTableEntries - 1)]; \ - } \ - crc0 ^= crc1; \ -} while (0) - -#endif - -// Given CRCs (crc0, crc1, etc.) of consequitive -// stripes of STRIPE_SIZE(block_size) bytes each, -// produces CRC of concatenated stripes. -#define COMBINE_STRIPE_CRCS(block_size, num_stripes) do { \ - MULTIPLY_CRC(crc0, block_size, num_stripes); \ - crc0 ^= crc1; \ - MULTIPLY_CRC(crc0, block_size, num_stripes); \ - crc0 ^= crc2; \ - if (num_stripes > 3) { \ - MULTIPLY_CRC(crc0, block_size, num_stripes); \ - crc0 ^= crc3; \ - } \ -} while (0) - -// Processes input BLOCK_SIZE(block) bytes per iteration -// by splitting a block of BLOCK_SIZE(block) bytes into N -// equally-sized stripes of STRIPE_SIZE(block_size) each, -// computing CRC of each stripe, and concatenating stripe CRCs. -#define PROCESS_BLOCK(block_size, num_stripes) do { \ - while (bytes >= CRC32C_SSE4_BLOCK_SIZE(block_size, num_stripes)) { \ - Crc crc1 = 0; \ - Crc crc2 = 0; \ - Crc crc3; \ - if (num_stripes > 3) crc3 = 0; \ - { \ - const uint8 *stripe_end = src + \ - (CRC32C_SSE4_STRIPE_SIZE(block_size, num_stripes) / \ - kUnrolledLoopBytes) * kUnrolledLoopBytes; \ - do { \ - UPDATE_STRIPE_CRCS(0, block_size, num_stripes); \ - UPDATE_STRIPE_CRCS(1, block_size, num_stripes); \ - UPDATE_STRIPE_CRCS(2, block_size, num_stripes); \ - UPDATE_STRIPE_CRCS(3, block_size, num_stripes); \ - UPDATE_STRIPE_CRCS(4, block_size, num_stripes); \ - UPDATE_STRIPE_CRCS(5, block_size, num_stripes); \ - UPDATE_STRIPE_CRCS(6, block_size, num_stripes); \ - UPDATE_STRIPE_CRCS(7, block_size, num_stripes); \ - src += kUnrolledLoopBytes; \ - } while (src < stripe_end); \ - if ((CRC32C_SSE4_STRIPE_SIZE(block_size, num_stripes) % \ - kUnrolledLoopBytes) != 0) { \ - stripe_end += \ - CRC32C_SSE4_STRIPE_SIZE(block_size, num_stripes) % \ - kUnrolledLoopBytes; \ - do { \ - UPDATE_STRIPE_CRCS(0, block_size, num_stripes); \ - src += sizeof(size_t); \ - } while (src < stripe_end); \ - } \ - } \ - COMBINE_STRIPE_CRCS(block_size, num_stripes); \ - src += CRC32C_SSE4_STRIPE_SIZE(block_size, num_stripes) * \ - ((num_stripes) - 1); \ - bytes = static_cast(end - src); \ - } \ - no_more_##block_size##_##num_stripes:; \ -} while (0) - -size_t Crc32cSSE4::Crc32c(const void *data, size_t bytes, Crc crc0) const { - const uint8 *src = static_cast(data); - const uint8 *end = src + bytes; - crc0 ^= Base().Canonize(); - - // If we don't have too much data to process, - // do not waste time trying to align input etc. - // Noticeably improves performance on small inputs. - if (bytes < 4 * sizeof(size_t)) goto less_than_4_size_t; - if (bytes < 8 * sizeof(size_t)) goto less_than_8_size_t; - if (bytes < 16 * sizeof(size_t)) goto less_than_16_size_t; - -#define PROCESS_TAIL_IF_SMALL(block_size, num_stripes) do { \ - if (bytes < CRC32C_SSE4_BLOCK_SIZE(block_size, num_stripes)) { \ - goto no_more_##block_size##_##num_stripes; \ - } \ -} while (0) -#define NOOP(block_size, num_stripes) - - CRC32C_SSE4_ENUMERATE_ALL_BLOCKS_ASCENDING(PROCESS_TAIL_IF_SMALL, - NOOP, - NOOP); - -#undef PROCESS_TAIL_IF_SMALL - - - // Do not use ALIGN_ON_WORD_BOUNDARY_IF_NEEDED() here because: - // 1. It uses CRC_BYTE() which won't work. - // 2. Its threshold may be incorrect becuase Crc32 that uses - // native CPU crc32 instruction is much faster than - // generic table-based CRC computation. - // - // In case of X5550 CPU, break even point is at 2KB -- exactly. - if (bytes >= 2 * 1024) { - while ((reinterpret_cast(src) & (sizeof(Word) - 1)) != 0) { - if (src >= end) { - return (crc0 ^ Base().Canonize()); - } - CRC_UPDATE_BYTE(crc0, src[0]); - src += 1; - } - bytes = static_cast(end - src); - } - if (src >= end) { - return (crc0 ^ Base().Canonize()); - } - - // Quickly skip processing of too large blocks - // Noticeably improves performance on small inputs. -#define SKIP_BLOCK_IF_NEEDED(block_size, num_stripes) do { \ - if (bytes < CRC32C_SSE4_BLOCK_SIZE(block_size, num_stripes)) { \ - goto no_more_##block_size##_##num_stripes; \ - } \ -} while (0) - - CRC32C_SSE4_ENUMERATE_ALL_BLOCKS_ASCENDING(NOOP, - SKIP_BLOCK_IF_NEEDED, - SKIP_BLOCK_IF_NEEDED); - -#undef SKIP_BLOCK_IF_NEEDED - - // Process data in all blocks. - CRC32C_SSE4_ENUMERATE_ALL_BLOCKS_DESCENDING(PROCESS_BLOCK, - PROCESS_BLOCK, - PROCESS_BLOCK); - - // Finish the tail word-by-word and then byte-by-byte. -#define CRC_UPDATE_WORD_4(index) do { \ - CRC_UPDATE_WORD(crc0, reinterpret_cast(src)[index]); \ - CRC_UPDATE_WORD(crc0, reinterpret_cast(src)[index + 1]); \ - CRC_UPDATE_WORD(crc0, reinterpret_cast(src)[index + 2]); \ - CRC_UPDATE_WORD(crc0, reinterpret_cast(src)[index + 3]); \ -} while (0) - - if (bytes >= 4 * 4 * sizeof(size_t)) { - end -= 4 * 4 * sizeof(size_t); - do { - CRC_UPDATE_WORD_4(4 * 0); - CRC_UPDATE_WORD_4(4 * 1); - CRC_UPDATE_WORD_4(4 * 2); - CRC_UPDATE_WORD_4(4 * 3); - src += 4 * 4 * sizeof(size_t); - } while (src <= end); - end += 4 * 4 * sizeof(size_t); - bytes = static_cast(end - src); - } - less_than_16_size_t: - - if (bytes >= 4 * 2 * sizeof(size_t)) { - CRC_UPDATE_WORD_4(4 * 0); - CRC_UPDATE_WORD_4(4 * 1); - src += 4 * 2 * sizeof(size_t); - bytes -= 4 * 2 * sizeof(size_t); - } - less_than_8_size_t: - - if (bytes >= 4 * sizeof(size_t)) { - CRC_UPDATE_WORD_4(0); - src += 4 * sizeof(size_t); - bytes -= 4 * sizeof(size_t); - } - less_than_4_size_t: - - if (bytes >= 1 * sizeof(size_t)) { - end -= 1 * sizeof(size_t); - do { - CRC_UPDATE_WORD(crc0, reinterpret_cast(src)[0]); - src += 1 * sizeof(size_t); - } while (src <= end); - end += 1 * sizeof(size_t); - } - - while (src < end) { - CRC_UPDATE_BYTE(crc0, src[0]); - src += 1; - } - - return (crc0 ^ Base().Canonize()); -} - - -void Crc32cSSE4::Init(bool constant) { - base_.Init(FixedGeneratingPolynomial(), FixedDegree(), constant); - -#define INIT_MUL_TABLE(block_size, num_stripes) do { \ - size_t multiplier = \ - Base().Xpow8N(CRC32C_SSE4_STRIPE_SIZE(block_size, num_stripes)); \ - for (size_t table = 0; table < kNumTables; ++table) { \ - for (size_t entry = 0; entry < kTableEntries; ++entry) { \ - size_t value = static_cast(entry << (kTableEntryBits * table)); \ - CRC32C_SSE4_MUL_TABLE(block_size, num_stripes)[table][entry] = \ - static_cast(Base().Multiply(value, multiplier)); \ - } \ - } \ -} while (0) - - CRC32C_SSE4_ENUMERATE_ALL_BLOCKS(INIT_MUL_TABLE); - -#undef INIT_MUL_TABLE - -#if !CRCUTIL_USE_MM_CRC32 - for (size_t j = 0; j < sizeof(Word); ++j) { - Crc k = Base().XpowN((sizeof(Word) - 1 - j) * 8 + 32); - for (size_t i = 0; i < 256; ++i) { - crc_word_[j][i] = Base().MultiplyUnnormalized(i, 8, k); - } - } -#endif // !CRCUTIL_USE_MM_CRC32 -} - - -bool Crc32cSSE4::IsSSE42Available() { -#if defined(_MSC_VER) - int cpu_info[4]; - __cpuid(cpu_info, 1); - return ((cpu_info[3] & (1 << 20)) != 0); -#elif defined(__GNUC__) && (HAVE_AMD64 || HAVE_I386) - // Not using "cpuid.h" intentionally: it is missing from - // too many installations. - uint32 eax; - uint32 ecx; - uint32 edx; - __asm__ volatile( -#if HAVE_I386 && defined(__PIC__) - "push %%ebx\n" - "cpuid\n" - "pop %%ebx\n" -#define EBX_CLOBBER -// ^ GCC complains if we say we're clobbering ebx... -#else - "cpuid\n" -#define EBX_CLOBBER "ebx" -#endif // HAVE_I386 && defined(__PIC__) - : "=a" (eax), "=c" (ecx), "=d" (edx) - : "a" (1), "2" (0) - : EBX_CLOBBER - ); - return ((ecx & (1 << 20)) != 0); -#else - return false; -#endif -} - - -void RollingCrc32cSSE4::Init(const Crc32cSSE4 &crc, - size_t roll_window_bytes, - const Crc &start_value) { - crc_ = &crc; - roll_window_bytes_ = roll_window_bytes; - start_value_ = start_value; - - Crc add = crc.Base().Canonize() ^ start_value; - add = crc.Base().Multiply(add, crc.Base().Xpow8N(roll_window_bytes)); - add ^= crc.Base().Canonize(); - Crc mul = crc.Base().One() ^ crc.Base().Xpow8N(1); - add = crc.Base().Multiply(add, mul); - - mul = crc.Base().XpowN(8 * roll_window_bytes + crc.Base().Degree()); - for (size_t i = 0; i < 256; ++i) { - out_[i] = static_cast( - crc.Base().MultiplyUnnormalized( - static_cast(i), 8, mul) ^ add); - } - -#if !CRCUTIL_USE_MM_CRC32 - memcpy(crc_word_, crc_->crc_word_, sizeof(crc_word_)); -#endif // !CRCUTIL_USE_MM_CRC32 -} - -} // namespace crcutil - -#endif // HAVE_I386 || HAVE_AMD64 diff --git a/rapidyenc/rapidyenc/crcutil-1.0/code/crc32c_sse4.h b/rapidyenc/rapidyenc/crcutil-1.0/code/crc32c_sse4.h deleted file mode 100644 index ac3d842..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/code/crc32c_sse4.h +++ /dev/null @@ -1,252 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Implements CRC32C using Intel's SSE4 crc32 instruction. -// Uses _mm_crc32_u64/32/8 intrinsics if CRCUTIL_USE_MM_CRC32 is not zero, -// emilates intrinsics via CRC_WORD/CRC_BYTE otherwise. - -#ifndef CRCUTIL_CRC32C_SSE4_H_ -#define CRCUTIL_CRC32C_SSE4_H_ - -#include "gf_util.h" // base types, gf_util class, etc. -#include "crc32c_sse4_intrin.h" // _mm_crc32_u* intrinsics - -#if HAVE_I386 || HAVE_AMD64 - -#if CRCUTIL_USE_MM_CRC32 - -#if HAVE_I386 -#define CRC_UPDATE_WORD(crc, value) (crc = _mm_crc32_u32(crc, (value))) -#else -#define CRC_UPDATE_WORD(crc, value) (crc = _mm_crc32_u64(crc, (value))) -#endif // HAVE_I386 - -#define CRC_UPDATE_BYTE(crc, value) \ - (crc = _mm_crc32_u8(static_cast(crc), static_cast(value))) - -#else - -#include "generic_crc.h" - -#define CRC_UPDATE_WORD(crc, value) do { \ - size_t buf = (value); \ - CRC_WORD(this, crc, buf); \ -} while (0) -#define CRC_UPDATE_BYTE(crc, value) do { \ - CRC_BYTE(this, crc, (value)); \ -} while (0) - -#endif // CRCUTIL_USE_MM_CRC32 - -namespace crcutil { - -#pragma pack(push, 16) - -// Since the same pieces should be parameterized in many different places -// and we do not want to introduce a mistake which is rather hard to find, -// use a macro to enumerate all block sizes. -// -// Block sizes and number of stripes were tuned for best performance. -// -// All constants should be literal constants (too lazy to fix the macro). -// -// The use of different "macro_first", "macro", and "macro_last" -// allows generation of different code for smallest, in between, -// and largest block sizes. -// -// This macro shall be kept in sync with -// CRC32C_SSE4_ENUMERATE_ALL_BLOCKS_DESCENDING. -// Failure to do so will cause compile-time error. -#define CRC32C_SSE4_ENUMERATE_ALL_BLOCKS_ASCENDING( \ - macro_smallest, macro, macro_largest) \ - macro_smallest(512, 3); \ - macro(1024, 3); \ - macro(4096, 3); \ - macro_largest(32768, 3) - -// This macro shall be kept in sync with -// CRC32C_SSE4_ENUMERATE_ALL_BLOCKS_ASCENDING. -// Failure to do so will cause compile-time error. -#define CRC32C_SSE4_ENUMERATE_ALL_BLOCKS_DESCENDING( \ - macro_smallest, macro, macro_largest) \ - macro_largest(32768, 3); \ - macro(4096, 3); \ - macro(1024, 3); \ - macro_smallest(512, 3) - -// Enumerates all block sizes. -#define CRC32C_SSE4_ENUMERATE_ALL_BLOCKS(macro) \ - CRC32C_SSE4_ENUMERATE_ALL_BLOCKS_ASCENDING(macro, macro, macro) - -#define CRC32C_SSE4_STRIPE_SIZE(block_size, num_stripes) \ - (((block_size) / (num_stripes)) & ~(sizeof(size_t) - 1)) - -#define CRC32C_SSE4_BLOCK_SIZE(block_size, num_stripes) \ - (CRC32C_SSE4_STRIPE_SIZE(block_size, num_stripes) * (num_stripes)) - -#define CRC32C_SSE4_MUL_TABLE(block_size, num_stripes) \ - mul_table_##block_size##_##num_blocks##_ - -class RollingCrc32cSSE4; - -class Crc32cSSE4 { - public: - // Exports Crc, TableEntry, and Word (needed by RollingCrc). - typedef size_t Crc; - typedef Crc Word; - typedef Crc TableEntry; - - Crc32cSSE4() {} - - // Initializes the tables given generating polynomial of degree (degree). - // If "canonical" is true, crc value will be XOR'ed with (-1) before and - // after actual CRC computation. - explicit Crc32cSSE4(bool canonical) { - Init(canonical); - } - void Init(bool canonical); - - // Initializes the tables given generating polynomial of degree. - // If "canonical" is true, crc value will be XOR'ed with (-1) before and - // after actual CRC computation. - // Provided for compatibility with GenericCrc. - Crc32cSSE4(const Crc &generating_polynomial, - size_t degree, - bool canonical) { - Init(generating_polynomial, degree, canonical); - } - void Init(const Crc &generating_polynomial, - size_t degree, - bool canonical) { - if (generating_polynomial == FixedGeneratingPolynomial() && - degree == FixedDegree()) { - Init(canonical); - } - } - - // Returns fixed generating polymonial the class implements. - static Crc FixedGeneratingPolynomial() { - return 0x82f63b78; - } - - // Returns degree of fixed generating polymonial the class implements. - static Crc FixedDegree() { - return 32; - } - - // Returns base class. - const GfUtil &Base() const { return base_; } - - // Computes CRC32. - size_t CrcDefault(const void *data, size_t bytes, const Crc &crc) const { - return Crc32c(data, bytes, crc); - } - - // Returns true iff crc32 instruction is available. - static bool IsSSE42Available(); - - protected: - // Actual implementation. - size_t Crc32c(const void *data, size_t bytes, Crc crc) const; - - enum { - kTableEntryBits = 8, - kTableEntries = 1 << kTableEntryBits, - kNumTables = (32 + kTableEntryBits - 1) / kTableEntryBits, - kNumTablesHalfLo = kNumTables / 2, - kNumTablesHalfHi = (kNumTables + 1) / 2, - - kUnrolledLoopCount = 8, - kUnrolledLoopBytes = kUnrolledLoopCount * sizeof(size_t), - }; - - // May be set to size_t or uint32, whichever is faster. - typedef uint32 Entry; - -#define DECLARE_MUL_TABLE(block_size, num_stripes) \ - Entry CRC32C_SSE4_MUL_TABLE(block_size, num_stripes) \ - [kNumTables][kTableEntries] - - CRC32C_SSE4_ENUMERATE_ALL_BLOCKS(DECLARE_MUL_TABLE); - -#undef DECLARE_MUL_TABLE - - GfUtil base_; - -#if !CRCUTIL_USE_MM_CRC32 - TableEntry crc_word_[sizeof(Word)][256]; - friend class RollingCrc32cSSE4; -#endif // !CRCUTIL_USE_MM_CRC32 -} GCC_ALIGN_ATTRIBUTE(16); - -class RollingCrc32cSSE4 { - public: - typedef Crc32cSSE4::Crc Crc; - typedef Crc32cSSE4::TableEntry TableEntry; - typedef Crc32cSSE4::Word Word; - - RollingCrc32cSSE4() {} - - // Initializes internal data structures. - // Retains reference to "crc" instance -- it is used by Start(). - RollingCrc32cSSE4(const Crc32cSSE4 &crc, - size_t roll_window_bytes, - const Crc &start_value) { - Init(crc, roll_window_bytes, start_value); - } - void Init(const Crc32cSSE4 &crc, - size_t roll_window_bytes, - const Crc &start_value); - - // Computes crc of "roll_window_bytes" using - // "start_value" of "crc" (see Init()). - Crc Start(const void *data) const { - return crc_->CrcDefault(data, roll_window_bytes_, start_value_); - } - - // Computes CRC of "roll_window_bytes" starting in next position. - Crc Roll(const Crc &old_crc, size_t byte_out, size_t byte_in) const { - Crc crc = old_crc; - CRC_UPDATE_BYTE(crc, byte_in); - crc ^= out_[byte_out]; - return crc; - } - - // Returns start value. - Crc StartValue() const { return start_value_; } - - // Returns length of roll window. - size_t WindowBytes() const { return roll_window_bytes_; } - - protected: - typedef Crc Entry; - Entry out_[256]; - - // Used only by Start(). - Crc start_value_; - const Crc32cSSE4 *crc_; - size_t roll_window_bytes_; - -#if !CRCUTIL_USE_MM_CRC32 - TableEntry crc_word_[sizeof(Word)][256]; -#endif // !CRCUTIL_USE_MM_CRC32 -} GCC_ALIGN_ATTRIBUTE(16); - -#pragma pack(pop) - -} // namespace crcutil - -#endif // HAVE_I386 || HAVE_AMD64 - -#endif // CRCUTIL_CRC32C_SSE4_H_ diff --git a/rapidyenc/rapidyenc/crcutil-1.0/code/crc32c_sse4_intrin.h b/rapidyenc/rapidyenc/crcutil-1.0/code/crc32c_sse4_intrin.h deleted file mode 100644 index b57a7c4..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/code/crc32c_sse4_intrin.h +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Provides _mm_crc32_u64/32/8 intrinsics. - -#ifndef CRCUTIL_CRC32C_SSE4_INTRIN_H_ -#define CRCUTIL_CRC32C_SSE4_INTRIN_H_ - -#include "platform.h" -#include "base_types.h" - -#if CRCUTIL_USE_MM_CRC32 && (HAVE_I386 || HAVE_AMD64) - -#if defined(_MSC_VER) || defined(__SSE4_2__) - -#if defined(_MSC_VER) -#pragma warning(push) -// '_M_IA64' is not defined as a preprocessor macro -#pragma warning(disable: 4668) -#endif // defined(_MSC_VER) - -#include - -#if defined(_MSC_VER) -#pragma warning(pop) -#endif // defined(_MSC_VER) - -#elif GCC_VERSION_AVAILABLE(4, 5) && !defined(CRCUTIL_FORCE_ASM_CRC32C) -// Allow the use of _mm_crc32_u* intrinsic when CRCUTIL_USE_MM_CRC32 -// is set irrespective of "-msse*" settings. This way, the sources -// may be compiled with "-msse2 -mcrc32" and work on older CPUs, -// while taking full advantage of "crc32" instruction on newer -// CPUs (requires dynamic CPU detection). See "interface.cc". -// -// If neither -msse4 or -mcrc32 is provided and CRCUTIL_USE_MM_CRC32 is set -// and CRCUTIL_FORCE_ASM_CRC32 is not set, compile-time error will happen. -// Why? Becuase GCC disables __builtin_ia32_crc32* intrinsics when compiled -// without -msse4 or -mcrc32. -msse4 could be detected at run time by checking -// whether __SSE4_2__ is defined, but there is no way to tell whether the -// sources are compiled with -mcrc32. - -extern __inline unsigned int __attribute__(( - __gnu_inline__, __always_inline__, __artificial__)) -_mm_crc32_u8(unsigned int __C, unsigned char __V) { - //return __builtin_ia32_crc32qi(__C, __V); - return 0; -} -#ifdef __x86_64__ -extern __inline unsigned long long __attribute__(( - __gnu_inline__, __always_inline__, __artificial__)) -_mm_crc32_u64(unsigned long long __C, unsigned long long __V) { - //return __builtin_ia32_crc32di(__C, __V); - return 0; -} -#else -extern __inline unsigned int __attribute__(( - __gnu_inline__, __always_inline__, __artificial__)) -_mm_crc32_u32(unsigned int __C, unsigned int __V) { - //return __builtin_ia32_crc32si (__C, __V); - return 0; -} -#endif // __x86_64__ - -#else - -// GCC 4.4.x and earlier: use inline asm. - -namespace crcutil { - -__forceinline uint64 _mm_crc32_u64(uint64 crc, uint64 value) { - asm("crc32q %[value], %[crc]\n" : [crc] "+r" (crc) : [value] "rm" (value)); - return crc; -} - -__forceinline uint32 _mm_crc32_u32(uint32 crc, uint64 value) { - asm("crc32l %[value], %[crc]\n" : [crc] "+r" (crc) : [value] "rm" (value)); - return crc; -} - -__forceinline uint32 _mm_crc32_u8(uint32 crc, uint8 value) { - asm("crc32b %[value], %[crc]\n" : [crc] "+r" (crc) : [value] "rm" (value)); - return crc; -} - -} // namespace crcutil - -#endif - -#endif // CRCUTIL_USE_MM_CRC32 && (HAVE_I386 || HAVE_AMD64) - -#endif // CRCUTIL_CRC32C_SSE4_INTRIN_H_ diff --git a/rapidyenc/rapidyenc/crcutil-1.0/code/crc_casts.h b/rapidyenc/rapidyenc/crcutil-1.0/code/crc_casts.h deleted file mode 100644 index a14044f..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/code/crc_casts.h +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Casting between integers and compound CRC types. - -#ifndef CRCUTIL_CRC_CASTS_H_ -#define CRCUTIL_CRC_CASTS_H_ - -#include "base_types.h" // uint8, uint64 -#include "platform.h" // __forceinline - -namespace crcutil { - -// Downcasts a value of (oftentimes larger) Crc type to (smaller base integer) -// Result type, enabling specialized downcasts implemented by "large integer" -// classes (e.g. uint128_sse2). -template -__forceinline Result Downcast(const Crc &x) { - return static_cast(x); -} - -// Extracts 8 least significant bits from a value of Crc type. -#define TO_BYTE(x) Downcast(x) - -// Converts a pair of uint64 bit values into single value of CRC type. -// It is caller's responsibility to ensure that the input is correct. -template -__forceinline Crc CrcFromUint64(uint64 lo, uint64 hi = 0) { - if (sizeof(Crc) <= sizeof(lo)) { - return static_cast(lo); - } else { - // static_cast to keep compiler happy. - Crc result = static_cast(hi); - result = SHIFT_LEFT_SAFE(result, 8 * sizeof(lo)); - result ^= lo; - return result; - } -} - -// Converts Crc value to a pair of uint64 values. -template -__forceinline void Uint64FromCrc(const Crc &crc, - uint64 *lo, uint64 *hi = NULL) { - if (sizeof(*lo) >= sizeof(crc)) { - *lo = Downcast(crc); - if (hi != NULL) { - *hi = 0; - } - } else { - *lo = Downcast(crc); - *hi = Downcast(SHIFT_RIGHT_SAFE(crc, 8 * sizeof(lo))); - } -} - -} // namespace crcutil - -#endif // CRCUTIL_CRC_CASTS_H_ diff --git a/rapidyenc/rapidyenc/crcutil-1.0/code/generic_crc.h b/rapidyenc/rapidyenc/crcutil-1.0/code/generic_crc.h deleted file mode 100644 index 06af21c..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/code/generic_crc.h +++ /dev/null @@ -1,687 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Defines GenericCrc class which implements arbitrary CRCs. -// -// Please read crc.pdf to understand how it all works. - -#ifndef CRCUTIL_GENERIC_CRC_H_ -#define CRCUTIL_GENERIC_CRC_H_ - -#include "base_types.h" // uint8 -#include "crc_casts.h" // TO_BYTE(), Downcast<>. -#include "gf_util.h" // GfUtil class. -#include "platform.h" // GCC_ALIGN_ATTRIBUTE(16) -#include "uint128_sse2.h" // uint128_sse2 type (if necessary) - -namespace crcutil { - -#pragma pack(push, 16) - -// Extends CRC by one byte. -// Technically, if degree of a polynomial does not exceed 8, -// right shift by 8 bits is not required, but who cares about CRC-8? -#define CRC_BYTE(table, crc, byte) do { \ - crc = ((sizeof(crc) > 1) ? SHIFT_RIGHT_SAFE(crc, 8) : 0) ^ \ - table->crc_word_[sizeof(Word) - 1][TO_BYTE(crc) ^ (byte)]; \ -} while (0) - -#define TABLE_ENTRY(table, byte, buf) \ - table[byte][Downcast(buf)] - -#define TABLE_ENTRY_LAST(table, buf) \ - table[sizeof(Word) - 1][buf] - -// Extends CRC by one word. -#define CRC_WORD(table, crc, buf) do { \ - buf ^= Downcast(crc); \ - if (sizeof(crc) > sizeof(buf)) { \ - crc = SHIFT_RIGHT_SAFE(crc, sizeof(buf) * 8); \ - crc ^= TABLE_ENTRY(table->crc_word_, 0, buf); \ - } else { \ - crc = TABLE_ENTRY(table->crc_word_, 0, buf); \ - } \ - buf >>= 8; \ - for (size_t byte = 1; byte < sizeof(buf) - 1; ++byte) { \ - crc ^= TABLE_ENTRY(table->crc_word_, byte, buf); \ - buf >>= 8; \ - } \ - crc ^= TABLE_ENTRY_LAST(table->crc_word_, buf); \ -} while (0) - -// Process beginning of data block byte by byte until source pointer -// becomes perfectly aligned on Word boundary. -#define ALIGN_ON_WORD_BOUNDARY(table, src, end, crc, Word) do { \ - while ((reinterpret_cast(src) & (sizeof(Word) - 1)) != 0) { \ - if (src >= end) { \ - return (crc ^ table->Base().Canonize()); \ - } \ - CRC_BYTE(table, crc, *src); \ - src += 1; \ - } \ -} while (0) - - -// On amd64, enforcing alignment is 2-4% slower on small (<= 64 bytes) blocks -// but 6-10% faster on larger blocks (>= 2KB). -// Break-even point (+-1%) is around 1KB (Q9650, E6600). -// -#define ALIGN_ON_WORD_BOUNDARY_IF_NEEDED(bytes, table, src, end, crc, Word) \ -do { \ - if (sizeof(Word) > 8 || (bytes) > CRCUTIL_MIN_ALIGN_SIZE) { \ - ALIGN_ON_WORD_BOUNDARY(table, src, end, crc, Word); \ - } \ -} while (0) - -#if defined(_MSC_VER) -#pragma warning(push) -#pragma warning(disable: 4127) // conditional expression is constant -#endif // defined(_MSC_VER) - -// Forward declarations. -template class RollingCrc; - -// Crc is the type used internally and to return values of N-bit CRC. -// It should be at least as large as "TableEntry" and "Word" but -// may be larger (e.g. for 16-bit CRC, TableEntry and Word may be -// set to uint16 but Crc may be set to uint32). -// -// TableEntry is the type of values stored in the tables. -// To implement N-bit CRC, TableEntry should be large enough -// to store N bits. -// -// Word is the type used to read data sizeof(Word) at a time. -// Ideally, it shoulde be "most suitable for given architecture" -// integer type -- typically "size_t". -// -// kStride is the number of words processed in interleaved manner by -// CrcMultiword() and CrcWordblock(). Shall be either 3 or 4. -// Optimal value depends on hardware architecture (AMD64, ARM, etc). -// -template - class GenericCrc { - public: - // Make Crc, TableEntry, and Word types visible (used by RollingCrc etc.) - typedef _Crc Crc; - typedef _TableEntry TableEntry; - typedef _Word Word; - - GenericCrc() {} - - // Initializes the tables given generating polynomial of degree. - // If "canonical" is true, crc value will be XOR'ed with (-1) before and - // after actual CRC computation. - GenericCrc(const Crc &generating_polynomial, size_t degree, bool canonical) { - Init(generating_polynomial, degree, canonical); - } - void Init(const Crc &generating_polynomial, size_t degree, bool canonical) { - base_.Init(generating_polynomial, degree, canonical); - - // Instead of computing - // table[j][i] = MultiplyUnnormalized(i, 8, k), - // for all i = 0...255, we may notice that - // if i = 2**n then for all m = 1...(i-1) - // MultiplyUnnormalized(i + m, 8, k) = - // MultiplyUnnormalized(i ^ m, 8, k) = - // MultiplyUnnormalized(i, 8, k) ^ MultiplyUnnormalized(m, 8, k) = - // MultiplyUnnormalized(i, 8, k) ^ crc_word_interleaved[j][m] = - // table[i] ^ table[m]. -#if 0 - for (size_t j = 0; j < sizeof(Word); ++j) { - Crc k = Base().XpowN((sizeof(Word) * kStride - 1 - j) * 8 + degree); - for (size_t i = 0; i < 256; ++i) { - Crc temp = Base().MultiplyUnnormalized(static_cast(i), 8, k); - this->crc_word_interleaved_[j][i] = Downcast(temp); - } - } -#else - for (size_t j = 0; j < sizeof(Word); ++j) { - Crc k = Base().XpowN((sizeof(Word) * kStride - 1 - j) * 8 + degree); - TableEntry *table = this->crc_word_interleaved_[j]; - table[0] = 0; // Init 0s entry -- multiply 0 by anything yields 0. - for (size_t i = 1; i < 256; i <<= 1) { - TableEntry value = Downcast( - Base().MultiplyUnnormalized(static_cast(i), 8, k)); - table[i] = value; - for (size_t m = 1; m < i; ++m) { - table[i + m] = value ^ table[m]; - } - } - } -#endif - -#if 0 - for (size_t j = 0; j < sizeof(Word); ++j) { - Crc k = Base().XpowN((sizeof(Word) - 1 - j) * 8 + degree); - for (size_t i = 0; i < 256; ++i) { - Crc temp = Base().MultiplyUnnormalized(static_cast(i), 8, k); - this->crc_word_[j][i] = Downcast(temp); - } - } -#else - for (size_t j = 0; j < sizeof(Word); ++j) { - Crc k = Base().XpowN((sizeof(Word) - 1 - j) * 8 + degree); - TableEntry *table = this->crc_word_[j]; - table[0] = 0; // Init 0s entry -- multiply 0 by anything yields 0. - for (size_t i = 1; i < 256; i <<= 1) { - TableEntry value = Downcast( - Base().MultiplyUnnormalized(static_cast(i), 8, k)); - table[i] = value; - for (size_t m = 1; m < i; ++m) { - table[i + m] = value ^ table[m]; - } - } - } -#endif - } - - // Default CRC implementation - Crc CrcDefault(const void *data, size_t bytes, const Crc &start) const { -#if HAVE_AMD64 || HAVE_I386 - return CrcMultiword(data, bytes, start); -#else - // Very few CPUs have multiple ALUs and speculative execution - // (Itanium is an exception) so sophisticated algorithms will - // not perform better than good old Sarwate algorithm. - return CrcByteUnrolled(data, bytes, start); -#endif // HAVE_AMD64 || HAVE_I386 - } - - // Returns base class. - const GfUtil &Base() const { return base_; } - - protected: - // Canonical, byte-by-byte CRC computation. - Crc CrcByte(const void *data, size_t bytes, const Crc &start) const { - const uint8 *src = static_cast(data); - Crc crc = start ^ Base().Canonize(); - for (const uint8 *end = src + bytes; src < end; ++src) { - CRC_BYTE(this, crc, *src); - } - return (crc ^ Base().Canonize()); - } - - // Byte-by-byte CRC with main loop unrolled. - Crc CrcByteUnrolled(const void *data, size_t bytes, const Crc &start) const { - if (bytes == 0) { - return start; - } - - const uint8 *src = static_cast(data); - const uint8 *end = src + bytes; - Crc crc = start ^ Base().Canonize(); - - // Unroll loop 4 times. - end -= 3; - for (; src < end; src += 4) { - PREFETCH(src); - CRC_BYTE(this, crc, src[0]); - CRC_BYTE(this, crc, src[1]); - CRC_BYTE(this, crc, src[2]); - CRC_BYTE(this, crc, src[3]); - } - end += 3; - - // Compute CRC of remaining bytes. - for (; src < end; ++src) { - CRC_BYTE(this, crc, *src); - } - - return (crc ^ Base().Canonize()); - } - - // Canonical, byte-by-byte CRC computation. - Crc CrcByteWord(const void *data, size_t bytes, const Crc &start) const { - const uint8 *src = static_cast(data); - const uint8 *end = src + bytes; - Crc crc0 = start ^ Base().Canonize(); - - ALIGN_ON_WORD_BOUNDARY_IF_NEEDED(bytes, this, src, end, crc0, Crc); - if (src >= end) { - return (crc0 ^ Base().Canonize()); - } - - // Process 4*sizeof(Crc) bytes at a time. - end -= 4 * sizeof(Crc) - 1; - for (; src < end; src += 4 * sizeof(Crc)) { - for (size_t i = 0; i < 4; ++i) { - crc0 ^= reinterpret_cast(src)[i]; - if (i == 0) { - PREFETCH(src); - } - for (size_t byte = 0; byte < sizeof(crc0); ++byte) { - CRC_BYTE(this, crc0, 0); - } - } - } - end += 4 * sizeof(Crc) - 1; - - // Process sizeof(Crc) bytes at a time. - end -= sizeof(Crc) - 1; - for (; src < end; src += sizeof(Crc)) { - crc0 ^= reinterpret_cast(src)[0]; - for (size_t byte = 0; byte < sizeof(crc0); ++byte) { - CRC_BYTE(this, crc0, 0); - } - } - end += sizeof(Crc) - 1; - - // Compute CRC of remaining bytes. - for (;src < end; ++src) { - CRC_BYTE(this, crc0, *src); - } - - return (crc0 ^ Base().Canonize()); - } - - // Faster, word-by-word CRC. - Crc CrcWord(const void *data, size_t bytes, const Crc &start) const { - const uint8 *src = static_cast(data); - const uint8 *end = src + bytes; - Crc crc0 = start ^ Base().Canonize(); - - ALIGN_ON_WORD_BOUNDARY_IF_NEEDED(bytes, this, src, end, crc0, Word); - if (src >= end) { - return (crc0 ^ Base().Canonize()); - } - - // Process 4 sizeof(Word) bytes at once. - end -= 4 * sizeof(Word) - 1; - for (; src < end; src += 4 * sizeof(Word)) { - Word buf0 = reinterpret_cast(src)[0]; - PREFETCH(src); - CRC_WORD(this, crc0, buf0); - buf0 = reinterpret_cast(src)[1]; - CRC_WORD(this, crc0, buf0); - buf0 = reinterpret_cast(src)[2]; - CRC_WORD(this, crc0, buf0); - buf0 = reinterpret_cast(src)[3]; - CRC_WORD(this, crc0, buf0); - } - end += 4 * sizeof(Word) - 1; - - // Process sizeof(Word) bytes at a time. - end -= sizeof(Word) - 1; - for (; src < end; src += sizeof(Word)) { - Word buf0 = reinterpret_cast(src)[0]; - CRC_WORD(this, crc0, buf0); - } - end += sizeof(Word) - 1; - - // Compute CRC of remaining bytes. - for (;src < end; ++src) { - CRC_BYTE(this, crc0, *src); - } - - return (crc0 ^ Base().Canonize()); - } - -#define REPEAT_FROM_1(macro) \ - macro(1); \ - macro(2); \ - macro(3); \ - macro(4); \ - macro(5); \ - macro(6); \ - macro(7); - -#define REPEAT_FROM_0(macro) \ - macro(0); \ - REPEAT_FROM_1(macro) - - // Faster, process adjusent blocks in parallel and concatenate CRCs. - Crc CrcBlockword(const void *data, size_t bytes, const Crc &start) const { - if (kStride < 2 || kStride > 8) { - // Unsupported configuration; - // fall back to something sensible. - return CrcWord(data, bytes, start); - } - - const uint8 *src = static_cast(data); - const uint8 *end = src + bytes; - Crc crc0 = start ^ Base().Canonize(); - enum { - // Add 16 to avoid false L1 cache collisions. - kStripe = (15*1024 + 16) & ~(sizeof(Word) - 1), - }; - - ALIGN_ON_WORD_BOUNDARY_IF_NEEDED(bytes, this, src, end, crc0, Word); - if (src >= end) { - return (crc0 ^ Base().Canonize()); - } - - end -= kStride * kStripe - 1; - if (src < end) { - Crc x_pow_8kStripe = Base().Xpow8N(kStripe); - do { - const uint8 *stripe_end = src + kStripe; - -#define INIT_CRC(reg) \ - Crc crc##reg; \ - if (kStride >= reg) { \ - crc##reg = 0; \ - } - REPEAT_FROM_1(INIT_CRC); -#undef INIT_CRC - - do { -#define FIRST(reg) \ - Word buf##reg; \ - if (kStride > reg) { \ - buf##reg = reinterpret_cast(src + reg * kStripe)[0]; \ - buf##reg ^= Downcast(crc##reg); \ - if (sizeof(crc##reg) > sizeof(buf##reg)) { \ - crc##reg = SHIFT_RIGHT_SAFE(crc##reg, sizeof(buf##reg) * 8); \ - crc##reg ^= TABLE_ENTRY(this->crc_word_, 0, buf##reg); \ - } else { \ - crc##reg = TABLE_ENTRY(this->crc_word_, 0, buf##reg); \ - } \ - buf##reg >>= 8; \ - } - REPEAT_FROM_0(FIRST); -#undef FIRST - - for (size_t byte = 1; byte < sizeof(buf0) - 1; ++byte) { -#define NEXT(reg) do { \ - if (kStride > reg) { \ - crc##reg ^= TABLE_ENTRY(this->crc_word_, byte, buf##reg); \ - buf##reg >>= 8; \ - } \ -} while (0) - REPEAT_FROM_0(NEXT); -#undef NEXT - } - -#define LAST(reg) do { \ - if (kStride > reg) { \ - crc##reg ^= TABLE_ENTRY_LAST(this->crc_word_, buf##reg); \ - } \ -} while (0) - REPEAT_FROM_0(LAST); -#undef LAST - - src += sizeof(Word); - } while (src < stripe_end); - -#if 0 -// The code is left for illustrational purposes only. -#define COMBINE(reg) do { \ - if (reg > 0 && kStride > reg) { \ - crc0 = Base().ChangeStartValue(crc##reg, kStripe, 0, crc0); \ - } \ -} while (0) -#else -#define COMBINE(reg) do { \ - if (reg > 0 && kStride > reg) { \ - crc0 = crc##reg ^ Base().Multiply(crc0, x_pow_8kStripe); \ - } \ -} while (0) -#endif - REPEAT_FROM_0(COMBINE); -#undef COMBINE - - src += (kStride - 1) * kStripe; - } - while (src < end); - } - end += kStride * kStripe - 1; - - // Process sizeof(Word) bytes at a time. - end -= sizeof(Word) - 1; - for (; src < end; src += sizeof(Word)) { - Word buf0 = reinterpret_cast(src)[0]; - CRC_WORD(this, crc0, buf0); - } - end += sizeof(Word) - 1; - - // Compute CRC of remaining bytes. - for (;src < end; ++src) { - CRC_BYTE(this, crc0, *src); - } - - return (crc0 ^ Base().Canonize()); - } - - // Fastest, interleaved multi-byte CRC. - Crc CrcMultiword(const void *data, size_t bytes, const Crc &start) const { - if (kStride < 2 || kStride > 8) { - // Unsupported configuration; - // fall back to something sensible. - return CrcWord(data, bytes, start); - } - - const uint8 *src = static_cast(data); - const uint8 *end = src + bytes; - Crc crc0 = start ^ Base().Canonize(); - - ALIGN_ON_WORD_BOUNDARY_IF_NEEDED(bytes, this, src, end, crc0, Word); - if (src >= end) { - return (crc0 ^ Base().Canonize()); - } - - // Process kStride Word registers at once; - // should have have at least 2*kInterleaveBytes of data to start. - end -= 2*kInterleaveBytes - 1; - if (src < end) { - Crc crc_carryover; - if (sizeof(Crc) > sizeof(Word)) { - // crc_carryover is used if and only if Crc is wider than Word. - crc_carryover = 0; - } -#define INIT_CRC(reg) \ - Crc crc##reg; \ - if (reg > 0 && kStride > reg) { \ - crc##reg = 0; \ - } - REPEAT_FROM_1(INIT_CRC); -#undef INIT_CRC - -#define INIT_BUF(reg) \ - Word buf##reg; \ - if (kStride > reg) { \ - buf##reg = reinterpret_cast(src)[reg]; \ - } - REPEAT_FROM_0(INIT_BUF); -#undef INIT_BUF - - do { - PREFETCH(src); - src += kInterleaveBytes; - - if (sizeof(Crc) > sizeof(Word)) { - crc0 ^= crc_carryover; - } - -#define FIRST(reg, next_reg) do { \ - if (kStride > reg) { \ - buf##reg ^= Downcast(crc##reg); \ - if (sizeof(Crc) > sizeof(Word)) { \ - if (reg < kStride - 1) { \ - crc##next_reg ^= SHIFT_RIGHT_SAFE(crc##reg, 8 * sizeof(buf0)); \ - } else { \ - crc_carryover = SHIFT_RIGHT_SAFE(crc##reg, 8 * sizeof(buf0)); \ - } \ - } \ - crc##reg = TABLE_ENTRY(this->crc_word_interleaved_, 0, buf##reg); \ - buf##reg >>= 8; \ - } \ -} while (0) - FIRST(0, 1); - FIRST(1, 2); - FIRST(2, 3); - FIRST(3, 4); - FIRST(4, 5); - FIRST(5, 6); - FIRST(6, 7); - FIRST(7, 0); -#undef FIRST - - for (size_t byte = 1; byte < sizeof(Word) - 1; ++byte) { -#define NEXT(reg) do { \ - if (kStride > reg) { \ - crc##reg ^= \ - TABLE_ENTRY(this->crc_word_interleaved_, byte, buf##reg); \ - buf##reg >>= 8; \ - } \ -} while(0) - REPEAT_FROM_0(NEXT); -#undef NEXT - } - -#define LAST(reg) do { \ - if (kStride > reg) { \ - crc##reg ^= TABLE_ENTRY_LAST(this->crc_word_interleaved_, buf##reg); \ - buf##reg = reinterpret_cast(src)[reg]; \ - } \ -} while(0) - REPEAT_FROM_0(LAST); -#undef LAST - } - while (src < end); - - if (sizeof(Crc) > sizeof(Word)) { - crc0 ^= crc_carryover; - } - -#define COMBINE(reg) do { \ - if (kStride > reg) { \ - if (reg != 0) { \ - crc0 ^= crc##reg; \ - } \ - CRC_WORD(this, crc0, buf##reg); \ - } \ -} while (0) - REPEAT_FROM_0(COMBINE); -#undef COMBINE - - src += kInterleaveBytes; - } - end += 2*kInterleaveBytes - 1; - - // Process sizeof(Word) bytes at once. - end -= sizeof(Word) - 1; - for (; src < end; src += sizeof(Word)) { - Word buf0 = reinterpret_cast(src)[0]; - CRC_WORD(this, crc0, buf0); - } - end += sizeof(Word) - 1; - - // Compute CRC of remaining bytes. - for (;src < end; ++src) { - CRC_BYTE(this, crc0, *src); - } - - return (crc0 ^ Base().Canonize()); - } - - protected: - enum { - kInterleaveBytes = sizeof(Word) * kStride, - }; - - // Multiplication tables used by CRCs. - TableEntry crc_word_interleaved_[sizeof(Word)][256]; - TableEntry crc_word_[sizeof(Word)][256]; - - // Base class stored after CRC tables so that the most frequently - // used table is at offset 0 and may be accessed faster. - GfUtil base_; - - friend class RollingCrc< GenericCrc >; - - private: - // CrcMultiword on amd64 may run at 1.2 CPU cycles per byte which is - // noticeably faster than CrcWord (2.2-2.6 cycles/byte depending on - // hardware and compiler). However, there are problems with compilers. - // - // Test system: P45 chipset, Intel Q9650 CPU, 800MHz 4-4-4-12 memory. - // - // 64-bit compiler, <= 64-bit CRC, 64-bit tables, 64-bit reads: - // CL 15.00.307291.1 C++ >1.2< CPU cycles/byte - // ICL 11.1.051 -O3 C++ 1.5 CPU cycles/byte - // GCC 4.5 -O3 C++ 2.0 CPU cycles/byte - // GCC 4.x -O3 ASM >1.2< CPU cycles/byte - // - // 32-bit compiler, MMX used, <= 64-bit CRC, 64-bit tables, 64-bit reads - // CL 15.00.307291.1 C++ 2.0 CPU cycles/byte - // GCC 4.5 -O3 C++ 1.9 CPU cycles/byte - // ICL 11.1.051 -S C++ 1.6 CPU cycles/byte - // GCC 4.x -O3 ASM >1.3< CPU cycles/byte - // - // So, use inline ASM code for GCC for both i386 and amd64. - - Crc CrcMultiwordI386Mmx( - const void *data, size_t bytes, const Crc &start) const; - Crc CrcMultiwordGccAmd64( - const void *data, size_t bytes, const Crc &start) const; - Crc CrcMultiwordGccAmd64Sse2( - const uint8 *src, const uint8 *end, const Crc &start) const; -} GCC_ALIGN_ATTRIBUTE(16); - -#undef REPEAT_FROM_0 -#undef REPEAT_FROM_1 - - -// Specialized variants. -#if CRCUTIL_USE_ASM - -#if (defined(__GNUC__) && (HAVE_AMD64 || (HAVE_I386 && HAVE_MMX))) - -// Declare specialized functions. -template<> uint64 GenericCrc::CrcMultiword( - const void *data, size_t bytes, const uint64 &start) const; - -#if HAVE_AMD64 && HAVE_SSE2 -template<> -uint128_sse2 -GenericCrc::CrcMultiword( - const void *data, size_t bytes, const uint128_sse2 &start) const; -#endif // HAVE_AMD64 && HAVE_SSE2 - -#elif defined(_MSC_FULL_VER) && _MSC_FULL_VER <= 150030729 && \ - (HAVE_I386 && HAVE_MMX) - -// Work around bug in MSC (present at least in v. 15.00.30729.1) -template<> uint64 GenericCrc::CrcMultiwordI386Mmx( - const void *data, - size_t bytes, - const uint64 &start) const; -template<> __forceinline -uint64 GenericCrc::CrcMultiword( - const void *data, - size_t bytes, - const uint64 &start) const { - typedef uint64 Word; - typedef uint64 Crc; - if (bytes <= 12) { - const uint8 *src = static_cast(data); - uint64 crc = start ^ Base().Canonize(); - for (const uint8 *end = src + bytes; src < end; ++src) { - CRC_BYTE(this, crc, *src); - } - return (crc ^ Base().Canonize()); - } - return CrcMultiwordI386Mmx(data, bytes, start); -} - -#endif // (defined(__GNUC__) && (HAVE_AMD64 || (HAVE_I386 && HAVE_MMX))) - -#endif // CRCUTIL_USE_ASM - - -#pragma pack(pop) - -} // namespace crcutil - -#endif // CRCUTIL_GENERIC_CRC_H_ diff --git a/rapidyenc/rapidyenc/crcutil-1.0/code/gf_util.h b/rapidyenc/rapidyenc/crcutil-1.0/code/gf_util.h deleted file mode 100644 index 43c1d6b..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/code/gf_util.h +++ /dev/null @@ -1,304 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Defines GfUtil template class which implements -// 1. some useful operations in GF(2^n), -// 2. CRC helper function (e.g. concatenation of CRCs) which are -// not affected by specific implemenation of CRC computation per se. -// -// Please read crc.pdf to understand how it all works. - -#ifndef CRCUTIL_GF_UTIL_H_ -#define CRCUTIL_GF_UTIL_H_ - -#include "base_types.h" // uint8, uint64 -#include "crc_casts.h" // TO_BYTE() -#include "platform.h" // GCC_ALIGN_ATTRIBUTE(16), SHIFT_*_SAFE - -namespace crcutil { - -#pragma pack(push, 16) - -// "Crc" is the type used internally and to return values of N-bit CRC. -template class GfUtil { - public: - // Initializes the tables given generating polynomial of degree (degree). - // If "canonical" is true, starting CRC value and computed CRC value will be - // XOR-ed with 111...111. - GfUtil() {} - GfUtil(const Crc &generating_polynomial, size_t degree, bool canonical) { - Init(generating_polynomial, degree, canonical); - } - void Init(const Crc &generating_polynomial, size_t degree, bool canonical) { - Crc one = 1; - one <<= degree - 1; - this->generating_polynomial_ = generating_polynomial; - this->crc_bytes_ = (degree + 7) >> 3; - this->degree_ = degree; - this->one_ = one; - if (canonical) { - this->canonize_ = one | (one - 1); - } else { - this->canonize_ = 0; - } - this->normalize_[0] = 0; - this->normalize_[1] = generating_polynomial; - - Crc k = one >> 1; - for (size_t i = 0; i < sizeof(uint64) * 8; ++i) { - this->x_pow_2n_[i] = k; - k = Multiply(k, k); - } - - this->crc_of_crc_ = Multiply(this->canonize_, - this->one_ ^ Xpow8N((degree + 7) >> 3)); - - FindLCD(Xpow8N(this->crc_bytes_), &this->x_pow_minus_W_); - } - - // Returns generating polynomial. - Crc GeneratingPolynomial() const { - return this->generating_polynomial_; - } - - // Returns number of bits in CRC (degree of generating polynomial). - size_t Degree() const { - return this->degree_; - } - - // Returns start/finish adjustment constant. - Crc Canonize() const { - return this->canonize_; - } - - // Returns normalized value of 1. - Crc One() const { - return this->one_; - } - - // Returns value of CRC(A, |A|, start_new) given known - // crc=CRC(A, |A|, start_old) -- without touching the data. - Crc ChangeStartValue(const Crc &crc, uint64 bytes, - const Crc &start_old, - const Crc &start_new) const { - return (crc ^ Multiply(start_new ^ start_old, Xpow8N(bytes))); - } - - // Returns CRC of concatenation of blocks A and B when CRCs - // of blocks A and B are known -- without touching the data. - // - // To be precise, given CRC(A, |A|, startA) and CRC(B, |B|, 0), - // returns CRC(AB, |AB|, startA). - Crc Concatenate(const Crc &crc_A, const Crc &crc_B, uint64 bytes_B) const { - return ChangeStartValue(crc_B, bytes_B, 0 /* start_B */, crc_A); - } - - // Returns CRC of sequence of zeroes -- without touching the data. - Crc CrcOfZeroes(uint64 bytes, const Crc &start) const { - Crc tmp = Multiply(start ^ this->canonize_, Xpow8N(bytes)); - return (tmp ^ this->canonize_); - } - - // Given CRC of a message, stores extra (degree + 7)/8 bytes after - // the message so that CRC(message+extra, start) = result. - // Does not change CRC start value (use ChangeStartValue for that). - // Returns number of stored bytes. - size_t StoreComplementaryCrc(void *dst, - const Crc &message_crc, - const Crc &result) const { - Crc crc0 = Multiply(result ^ this->canonize_, this->x_pow_minus_W_); - crc0 ^= message_crc ^ this->canonize_; - uint8 *d = reinterpret_cast(dst); - for (size_t i = 0; i < this->crc_bytes_; ++i) { - d[i] = TO_BYTE(crc0); - crc0 >>= 8; - } - return this->crc_bytes_; - } - - // Stores given CRC of a message as (degree + 7)/8 bytes filled - // with 0s to the right. Returns number of stored bytes. - // CRC of the message and stored CRC is a constant value returned - // by CrcOfCrc() -- it does not depend on contents of the message. - size_t StoreCrc(void *dst, const Crc &crc) const { - uint8 *d = reinterpret_cast(dst); - Crc crc0 = crc; - for (size_t i = 0; i < this->crc_bytes_; ++i) { - d[i] = TO_BYTE(crc0); - crc0 >>= 8; - } - return this->crc_bytes_; - } - - // Returns expected CRC value of CRC(Message,CRC(Message)) - // when CRC is stored after the message. This value is fixed - // and does not depend on the message or CRC start value. - Crc CrcOfCrc() const { - return this->crc_of_crc_; - } - - // Returns ((a * b) mod P) where "a" and "b" are of degree <= (D-1). - Crc Multiply(const Crc &aa, const Crc &bb) const { - Crc a = aa; - Crc b = bb; - if ((a ^ (a - 1)) < (b ^ (b - 1))) { - Crc temp = a; - a = b; - b = temp; - } - - if (a == 0) { - return a; - } - - Crc product = 0; - Crc one = this->one_; - for (; a != 0; a <<= 1) { - if ((a & one) != 0) { - product ^= b; - a ^= one; - } - b = (b >> 1) ^ this->normalize_[Downcast(b & 1)]; - } - - return product; - } - - // Returns ((unnorm * m) mod P) where degree of m is <= (D-1) - // and degree of value "unnorm" is provided explicitly. - Crc MultiplyUnnormalized(const Crc &unnorm, size_t degree, - const Crc &m) const { - Crc v = unnorm; - Crc result = 0; - while (degree > this->degree_) { - degree -= this->degree_; - Crc value = v & (this->one_ | (this->one_ - 1)); - result ^= Multiply(value, Multiply(m, XpowN(degree))); - v >>= this->degree_; - } - result ^= Multiply(v << (this->degree_ - degree), m); - return result; - } - - // returns ((x ** n) mod P). - Crc XpowN(uint64 n) const { - Crc one = this->one_; - Crc result = one; - - for (size_t i = 0; n != 0; ++i, n >>= 1) { - if (n & 1) { - result = Multiply(result, this->x_pow_2n_[i]); - } - } - - return result; - } - - // Returns (x ** (8 * n) mod P). - Crc Xpow8N(uint64 n) const { - return XpowN(n << 3); - } - - // Returns remainder (A mod B) and sets *q = (A/B) of division - // of two polynomials: - // A = dividend + dividend_x_pow_D_coef * x**degree, - // B = divisor. - Crc Divide(const Crc ÷nd0, int dividend_x_pow_D_coef, - const Crc &divisor0, Crc *q) const { - Crc divisor = divisor0; - Crc dividend = dividend0; - Crc quotient = 0; - Crc coef = this->one_; - - while ((divisor & 1) == 0) { - divisor >>= 1; - coef >>= 1; - } - - if (dividend_x_pow_D_coef) { - quotient = coef >> 1; - dividend ^= divisor >> 1; - } - - Crc x_pow_degree_b = 1; - for (;;) { - if ((dividend & x_pow_degree_b) != 0) { - dividend ^= divisor; - quotient ^= coef; - } - if (coef == this->one_) { - break; - } - coef <<= 1; - x_pow_degree_b <<= 1; - divisor <<= 1; - } - - *q = quotient; - return dividend; - } - - // Extended Euclid's algorith -- for given A finds LCD(A, P) and - // value B such that (A * B) mod P = LCD(A, P). - Crc FindLCD(const Crc &A, Crc *B) const { - if (A == 0 || A == this->one_) { - *B = A; - return A; - } - - // Actually, generating polynomial is - // (generating_polynomial_ + x**degree). - int r0_x_pow_D_coef = 1; - Crc r0 = this->generating_polynomial_; - Crc b0 = 0; - Crc r1 = A; - Crc b1 = this->one_; - - for (;;) { - Crc q; - Crc r = Divide(r0, r0_x_pow_D_coef, r1, &q); - if (r == 0) { - break; - } - r0_x_pow_D_coef = 0; - - r0 = r1; - r1 = r; - - Crc b = b0 ^ Multiply(q, b1); - b0 = b1; - b1 = b; - } - - *B = b1; - return r1; - } - - protected: - Crc canonize_; - Crc x_pow_2n_[sizeof(uint64) * 8]; - Crc generating_polynomial_; - Crc one_; - Crc x_pow_minus_W_; - Crc crc_of_crc_; - Crc normalize_[2]; - size_t crc_bytes_; - size_t degree_; -} GCC_ALIGN_ATTRIBUTE(16); - -#pragma pack(pop) - -} // namespace crcutil - -#endif // CRCUTIL_GF_UTIL_H_ diff --git a/rapidyenc/rapidyenc/crcutil-1.0/code/multiword_128_64_gcc_amd64_sse2.cc b/rapidyenc/rapidyenc/crcutil-1.0/code/multiword_128_64_gcc_amd64_sse2.cc deleted file mode 100644 index f94fd1f..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/code/multiword_128_64_gcc_amd64_sse2.cc +++ /dev/null @@ -1,291 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Implements multiword CRC for GCC on i386. -// -// Small comment: the trick described in -// http://software.intel.com/en-us/articles/fast-simd-integer-move-for-the-intel-pentiumr-4-processor -// (replace "movdqa dst, src" with "pshufd $0xE4, src, dst") -// did not work: execution time increased from -// 1.8 CPU cycles/byte to 2.1 CPU cycles/byte. -// So it may be good idea on P4 but it's not on newer CPUs. -// -// movaps/xorps vs. movdqa/pxor did not make any difference. - -#include "generic_crc.h" -#include "uint128_sse2.h" - -#if defined(__GNUC__) && CRCUTIL_USE_ASM && HAVE_AMD64 && HAVE_SSE2 - -namespace crcutil { - -template<> uint128_sse2 -GenericCrc::CrcMultiwordGccAmd64Sse2( - const uint8 *src, const uint8 *end, const uint128_sse2 &start) const; - -template<> -uint128_sse2 GenericCrc::CrcMultiword( - const void *data, size_t bytes, const uint128_sse2 &start) const { - const uint8 *src = static_cast(data); - uint128_sse2 crc = start ^ this->Base().Canonize(); - const uint8 *end = src + bytes; - if (bytes <= 7) { - for (; src < end; ++src) { - CRC_BYTE(this, crc, *src); - } - return (crc ^ this->Base().Canonize()); - } - - ALIGN_ON_WORD_BOUNDARY_IF_NEEDED(bytes, this, src, end, crc, uint64); - if (src >= end) { - return (crc ^ this->Base().Canonize()); - } - - return CrcMultiwordGccAmd64Sse2(src, end, crc); -} - -#define CRC_WORD_ASM() \ - SSE2_MOVQ " %[crc0], %[tmp0]\n" \ - "xorq %[tmp0], %[buf0]\n" \ - "psrldq $8, %[crc0]\n" \ - "movzbq %b[buf0], %[tmp0]\n" \ - "shrq $8, %[buf0]\n" \ - "addq %[tmp0], %[tmp0]\n" \ - "pxor (%[table_word], %[tmp0], 8), %[crc0]\n" \ - "movzbq %b[buf0], %[tmp1]\n" \ - "shrq $8, %[buf0]\n" \ - "addq %[tmp1], %[tmp1]\n" \ - "pxor 1*256*16(%[table_word], %[tmp1], 8), %[crc0]\n" \ - "movzbq %b[buf0], %[tmp0]\n" \ - "shrq $8, %[buf0]\n" \ - "addq %[tmp0], %[tmp0]\n" \ - "pxor 2*256*16(%[table_word], %[tmp0], 8), %[crc0]\n" \ - "movzbq %b[buf0], %[tmp1]\n" \ - "shrq $8, %[buf0]\n" \ - "addq %[tmp1], %[tmp1]\n" \ - "pxor 3*256*16(%[table_word], %[tmp1], 8), %[crc0]\n" \ - "movzbq %b[buf0], %[tmp0]\n" \ - "shrq $8, %[buf0]\n" \ - "addq %[tmp0], %[tmp0]\n" \ - "pxor 4*256*16(%[table_word], %[tmp0], 8), %[crc0]\n" \ - "movzbq %b[buf0], %[tmp1]\n" \ - "shrq $8, %[buf0]\n" \ - "addq %[tmp1], %[tmp1]\n" \ - "pxor 5*256*16(%[table_word], %[tmp1], 8), %[crc0]\n" \ - "movzbq %b[buf0], %[tmp0]\n" \ - "shrq $8, %[buf0]\n" \ - "addq %[tmp0], %[tmp0]\n" \ - "pxor 6*256*16(%[table_word], %[tmp0], 8), %[crc0]\n" \ - "addq %[buf0], %[buf0]\n" \ - "pxor 7*256*16(%[table_word], %[buf0], 8), %[crc0]\n" - -template<> uint128_sse2 -GenericCrc::CrcMultiwordGccAmd64Sse2( - const uint8 *src, const uint8 *end, const uint128_sse2 &start) const { - __m128i crc0 = start; - __m128i crc1; - __m128i crc2; - __m128i crc3; - __m128i crc_carryover; - - uint64 buf0; - uint64 buf1; - uint64 buf2; - uint64 buf3; - - uint64 tmp0; - uint64 tmp1; - - asm( - "sub $2*4*8 - 1, %[end]\n" - "cmpq %[src], %[end]\n" - "jbe 2f\n" - - "pxor %[crc1], %[crc1]\n" - "pxor %[crc2], %[crc2]\n" - "pxor %[crc3], %[crc3]\n" - "pxor %[crc_carryover], %[crc_carryover]\n" - "movq (%[src]), %[buf0]\n" - "movq 1*8(%[src]), %[buf1]\n" - "movq 2*8(%[src]), %[buf2]\n" - "movq 3*8(%[src]), %[buf3]\n" - - "1:\n" -#if HAVE_SSE && CRCUTIL_PREFETCH_WIDTH > 0 - "prefetcht0 " TO_STRING(CRCUTIL_PREFETCH_WIDTH) "(%[src])\n" -#endif -#if GCC_VERSION_AVAILABLE(4, 5) - // Bug in GCC 4.2.4? - "add $4*8, %[src]\n" -#else - "lea 4*8(%[src]), %[src]\n" -#endif - "pxor %[crc_carryover], %[crc0]\n" - - SSE2_MOVQ " %[crc0], %[tmp0]\n" - "psrldq $8, %[crc0]\n" - "xorq %[tmp0], %[buf0]\n" - "movzbq %b[buf0], %[tmp0]\n" - "pxor %[crc0], %[crc1]\n" - "addq %[tmp0], %[tmp0]\n" - "shrq $8, %[buf0]\n" - "movdqa (%[table], %[tmp0], 8), %[crc0]\n" - - SSE2_MOVQ " %[crc1], %[tmp1]\n" - "psrldq $8, %[crc1]\n" - "xorq %[tmp1], %[buf1]\n" - "movzbq %b[buf1], %[tmp1]\n" - "pxor %[crc1], %[crc2]\n" - "addq %[tmp1], %[tmp1]\n" - "shrq $8, %[buf1]\n" - "movdqa (%[table], %[tmp1], 8), %[crc1]\n" - - SSE2_MOVQ " %[crc2], %[tmp0]\n" - "psrldq $8, %[crc2]\n" - "xorq %[tmp0], %[buf2]\n" - "movzbq %b[buf2], %[tmp0]\n" - "pxor %[crc2], %[crc3]\n" - "addq %[tmp0], %[tmp0]\n" - "shrq $8, %[buf2]\n" - "movdqa (%[table], %[tmp0], 8), %[crc2]\n" - - SSE2_MOVQ " %[crc3], %[tmp1]\n" - "psrldq $8, %[crc3]\n" - "xorq %[tmp1], %[buf3]\n" - "movzbq %b[buf3], %[tmp1]\n" - "movdqa %[crc3], %[crc_carryover]\n" - "addq %[tmp1], %[tmp1]\n" - "shrq $8, %[buf3]\n" - "movdqa (%[table], %[tmp1], 8), %[crc3]\n" - -#define XOR(byte) \ - "movzbq %b[buf0], %[tmp0]\n" \ - "shrq $8, %[buf0]\n" \ - "addq %[tmp0], %[tmp0]\n" \ - "pxor " #byte "*256*16(%[table], %[tmp0], 8), %[crc0]\n" \ - "movzbq %b[buf1], %[tmp1]\n" \ - "shrq $8, %[buf1]\n" \ - "addq %[tmp1], %[tmp1]\n" \ - "pxor " #byte "*256*16(%[table], %[tmp1], 8), %[crc1]\n" \ - "movzbq %b[buf2], %[tmp0]\n" \ - "shrq $8, %[buf2]\n" \ - "addq %[tmp0], %[tmp0]\n" \ - "pxor " #byte "*256*16(%[table], %[tmp0], 8), %[crc2]\n" \ - "movzbq %b[buf3], %[tmp1]\n" \ - "shrq $8, %[buf3]\n" \ - "addq %[tmp1], %[tmp1]\n" \ - "pxor " #byte "*256*16(%[table], %[tmp1], 8), %[crc3]\n" - - XOR(1) - XOR(2) - XOR(3) - XOR(4) - XOR(5) - XOR(6) -#undef XOR - - "addq %[buf0], %[buf0]\n" - "pxor 7*256*16(%[table], %[buf0], 8), %[crc0]\n" - "movq (%[src]), %[buf0]\n" - - "addq %[buf1], %[buf1]\n" - "pxor 7*256*16(%[table], %[buf1], 8), %[crc1]\n" - "movq 1*8(%[src]), %[buf1]\n" - - "addq %[buf2], %[buf2]\n" - "pxor 7*256*16(%[table], %[buf2], 8), %[crc2]\n" - "movq 2*8(%[src]), %[buf2]\n" - - "addq %[buf3], %[buf3]\n" - "pxor 7*256*16(%[table], %[buf3], 8), %[crc3]\n" - "movq 3*8(%[src]), %[buf3]\n" - - "cmpq %[src], %[end]\n" - "ja 1b\n" - - "pxor %[crc_carryover], %[crc0]\n" - CRC_WORD_ASM() - - "pxor %[crc1], %[crc0]\n" - "movq %[buf1], %[buf0]\n" - CRC_WORD_ASM() - - "pxor %[crc2], %[crc0]\n" - "movq %[buf2], %[buf0]\n" - CRC_WORD_ASM() - - "pxor %[crc3], %[crc0]\n" - "movq %[buf3], %[buf0]\n" - CRC_WORD_ASM() - - "add $4*8, %[src]\n" - "2:\n" - - "add $2*4*8 - 8, %[end]\n" - - "cmpq %[src], %[end]\n" - "jbe 4f\n" - "3:\n" - "movq (%[src]), %[buf0]\n" - "addq $8, %[src]\n" - CRC_WORD_ASM() - "cmpq %[src], %[end]\n" - "ja 3b\n" - - "4:\n" - "add $7, %[end]\n" - - "cmpq %[src], %[end]\n" - "jbe 6f\n" - - "5:\n" - "movzbq (%[src]), %[buf0]\n" - "add $1, %[src]\n" - SSE2_MOVQ " %[crc0], %[tmp0]\n" - "movzx %b[tmp0], %[tmp0]\n" - "psrldq $1, %[crc0]\n" - "xor %[buf0], %[tmp0]\n" - "addq %[tmp0], %[tmp0]\n" - "pxor 7*256*16(%[table_word], %[tmp0], 8), %[crc0]\n" - - "cmpq %[src], %[end]\n" - "ja 5b\n" - - "6:\n" - - : // outputs - [src] "+r" (src), - [end] "+r" (end), - [crc0] "+x" (crc0), - [crc1] "=&x" (crc1), - [crc2] "=&x" (crc2), - [crc3] "=&x" (crc3), - [crc_carryover] "=&x" (crc_carryover), - [buf0] "=&r" (buf0), - [buf1] "=&r" (buf1), - [buf2] "=&r" (buf2), - [buf3] "=&r" (buf3), - [tmp0] "=&r" (tmp0), - [tmp1] "=&r" (tmp1) - - : // inputs - [table_word] "r" (this->crc_word_), - [table] "r" (this->crc_word_interleaved_)); - - return (this->Base().Canonize() ^ crc0); -} - -} // namespace crcutil - -#endif // defined(__GNUC__) && CRCUTIL_USE_ASM && HAVE_AMD64 && HAVE_SSE2 diff --git a/rapidyenc/rapidyenc/crcutil-1.0/code/multiword_64_64_cl_i386_mmx.cc b/rapidyenc/rapidyenc/crcutil-1.0/code/multiword_64_64_cl_i386_mmx.cc deleted file mode 100644 index af7352a..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/code/multiword_64_64_cl_i386_mmx.cc +++ /dev/null @@ -1,304 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Implements 64-bit multiword CRC for Microsoft and Intel compilers -// using MMX instructions (i386). - -#include "generic_crc.h" - -#if CRCUTIL_USE_ASM && HAVE_I386 && HAVE_MMX && defined(_MSC_VER) - -namespace crcutil { - -#define CRC_WORD_MMX() \ - __asm pxor BUF0, CRC0 \ - __asm movd TMP0, BUF0 \ - __asm psrlq BUF0, 32 \ - __asm movzx TEMP, TMP0L \ - __asm shr TMP0, 8 \ - __asm movq CRC0, [TABLE + TEMP * 8] \ - __asm movzx TEMP, TMP0L \ - __asm shr TMP0, 8 \ - __asm pxor CRC0, [TABLE + TEMP * 8 + 1 * 256 * 8] \ - __asm movzx TEMP, TMP0L \ - __asm shr TMP0, 8 \ - __asm pxor CRC0, [TABLE + TEMP * 8 + 2 * 256 * 8] \ - __asm pxor CRC0, [TABLE + TMP0 * 8 + 3 * 256 * 8] \ - __asm movd TMP0, BUF0 \ - __asm movzx TEMP, TMP0L \ - __asm shr TMP0, 8 \ - __asm pxor CRC0, [TABLE + TEMP * 8 + 4 * 256 * 8] \ - __asm movzx TEMP, TMP0L \ - __asm shr TMP0, 8 \ - __asm pxor CRC0, [TABLE + TEMP * 8 + 5 * 256 * 8] \ - __asm movzx TEMP, TMP0L \ - __asm shr TMP0, 8 \ - __asm pxor CRC0, [TABLE + TEMP * 8 + 6 * 256 * 8] \ - __asm pxor CRC0, [TABLE + TMP0 * 8 + 7 * 256 * 8] - -// frame pointer register 'ebp' modified by inline assembly code -#pragma warning(disable: 4731) - -template<> uint64 GenericCrc::CrcMultiwordI386Mmx( - const void *data, - size_t bytes, - const uint64 &start) const { - const uint8 *src = static_cast(data); - const uint8 *end = src + bytes; - uint64 crc0 = start ^ this->Base().Canonize(); - - ALIGN_ON_WORD_BOUNDARY_IF_NEEDED(bytes, this, src, end, crc0, uint64); - if (src >= end) { - return (crc0 ^ this->Base().Canonize()); - } - -#define CRC0 mm0 -#define CRC1 mm1 -#define CRC2 mm2 -#define CRC3 mm3 -#define BUF0 mm4 -#define BUF1 mm5 -#define BUF2 mm6 -#define BUF3 mm7 -#define TMP0 eax -#define TMP0L al -#define TMP0H ah -#define TMP1 ebx -#define TMP1L bl -#define TMP1H bh -#define TMP2 ecx -#define TMP2L cl -#define TMP2H ch -#define TMP3 edx -#define TMP3L dl -#define TMP3H dh -#define TEMP edi -#define SRC esi -#define END [esp] -#define TABLE ebp - - - const uint64 *interleaved_table_address = - &this->crc_word_interleaved_[0][0]; - const uint64 *word_table_address = &this->crc_word_[0][0]; - - __asm { - push ebp - - mov TMP0, interleaved_table_address - - movq CRC0, crc0 - mov SRC, src - mov TMP1, end - sub TMP1, 2*4*8 - 1 - cmp SRC, TMP1 - mov TABLE, word_table_address - jae end_main_loop - - push TABLE - mov TABLE, TMP0 - push TMP1 - - pxor CRC1, CRC1 - pxor CRC2, CRC2 - pxor CRC3, CRC3 - - movq BUF0, [SRC] - movq BUF1, [SRC + 1 * 8] - movq BUF2, [SRC + 2 * 8] - movq BUF3, [SRC + 3 * 8] - - main_loop: -#if HAVE_SSE && CRCUTIL_PREFETCH_WIDTH > 0 - prefetcht0 [SRC + CRCUTIL_PREFETCH_WIDTH] -#endif - add SRC, 32 - pxor BUF0, CRC0 - pxor BUF1, CRC1 - pxor BUF2, CRC2 - pxor BUF3, CRC3 - - movd TMP0, BUF0 - psrlq BUF0, 32 - movd TMP1, BUF1 - psrlq BUF1, 32 - movd TMP2, BUF2 - psrlq BUF2, 32 - movd TMP3, BUF3 - psrlq BUF3, 32 - - movzx TEMP, TMP0L - movq CRC0, [TABLE + TEMP * 8] - movzx TEMP, TMP1L - movq CRC1, [TABLE + TEMP * 8] - movzx TEMP, TMP2L - movq CRC2, [TABLE + TEMP * 8] - movzx TEMP, TMP3L - movq CRC3, [TABLE + TEMP * 8] - - movzx TEMP, TMP0H - shr TMP0, 16 - pxor CRC0, [TABLE + TEMP * 8 + 1 * 256 * 8] - movzx TEMP, TMP1H - shr TMP1, 16 - pxor CRC1, [TABLE + TEMP * 8 + 1 * 256 * 8] - movzx TEMP, TMP2H - shr TMP2, 16 - pxor CRC2, [TABLE + TEMP * 8 + 1 * 256 * 8] - movzx TEMP, TMP3H - shr TMP3, 16 - pxor CRC3, [TABLE + TEMP * 8 + 1 * 256 * 8] - - movzx TEMP, TMP0L - shr TMP0, 8 - pxor CRC0, [TABLE + TEMP * 8 + 2 * 256 * 8] - movzx TEMP, TMP1L - shr TMP1, 8 - pxor CRC1, [TABLE + TEMP * 8 + 2 * 256 * 8] - movzx TEMP, TMP2L - shr TMP2, 8 - pxor CRC2, [TABLE + TEMP * 8 + 2 * 256 * 8] - movzx TEMP, TMP3L - shr TMP3, 8 - pxor CRC3, [TABLE + TEMP * 8 + 2 * 256 * 8] - - pxor CRC0, [TABLE + TMP0 * 8 + 3 * 256 * 8] - movd TMP0, BUF0 - pxor CRC1, [TABLE + TMP1 * 8 + 3 * 256 * 8] - movd TMP1, BUF1 - pxor CRC2, [TABLE + TMP2 * 8 + 3 * 256 * 8] - movd TMP2, BUF2 - pxor CRC3, [TABLE + TMP3 * 8 + 3 * 256 * 8] - movd TMP3, BUF3 - - movzx TEMP, TMP0L - pxor CRC0, [TABLE + TEMP * 8 + 4 * 256 * 8] - movzx TEMP, TMP1L - pxor CRC1, [TABLE + TEMP * 8 + 4 * 256 * 8] - movzx TEMP, TMP2L - pxor CRC2, [TABLE + TEMP * 8 + 4 * 256 * 8] - movzx TEMP, TMP3L - pxor CRC3, [TABLE + TEMP * 8 + 4 * 256 * 8] - - movzx TEMP, TMP0H - shr TMP0, 16 - pxor CRC0, [TABLE + TEMP * 8 + 5 * 256 * 8] - movzx TEMP, TMP1H - shr TMP1, 16 - pxor CRC1, [TABLE + TEMP * 8 + 5 * 256 * 8] - movzx TEMP, TMP2H - shr TMP2, 16 - pxor CRC2, [TABLE + TEMP * 8 + 5 * 256 * 8] - movzx TEMP, TMP3H - shr TMP3, 16 - pxor CRC3, [TABLE + TEMP * 8 + 5 * 256 * 8] - - movzx TEMP, TMP0L - shr TMP0, 8 - pxor CRC0, [TABLE + TEMP * 8 + 6 * 256 * 8] - movzx TEMP, TMP1L - shr TMP1, 8 - pxor CRC1, [TABLE + TEMP * 8 + 6 * 256 * 8] - movzx TEMP, TMP2L - shr TMP2, 8 - pxor CRC2, [TABLE + TEMP * 8 + 6 * 256 * 8] - movzx TEMP, TMP3L - shr TMP3, 8 - pxor CRC3, [TABLE + TEMP * 8 + 6 * 256 * 8] - - pxor CRC0, [TABLE + TMP0 * 8 + 7 * 256 * 8] - movq BUF0, [SRC] - pxor CRC1, [TABLE + TMP1 * 8 + 7 * 256 * 8] - movq BUF1, [SRC + 1 * 8] - pxor CRC2, [TABLE + TMP2 * 8 + 7 * 256 * 8] - movq BUF2, [SRC + 2 * 8] - pxor CRC3, [TABLE + TMP3 * 8 + 7 * 256 * 8] - movq BUF3, [SRC + 3 * 8] - - cmp END, SRC - ja main_loop - -#undef END -#define END TMP1 - pop END - pop TABLE - add SRC, 32 - - CRC_WORD_MMX() - - pxor BUF1, CRC1 - movq BUF0, BUF1 - CRC_WORD_MMX() - - pxor BUF2, CRC2 - movq BUF0, BUF2 - CRC_WORD_MMX() - - pxor BUF3, CRC3 - movq BUF0, BUF3 - CRC_WORD_MMX() - - end_main_loop: - add END, 2*4*8 - 8 - cmp SRC, END - jae end_word_loop - - word_loop: - movq BUF0, [SRC] - add SRC, 8 - CRC_WORD_MMX() - cmp END, SRC - ja word_loop - end_word_loop: - -#if 0 // Plain C version is faster? - add END, 7 - cmp SRC, END - jae end_byte_loop - - byte_loop: - movd TMP0, CRC0 - movzx TEMP, byte ptr [SRC] - movzx TMP0, TMP0L - psrlq CRC0, 8 - xor TEMP, TMP0 - add SRC, 1 - pxor CRC0, [TABLE + TEMP*8 + 7*256*8] - cmp END, SRC - ja byte_loop - end_byte_loop: -#endif - - pop ebp - - mov src, SRC - movq crc0, CRC0 - - emms - } - -#if 1 - // Compute CRC of remaining bytes. - for (;src < end; ++src) { - CRC_BYTE(this, crc0, *src); - } -#endif - - return (crc0 ^ this->Base().Canonize()); -} - - -} // namespace crcutil - -#endif // CRCUTIL_USE_ASM && HAVE_I386 && HAVE_MMX && defined(_MSC_VER) diff --git a/rapidyenc/rapidyenc/crcutil-1.0/code/multiword_64_64_gcc_amd64_asm.cc b/rapidyenc/rapidyenc/crcutil-1.0/code/multiword_64_64_gcc_amd64_asm.cc deleted file mode 100644 index a697b27..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/code/multiword_64_64_gcc_amd64_asm.cc +++ /dev/null @@ -1,298 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Implements multiword CRC for GCC on AMD64. -// -// Accoding to "Software Optimization Guide for AMD Family 10h Processors" -// http://www.amd.com/us-en/assets/content_type/white_papers_and_tech_docs/40546.pdf -// instead of -// movzbq %al, %rsi -// shrq $8, %rax -// [use %rsi] -// movzbq %al, %rsi -// shrq $8, %rax -// [use %rsi] -// it is better to use 32-bit registers -// (high 32 bits will be cleared on assignment), i.e. -// movzbl %al, %esi -// [use %rsi] -// movzbl %ah, %esi -// shrq $16, %rax -// [use %rsi] -// Makes instructions shorter and removes one shift -// (the latter is not such a big deal as it's execution time -// is nicely masked by [use %rsi] instruction). -// -// Performance difference: -// About 10% degradation on bytes = 8 .. 16 -// (clobbering registers that should be saved) -// Break even at 32 bytes. -// 3% improvement starting from 64 bytes. - -#include "generic_crc.h" - -#if defined(__GNUC__) && CRCUTIL_USE_ASM && HAVE_AMD64 - -namespace crcutil { - -template<> uint64 GenericCrc::CrcMultiwordGccAmd64( - const void *data, size_t bytes, const uint64 &start) const; - -template<> uint64 GenericCrc::CrcMultiword( - const void *data, - size_t bytes, - const uint64 &start) const { - if (bytes <= 6 * sizeof(Word) - 1) { - const uint8 *src = static_cast(data); - uint64 crc = start ^ this->Base().Canonize(); - const uint8 *end = src + bytes; -#define PROCESS_ONE_WORD() do { \ - Word buf = reinterpret_cast(src)[0]; \ - CRC_WORD(this, crc, buf); \ - src += sizeof(Word); \ -} while (0) - if (bytes >= 1 * sizeof(Word)) { - PROCESS_ONE_WORD(); - if (bytes >= 2 * sizeof(Word)) { - PROCESS_ONE_WORD(); - if (bytes >= 3 * sizeof(Word)) { - PROCESS_ONE_WORD(); - if (bytes >= 4 * sizeof(Word)) { - PROCESS_ONE_WORD(); - if (bytes >= 5 * sizeof(Word)) { - PROCESS_ONE_WORD(); - } - } - } - } - } - for (; src < end; ++src) { - CRC_BYTE(this, crc, *src); - } - return (crc ^ this->Base().Canonize()); - } - return this->CrcMultiwordGccAmd64(data, bytes, start); -} - -#define TMP0 "%%rsi" -#define TMP0W "%%esi" - -#define BUF0 "%%rax" -#define BUF0L "%%al" -#define BUF0H "%%ah" - -#define BUF1 "%%rbx" -#define BUF1L "%%bl" -#define BUF1H "%%bh" - -#define BUF2 "%%rcx" -#define BUF2L "%%cl" -#define BUF2H "%%ch" - -#define BUF3 "%%rdx" -#define BUF3L "%%dl" -#define BUF3H "%%dh" - -#define CRC_WORD_ASM() \ - "xorq %[crc0], " BUF0 "\n" \ - "movzbq " BUF0L ", " TMP0 "\n" \ - "movq (%[table_word], " TMP0 ", 8), %[crc0]\n" \ - "movzbl " BUF0H ", " TMP0W "\n" \ - "shrq $16, " BUF0 "\n" \ - "xorq 1*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" \ - "movzbq " BUF0L ", " TMP0 "\n" \ - "xorq 2*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" \ - "movzbl " BUF0H ", " TMP0W "\n" \ - "shrq $16, " BUF0 "\n" \ - "xorq 3*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" \ - "movzbq " BUF0L ", " TMP0 "\n" \ - "xorq 4*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" \ - "movzbl " BUF0H ", " TMP0W "\n" \ - "shrq $16, " BUF0 "\n" \ - "xorq 5*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" \ - "movzbq " BUF0L ", " TMP0 "\n" \ - "xorq 6*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" \ - "movzbl " BUF0H ", " TMP0W "\n" \ - "xorq 7*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" - -template<> uint64 GenericCrc::CrcMultiwordGccAmd64( - const void *data, size_t bytes, const uint64 &start) const { - const uint8 *src = static_cast(data); - const uint8 *end = src + bytes; - uint64 crc0 = start ^ this->Base().Canonize(); - - ALIGN_ON_WORD_BOUNDARY_IF_NEEDED(bytes, this, src, end, crc0, uint64); - if (src >= end) { - return (crc0 ^ this->Base().Canonize()); - } - - uint64 crc1; - uint64 crc2; - uint64 crc3; - - asm( - "subq $2*4*8 - 1, %[end]\n" - "cmpq %[src], %[end]\n" - "jbe 2f\n" - "xorq %[crc1], %[crc1]\n" - "movq (%[src]), " BUF0 "\n" - "movq 1*8(%[src]), " BUF1 "\n" - "movq 2*8(%[src]), " BUF2 "\n" - "movq 3*8(%[src]), " BUF3 "\n" - "movq %[crc1], %[crc2]\n" - "movq %[crc1], %[crc3]\n" - - "1:\n" -#if HAVE_SSE && CRCUTIL_PREFETCH_WIDTH > 0 - "prefetcht0 " TO_STRING(CRCUTIL_PREFETCH_WIDTH) "(%[src])\n" -#endif // HAVE_SSE - "addq $4*8, %[src]\n" - - // Set buffer data. - "xorq %[crc0], " BUF0 "\n" - "xorq %[crc1], " BUF1 "\n" - "xorq %[crc2], " BUF2 "\n" - "xorq %[crc3], " BUF3 "\n" - - // LOAD crc of byte 0 and shift buffers. - "movzbl " BUF0L ", " TMP0W "\n" - "movq (%[table], " TMP0 ", 8), %[crc0]\n" - "movzbl " BUF1L ", " TMP0W "\n" - "movq (%[table], " TMP0 ", 8), %[crc1]\n" - "movzbl " BUF2L ", " TMP0W "\n" - "movq (%[table], " TMP0 ", 8), %[crc2]\n" - "movzbl " BUF3L ", " TMP0W "\n" - "movq (%[table], " TMP0 ", 8), %[crc3]\n" - -#define XOR1(byte1) \ - "movzbl " BUF0L ", " TMP0W "\n" \ - "xorq " #byte1 "*256*8(%[table], " TMP0 ", 8), %[crc0]\n" \ - "movzbl " BUF1L ", " TMP0W "\n" \ - "xorq " #byte1 "*256*8(%[table], " TMP0 ", 8), %[crc1]\n" \ - "movzbl " BUF2L ", " TMP0W "\n" \ - "xorq " #byte1 "*256*8(%[table], " TMP0 ", 8), %[crc2]\n" \ - "movzbl " BUF3L ", " TMP0W "\n" \ - "xorq " #byte1 "*256*8(%[table], " TMP0 ", 8), %[crc3]\n" - -#define XOR2(byte2) \ - "movzbl " BUF0H ", " TMP0W "\n" \ - "shrq $16, " BUF0 "\n" \ - "xorq " #byte2 "*256*8(%[table], " TMP0 ", 8), %[crc0]\n" \ - "movzbl " BUF1H ", " TMP0W "\n" \ - "shrq $16, " BUF1 "\n" \ - "xorq " #byte2 "*256*8(%[table], " TMP0 ", 8), %[crc1]\n" \ - "movzbl " BUF2H ", " TMP0W "\n" \ - "shrq $16, " BUF2 "\n" \ - "xorq " #byte2 "*256*8(%[table], " TMP0 ", 8), %[crc2]\n" \ - "movzbl " BUF3H ", " TMP0W "\n" \ - "shrq $16, " BUF3 "\n" \ - "xorq " #byte2 "*256*8(%[table], " TMP0 ", 8), %[crc3]\n" - - XOR2(1) - XOR1(2) - XOR2(3) - XOR1(4) - XOR2(5) - XOR1(6) - - // Update CRC registers and load buffers. - "movzbl " BUF0H ", " TMP0W "\n" - "xorq 7*256*8(%[table], " TMP0 ", 8), %[crc0]\n" - "movq (%[src]), " BUF0 "\n" - "movzbl " BUF1H ", " TMP0W "\n" - "xorq 7*256*8(%[table], " TMP0 ", 8), %[crc1]\n" - "movq 1*8(%[src]), " BUF1 "\n" - "movzbl " BUF2H ", " TMP0W "\n" - "xorq 7*256*8(%[table], " TMP0 ", 8), %[crc2]\n" - "movq 2*8(%[src]), " BUF2 "\n" - "movzbl " BUF3H ", " TMP0W "\n" - "xorq 7*256*8(%[table], " TMP0 ", 8), %[crc3]\n" - "movq 3*8(%[src]), " BUF3 "\n" - - "cmpq %[src], %[end]\n" - "ja 1b\n" - - CRC_WORD_ASM() - - "xorq %[crc1], " BUF1 "\n" - "movq " BUF1 ", " BUF0 "\n" - CRC_WORD_ASM() - - "xorq %[crc2], " BUF2 "\n" - "movq " BUF2 ", " BUF0 "\n" - CRC_WORD_ASM() - - "xorq %[crc3], " BUF3 "\n" - "movq " BUF3 ", " BUF0 "\n" - CRC_WORD_ASM() - - "addq $4*8, %[src]\n" - - "2:\n" - "addq $2*4*8 - 8, %[end]\n" - "cmpq %[src], %[end]\n" - "jbe 4f\n" - - "3:\n" - "movq (%[src]), " BUF0 "\n" - "addq $8, %[src]\n" - CRC_WORD_ASM() - "cmpq %[src], %[end]\n" - "ja 3b\n" - - "4:\n" - "addq $7, %[end]\n" - - "cmpq %[src], %[end]\n" - "jbe 6f\n" - - "5:\n" - "movzbq (%[src]), " BUF0 "\n" - "movzbq %b[crc0], " TMP0 "\n" - "shrq $8, %[crc0]\n" - "xorq " BUF0 ", " TMP0 "\n" - "addq $1, %[src]\n" - "xorq 7*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" - "cmpq %[src], %[end]\n" - "ja 5b\n" - - "6:\n" - - - : // outputs - [src] "+r" (src), - [end] "+r" (end), - [crc0] "+r" (crc0), - [crc1] "=&r" (crc1), - [crc2] "=&r" (crc2), - [crc3] "=&r" (crc3) - - : // inputs - [table] "r" (&this->crc_word_interleaved_[0][0]), - [table_word] "r" (&this->crc_word_[0][0]) - - : // clobbers - "%rax", // BUF0 - "%rbx", // BUF1 - "%rcx", // BUF2 - "%rdx", // BUF3 - "%rsi" // TMP0 - ); - - return (crc0 ^ this->Base().Canonize()); -} - -} // namespace crcutil - -#endif // defined(__GNUC__) && HAVE_AMD64 && CRCUTIL_USE_ASM diff --git a/rapidyenc/rapidyenc/crcutil-1.0/code/multiword_64_64_gcc_i386_mmx.cc b/rapidyenc/rapidyenc/crcutil-1.0/code/multiword_64_64_gcc_i386_mmx.cc deleted file mode 100644 index 0ffda3b..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/code/multiword_64_64_gcc_i386_mmx.cc +++ /dev/null @@ -1,284 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Implements multiword CRC for GCC on i386. - -#include "generic_crc.h" - -#if defined(__GNUC__) && !defined(__clang__) && CRCUTIL_USE_ASM && HAVE_I386 && HAVE_MMX - -#if defined(__PIC__) && __GNUC__ < 5 -/* workaround for issue with PIC reserving ebx: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54232 */ -/* a little dangerous otherwise, since the compiler could allocate ebx to something else, so we only do it for GCC < 5.0 */ -#define PIC_WORKAROUND -#endif - -namespace crcutil { - -template<> uint64 GenericCrc::CrcMultiwordI386Mmx( - const void *data, size_t bytes, const uint64 &start) - const GCC_OMIT_FRAME_POINTER; - -template<> uint64 GenericCrc::CrcMultiword( - const void *data, size_t bytes, const uint64 &start) const { - if (bytes <= 7) { - const uint8 *src = static_cast(data); - uint64 crc = start ^ this->Base().Canonize(); - for (const uint8 *end = src + bytes; src < end; ++src) { - CRC_BYTE(this, crc, *src); - } - return (crc ^ this->Base().Canonize()); - } - return CrcMultiwordI386Mmx(data, bytes, start); -} - -#define CRC_WORD_MMX() \ - "pxor %[crc0], %[buf0]\n" \ - "movd %[buf0], %[tmp0]\n" \ - "psrlq $32, %[buf0]\n" \ - "movzbl %b[tmp0], %[temp]\n" \ - "shrl $8, %[tmp0]\n" \ - "movq (%[table], %[temp], 8), %[crc0]\n" \ - "movzbl %b[tmp0], %[temp]\n" \ - "shrl $8, %[tmp0]\n" \ - "pxor 1*256*8(%[table], %[temp], 8), %[crc0]\n" \ - "movzbl %b[tmp0], %[temp]\n" \ - "shrl $8, %[tmp0]\n" \ - "pxor 2*256*8(%[table], %[temp], 8), %[crc0]\n" \ - "pxor 3*256*8(%[table], %[tmp0], 8), %[crc0]\n" \ - "movd %[buf0], %[tmp0]\n" \ - "movzbl %b[tmp0], %[temp]\n" \ - "shrl $8, %[tmp0]\n" \ - "pxor 4*256*8(%[table], %[temp], 8), %[crc0]\n" \ - "movzbl %b[tmp0], %[temp]\n" \ - "shrl $8, %[tmp0]\n" \ - "pxor 5*256*8(%[table], %[temp], 8), %[crc0]\n" \ - "movzbl %b[tmp0], %[temp]\n" \ - "shrl $8, %[tmp0]\n" \ - "pxor 6*256*8(%[table], %[temp], 8), %[crc0]\n" \ - "pxor 7*256*8(%[table], %[tmp0], 8), %[crc0]\n" - -template<> uint64 GenericCrc::CrcMultiwordI386Mmx( - const void *data, size_t bytes, const uint64 &start) const { - const uint8 *src = static_cast(data); - const uint8 *end = src + bytes; - uint64 crc0 = start ^ this->Base().Canonize(); - - ALIGN_ON_WORD_BOUNDARY_IF_NEEDED(bytes, this, src, end, crc0, uint64); - if (src >= end) { - return (crc0 ^ this->Base().Canonize()); - } - - uint64 crc1; - uint64 crc2; - uint64 crc3; - - uint64 buf0; - uint64 buf1; - uint64 buf2; - uint64 buf3; - - uint32 tmp0; - uint32 tmp1; - uint32 tmp2; -#ifndef PIC_WORKAROUND - uint32 tmp3; -#endif - - uint32 temp; - - void *table_ptr; - const uint64 *table_interleaved = &this->crc_word_interleaved_[0][0]; - const uint64 *table_word = &this->crc_word_[0][0]; - - asm( -#ifdef PIC_WORKAROUND - "push %%ebx\n" - #define TMP3 "%%ebx" - #define TMP3b "%%bl" -#else - #define TMP3 "%[tmp3]" - #define TMP3b "%b[tmp3]" -#endif - "subl $2*4*8 - 1, %[end]\n" - "cmpl %[src], %[end]\n" - "jbe 2f\n" - - "pxor %[crc1], %[crc1]\n" - "pxor %[crc2], %[crc2]\n" - "pxor %[crc3], %[crc3]\n" - "movq (%[src]), %[buf0]\n" - "movq 1*8(%[src]), %[buf1]\n" - "movq 2*8(%[src]), %[buf2]\n" - "movq 3*8(%[src]), %[buf3]\n" - - "movl %[table_interleaved], %[table]\n" - "1:\n" -#if HAVE_SSE && CRCUTIL_PREFETCH_WIDTH > 0 - "prefetcht0 " TO_STRING(CRCUTIL_PREFETCH_WIDTH) "(%[src])\n" -#endif - "addl $0x20, %[src]\n" - "pxor %[crc0], %[buf0]\n" - "pxor %[crc1], %[buf1]\n" - "pxor %[crc2], %[buf2]\n" - "pxor %[crc3], %[buf3]\n" - - "movd %[buf0], %[tmp0]\n" - "psrlq $32, %[buf0]\n" - "movd %[buf1], %[tmp1]\n" - "psrlq $32, %[buf1]\n" - "movd %[buf2], %[tmp2]\n" - "psrlq $32, %[buf2]\n" - "movd %[buf3], " TMP3 "\n" - "psrlq $32, %[buf3]\n" - - "movzbl %b[tmp0], %[temp]\n" - "shrl $8, %[tmp0]\n" - "movq (%[table], %[temp], 8), %[crc0]\n" - "movzbl %b[tmp1], %[temp]\n" - "shrl $8, %[tmp1]\n" - "movq (%[table], %[temp], 8), %[crc1]\n" - "movzbl %b[tmp2], %[temp]\n" - "shrl $8, %[tmp2]\n" - "movq (%[table], %[temp], 8), %[crc2]\n" - "movzbl " TMP3b ", %[temp]\n" - "shrl $8, " TMP3 "\n" - "movq (%[table], %[temp], 8), %[crc3]\n" - -#define XOR(byte) \ - "movzbl %b[tmp0], %[temp]\n" \ - "shrl $8, %[tmp0]\n" \ - "pxor " #byte "*256*8(%[table], %[temp], 8), %[crc0]\n" \ - "movzbl %b[tmp1], %[temp]\n" \ - "shrl $8, %[tmp1]\n" \ - "pxor " #byte "*256*8(%[table], %[temp], 8), %[crc1]\n" \ - "movzbl %b[tmp2], %[temp]\n" \ - "shrl $8, %[tmp2]\n" \ - "pxor " #byte "*256*8(%[table], %[temp], 8), %[crc2]\n" \ - "movzbl " TMP3b ", %[temp]\n" \ - "shrl $8, " TMP3 "\n" \ - "pxor " #byte "*256*8(%[table], %[temp], 8), %[crc3]\n" - - XOR(1) - XOR(2) - - "pxor 3*256*8(%[table], %[tmp0], 8), %[crc0]\n" - "movd %[buf0], %[tmp0]\n" - "pxor 3*256*8(%[table], %[tmp1], 8), %[crc1]\n" - "movd %[buf1], %[tmp1]\n" - "pxor 3*256*8(%[table], %[tmp2], 8), %[crc2]\n" - "movd %[buf2], %[tmp2]\n" - "pxor 3*256*8(%[table], " TMP3 ", 8), %[crc3]\n" - "movd %[buf3], " TMP3 "\n" - - XOR(4) - XOR(5) - XOR(6) - - "pxor 7*256*8(%[table], %[tmp0], 8), %[crc0]\n" - "movq (%[src]), %[buf0]\n" - "pxor 7*256*8(%[table], %[tmp1], 8), %[crc1]\n" - "movq 1*8(%[src]), %[buf1]\n" - "pxor 7*256*8(%[table], %[tmp2], 8), %[crc2]\n" - "movq 2*8(%[src]), %[buf2]\n" - "pxor 7*256*8(%[table], " TMP3 ", 8), %[crc3]\n" - "movq 3*8(%[src]), %[buf3]\n" - "cmpl %[src], %[end]\n" - "ja 1b\n" -#undef XOR - - "movl %[table_word], %[table]\n" - CRC_WORD_MMX() - - "pxor %[crc1], %[buf1]\n" - "movq %[buf1], %[buf0]\n" - CRC_WORD_MMX() - - "pxor %[crc2], %[buf2]\n" - "movq %[buf2], %[buf0]\n" - CRC_WORD_MMX() - - "pxor %[crc3], %[buf3]\n" - "movq %[buf3], %[buf0]\n" - CRC_WORD_MMX() - - "addl $4*8, %[src]\n" - "2:\n" - "movl %[table_word], %[table]\n" - - "addl $2*4*8 - 8, %[end]\n" - "cmpl %[src], %[end]\n" - "jbe 4f\n" - "3:\n" - "movq (%[src]), %[buf0]\n" - "addl $0x8, %[src]\n" - CRC_WORD_MMX() - "cmpl %[src], %[end]\n" - "ja 3b\n" - "4:\n" - "addl $7, %[end]\n" - - "cmpl %[src], %[end]\n" - "jbe 6f\n" - - "5:\n" - "movd %[crc0], %[tmp0]\n" - "movzbl (%[src]), %[temp]\n" - "movzbl %b[tmp0], %[tmp0]\n" - "psrlq $8, %[crc0]\n" - "xorl %[tmp0], %[temp]\n" - "addl $1, %[src]\n" - "pxor 7*256*8(%[table], %[temp], 8), %[crc0]\n" - "cmpl %[src], %[end]\n" - "ja 5b\n" - - "6:\n" - -#ifdef PIC_WORKAROUND - "pop %%ebx\n" -#endif -#undef TMP3 - - : // outputs - [src] "+r" (src), - [end] "+m" (end), - [crc0] "+y" (crc0), - [crc1] "=&y" (crc1), - [crc2] "=&y" (crc2), - [crc3] "=&y" (crc3), - [buf0] "=&y" (buf0), - [buf1] "=&y" (buf1), - [buf2] "=&y" (buf2), - [buf3] "=&y" (buf3), - [tmp0] "=&q" (tmp0), - [tmp1] "=&q" (tmp1), - [tmp2] "=&q" (tmp2), -#ifndef PIC_WORKAROUND - [tmp3] "=&q" (tmp3), -#endif - [temp] "=&r" (temp), - [table] "=&r" (table_ptr) - - : // inputs - [table_interleaved] "m" (table_interleaved), - [table_word] "m" (table_word)); - - asm volatile("emms"); - - return (crc0 ^ this->Base().Canonize()); -} - -} // namespace crcutil - -#endif // defined(__GNUC__) && HAVE_AMD64 && CRCUTIL_USE_ASM diff --git a/rapidyenc/rapidyenc/crcutil-1.0/code/multiword_64_64_intrinsic_i386_mmx.cc b/rapidyenc/rapidyenc/crcutil-1.0/code/multiword_64_64_intrinsic_i386_mmx.cc deleted file mode 100644 index 9aa74d4..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/code/multiword_64_64_intrinsic_i386_mmx.cc +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Implements 64-bit multiword CRC using MMX built-in functions. - -#include "generic_crc.h" - -#if CRCUTIL_USE_ASM && HAVE_I386 && HAVE_MMX && !defined(_MSC_VER) && !(defined(__GNUC__) && !defined(__clang__)) - -namespace crcutil { - -template<> uint64 GenericCrc::CrcMultiwordI386Mmx( - const void *data, size_t bytes, const uint64 &start) - const GCC_OMIT_FRAME_POINTER; - -#if !defined(_MSC_VER) -template<> uint64 GenericCrc::CrcMultiword( - const void *data, - size_t bytes, - const uint64 &start) const { - if (bytes <= 7) { - const uint8 *src = static_cast(data); - uint64 crc = start ^ Base().Canonize(); - for (const uint8 *end = src + bytes; src < end; ++src) { - CRC_BYTE(this, crc, *src); - } - return (crc ^ Base().Canonize()); - } - return CrcMultiwordI386Mmx(data, bytes, start); -} -#else -#pragma warning(push) -// CL: uninitialized local variable 'crc1' used -// Wrong: crc1 = XOR(crc1, crc1) sets it to 0. -#pragma warning(disable: 4700) - -#pragma warning(disable: 4619) // there is no warning number '592' - -// ICL: variable "crc1" is used before its value is set -// Wrong: crc1 = XOR(crc1, crc1) sets it to 0. -#pragma warning(disable: 592) -#endif // !defined(_MSC_VER) - -#define MM64(adr) reinterpret_cast(adr) -#define MM64_TABLE(byte) MM64(crc_word_interleaved_[byte]) - -#define CRC_WORD_MMX(this, crc, buf) do { \ - buf = _mm_xor_si64(buf, crc); \ - uint32 tmp = static_cast(_mm_cvtsi64_si32(buf)); \ - buf = _mm_srli_si64(buf, 32); \ - crc = MM64(crc_word_[0])[TO_BYTE(tmp)]; \ - tmp >>= 8; \ - crc = _mm_xor_si64(crc, MM64(crc_word_[1])[TO_BYTE(tmp)]); \ - tmp >>= 8; \ - crc = _mm_xor_si64(crc, MM64(crc_word_[2])[TO_BYTE(tmp)]); \ - tmp >>= 8; \ - crc = _mm_xor_si64(crc, MM64(crc_word_[3])[tmp]); \ - tmp = static_cast(_mm_cvtsi64_si32(buf)); \ - crc = _mm_xor_si64(crc, MM64(crc_word_[4])[TO_BYTE(tmp)]); \ - tmp >>= 8; \ - crc = _mm_xor_si64(crc, MM64(crc_word_[5])[TO_BYTE(tmp)]); \ - tmp >>= 8; \ - crc = _mm_xor_si64(crc, MM64(crc_word_[6])[TO_BYTE(tmp)]); \ - tmp >>= 8; \ - crc = _mm_xor_si64(crc, MM64(crc_word_[7])[tmp]); \ -} while (0) - -template<> uint64 GenericCrc::CrcMultiwordI386Mmx( - const void *data, size_t bytes, const uint64 &start) const { - const uint8 *src = static_cast(data); - const uint8 *end = src + bytes; - uint64 crc = start ^ Base().Canonize(); - - ALIGN_ON_WORD_BOUNDARY_IF_NEEDED(bytes, this, src, end, crc, uint64); - if (src >= end) { - return (crc ^ Base().Canonize()); - } - - // Process 4 registers of sizeof(uint64) bytes at once. - bytes = static_cast(end - src) & ~(4*8 - 1); - if (bytes > 4*8) { - const uint8 *stop = src + bytes - 4*8; - union { - __m64 m64; - uint64 u64; - } temp; - __m64 crc0; - __m64 crc1; - __m64 crc2; - __m64 crc3; - __m64 buf0 = MM64(src)[0]; - __m64 buf1 = MM64(src)[1]; - __m64 buf2 = MM64(src)[2]; - __m64 buf3 = MM64(src)[3]; - - temp.u64 = crc; - crc0 = temp.m64; -#if defined(__GNUC__) && !GCC_VERSION_AVAILABLE(4, 4) - // There is no way to suppress a warning in GCC; - // generate extra assignments. - temp.u64 = 0; - crc1 = temp.m64; - crc2 = temp.m64; - crc3 = temp.m64; -#else - crc1 = _mm_xor_si64(crc1, crc1); - crc2 = _mm_xor_si64(crc2, crc2); - crc3 = _mm_xor_si64(crc3, crc3); -#endif // defined(__GNUC__) && !GCC_VERSION_AVAILABLE(4, 4) - - do { - PREFETCH(src); - src += 4*8; - - buf0 = _mm_xor_si64(buf0, crc0); - buf1 = _mm_xor_si64(buf1, crc1); - buf2 = _mm_xor_si64(buf2, crc2); - buf3 = _mm_xor_si64(buf3, crc3); - - uint32 tmp0 = static_cast(_mm_cvtsi64_si32(buf0)); - uint32 tmp1 = static_cast(_mm_cvtsi64_si32(buf1)); - uint32 tmp2 = static_cast(_mm_cvtsi64_si32(buf2)); - uint32 tmp3 = static_cast(_mm_cvtsi64_si32(buf3)); - - buf0 = _mm_srli_si64(buf0, 32); - buf1 = _mm_srli_si64(buf1, 32); - buf2 = _mm_srli_si64(buf2, 32); - buf3 = _mm_srli_si64(buf3, 32); - - crc0 = MM64_TABLE(0)[TO_BYTE(tmp0)]; - tmp0 >>= 8; - crc1 = MM64_TABLE(0)[TO_BYTE(tmp1)]; - tmp1 >>= 8; - crc2 = MM64_TABLE(0)[TO_BYTE(tmp2)]; - tmp2 >>= 8; - crc3 = MM64_TABLE(0)[TO_BYTE(tmp3)]; - tmp3 >>= 8; - -#define XOR(byte) do { \ - crc0 = _mm_xor_si64(crc0, MM64_TABLE(byte)[TO_BYTE(tmp0)]); \ - tmp0 >>= 8; \ - crc1 = _mm_xor_si64(crc1, MM64_TABLE(byte)[TO_BYTE(tmp1)]); \ - tmp1 >>= 8; \ - crc2 = _mm_xor_si64(crc2, MM64_TABLE(byte)[TO_BYTE(tmp2)]); \ - tmp2 >>= 8; \ - crc3 = _mm_xor_si64(crc3, MM64_TABLE(byte)[TO_BYTE(tmp3)]); \ - tmp3 >>= 8; \ -} while (0) - - XOR(1); - XOR(2); - - crc0 = _mm_xor_si64(crc0, MM64_TABLE(3)[tmp0]); - tmp0 = static_cast(_mm_cvtsi64_si32(buf0)); - crc1 = _mm_xor_si64(crc1, MM64_TABLE(3)[tmp1]); - tmp1 = static_cast(_mm_cvtsi64_si32(buf1)); - crc2 = _mm_xor_si64(crc2, MM64_TABLE(3)[tmp2]); - tmp2 = static_cast(_mm_cvtsi64_si32(buf2)); - crc3 = _mm_xor_si64(crc3, MM64_TABLE(3)[tmp3]); - tmp3 = static_cast(_mm_cvtsi64_si32(buf3)); - - XOR(4); - XOR(5); - XOR(6); - -#undef XOR - - crc0 = _mm_xor_si64(crc0, MM64_TABLE(sizeof(uint64) - 1)[tmp0]); - buf0 = MM64(src)[0]; - crc1 = _mm_xor_si64(crc1, MM64_TABLE(sizeof(uint64) - 1)[tmp1]); - buf1 = MM64(src)[1]; - crc2 = _mm_xor_si64(crc2, MM64_TABLE(sizeof(uint64) - 1)[tmp2]); - buf2 = MM64(src)[2]; - crc3 = _mm_xor_si64(crc3, MM64_TABLE(sizeof(uint64) - 1)[tmp3]); - buf3 = MM64(src)[3]; - } - while (src < stop); - - CRC_WORD_MMX(this, crc0, buf0); - buf1 = _mm_xor_si64(buf1, crc1); - CRC_WORD_MMX(this, crc0, buf1); - buf2 = _mm_xor_si64(buf2, crc2); - CRC_WORD_MMX(this, crc0, buf2); - buf3 = _mm_xor_si64(buf3, crc3); - CRC_WORD_MMX(this, crc0, buf3); - - temp.m64 = crc0; - crc = temp.u64; - - _mm_empty(); - - src += 4*8; - } - - // Process sizeof(uint64) bytes at once. - bytes = static_cast(end - src) & ~(sizeof(uint64) - 1); - if (bytes > 0) { - union { - __m64 m64; - uint64 u64; - } temp; - __m64 crc0; - - temp.u64 = crc; - crc0 = temp.m64; - - for (const uint8 *stop = src + bytes; src < stop; src += sizeof(uint64)) { - __m64 buf0 = MM64(src)[0]; - CRC_WORD_MMX(this, crc0, buf0); - } - - temp.m64 = crc0; - crc = temp.u64; - - _mm_empty(); - } - - // Compute CRC of remaining bytes. - for (;src < end; ++src) { - CRC_BYTE(this, crc, *src); - } - - return (crc ^ Base().Canonize()); -} - -#if defined(_MSC_VER) -#pragma warning(pop) -#endif // defined(_MSC_VER) - -} // namespace crcutil - -#endif // CRCUTIL_USE_ASM && HAVE_I386 && HAVE_MMX diff --git a/rapidyenc/rapidyenc/crcutil-1.0/code/platform.h b/rapidyenc/rapidyenc/crcutil-1.0/code/platform.h deleted file mode 100644 index 936cf7d..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/code/platform.h +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Detects configuration and defines compiler-specific macros. -// Also, sets user-defined CRUTIL_USE_* macros to default values. - -#ifndef CRCUTIL_PLATFORM_H_ -#define CRCUTIL_PLATFORM_H_ - -// Permanently disable some annoying warnings generated -// by Microsoft CL when compiling Microsoft's headers. -#include "std_headers.h" - -// Use inline asm version of the code? -#if !defined(CRCUTIL_USE_ASM) -#define CRCUTIL_USE_ASM 1 -#endif // !defined(CRCUTIL_USE_ASM) - - -#if !defined(HAVE_I386) -#if defined(__i386__) || defined(_M_IX86) -#define HAVE_I386 1 -#else -#define HAVE_I386 0 -#endif // defined(__i386__) || defined(_M_IX86) -#endif // defined(HAVE_I386) - - -#if !defined(HAVE_AMD64) -#if defined(__amd64__) || defined(_M_AMD64) -#define HAVE_AMD64 1 -#else -#define HAVE_AMD64 0 -#endif // defined(__amd64__) || defined(_M_AMD64) -#endif // defined(HAVE_AMD64) - - -#if HAVE_AMD64 || HAVE_I386 -#if defined(_MSC_VER) -#pragma warning(push) -// '_M_IX86' is not defined as a preprocessor macro -#pragma warning(disable: 4668) -#include -#pragma warning(pop) -#endif // defined(_MSC_VER) - - -#if !defined(HAVE_MMX) -#if defined(_MSC_VER) || (defined(__GNUC__) && defined(__MMX__)) -#define HAVE_MMX 1 -#else -#define HAVE_MMX 0 -#endif // defined(_MSC_VER) || (defined(__GNUC__) && defined(__MMX__)) -#endif // !defined(HAVE_MMX) - - -#if !defined(HAVE_SSE) -#if defined(_MSC_VER) || (defined(__GNUC__) && defined(__SSE__)) -#include -#define HAVE_SSE 1 -#else -#define HAVE_SSE 0 -#endif // defined(_MSC_VER) || (defined(__GNUC__) && defined(__SSE__)) -#endif // !defined(HAVE_SSE) - - -#if !defined(HAVE_SSE2) -#if defined(_MSC_VER) || (defined(__GNUC__) && defined(__SSE2__)) -#include -#define HAVE_SSE2 1 -#else -#define HAVE_SSE2 0 -#endif // defined(_MSC_VER) || (defined(__GNUC__) && defined(__SSE2__)) -#endif // !defined(HAVE_SSE2) - -#else - -#if !defined(HAVE_MMX) -#define HAVE_MMX 0 -#endif // !defined(HAVE_MMX) - -#if !defined(HAVE_SSE) -#define HAVE_SSE 0 -#endif // !defined(HAVE_SSE) - -#if !defined(HAVE_SSE2) -#define HAVE_SSE2 0 -#endif // !defined(HAVE_SSE2) - -#endif // HAVE_AMD64 || HAVE_I386 - -// Error checking -#if HAVE_SSE && !HAVE_MMX -#error SSE is available but not MMX? -#endif // HAVE_SSE && !HAVE_MMX - -#if HAVE_SSE2 && (!HAVE_SSE || !HAVE_MMX) -#error SSE2 is available but not SSE or MMX? -#endif // HAVE_SSE2 && (!HAVE_SSE || !HAVE_MMX) - - -#if !defined(CRCUTIL_PREFETCH_WIDTH) -// On newer X5550 CPU, heavily optimized CrcMultiword is 3% faster without -// prefetch for inputs smaller than 8MB and less than 1% slower for 8MB and -// larger blocks. On older Q9650 CPU, the code is 2-3% faster for inputs -// smaller than 8MB, 4-5% slower when length >= 8MB. -// Tested with prefetch length 256, 512, and 4096. -// -// At this moment there is no compelling reason to use prefetching. -// -#define CRCUTIL_PREFETCH_WIDTH 0 -#endif // !defined(CRCUTIL_PREFETCH_WIDTH) - - -#if HAVE_SSE && CRCUTIL_PREFETCH_WIDTH > 0 -#define PREFETCH(src) \ - _mm_prefetch(reinterpret_cast(src) + CRCUTIL_PREFETCH_WIDTH, \ - _MM_HINT_T0) -#else -#define PREFETCH(src) -#endif // HAVE_SSE && CRCUTIL_PREFETCH_WIDTH > 0 - - -// If block size exceeds CRCUTIL_MIN_ALIGN_SIZE, align the data -// before accessing it at word boundary. See generic_crc.cc, -// ALIGN_ON_WORD_BOUNDARY_IF_NEEDED() macro. -#if !defined(CRCUTIL_MIN_ALIGN_SIZE) -#if HAVE_AMD64 || HAVE_I386 -#define CRCUTIL_MIN_ALIGN_SIZE (1024) -#else -#define CRCUTIL_MIN_ALIGN_SIZE 0 -#endif // HAVE_AMD64 || HAVE_I386 -#endif // !defined(CRCUTIL_MIN_ALIGN_SIZE) - - -// Use _mm_crc32_u64/32/8 intrinics? -// If not, they will be implemented in software. -#if !HAVE_I386 && !HAVE_AMD64 - -#undef CRCUTIL_USE_MM_CRC32 -#define CRCUTIL_USE_MM_CRC32 0 - -#else - -#if !defined(CRCUTIL_USE_MM_CRC32) -#if defined(_MSC_VER) || defined(__GNUC__) -#define CRCUTIL_USE_MM_CRC32 1 -#else -#define CRCUTIL_USE_MM_CRC32 0 -#endif // defined(_MSC_VER) || defined(__GNUC__) -#endif // !defined(CRCUTIL_USE_MM_CRC32) - -#endif // !HAVE_I386 && !HAVE_AMD64 - - -// Stringize -- always handy. -#define TO_STRING_VALUE(arg) #arg -#define TO_STRING(arg) TO_STRING_VALUE(arg) - - -// Compilers give "right shift count >= width of type" warning even -// though the shift happens only under appropriate "if". -#define SHIFT_RIGHT_NO_WARNING(value, bits) \ - ((value) >> (((bits) < (8 * sizeof(value))) ? (bits) : 0)) -#define SHIFT_RIGHT_SAFE(value, bits) \ - ((bits) < (8 * sizeof(value)) ? SHIFT_RIGHT_NO_WARNING(value, bits) : 0) - -// The same for left shifts. -#define SHIFT_LEFT_NO_WARNING(value, bits) \ - ((value) << (((bits) < (8 * sizeof(value))) ? (bits) : 0)) -#define SHIFT_LEFT_SAFE(value, bits) \ - ((bits) < (8 * sizeof(value)) ? SHIFT_LEFT_NO_WARNING(value, bits) : 0) - -// GCC-specific macros. -// -#define GCC_VERSION_AVAILABLE(major, minor) \ - (defined(__GNUC__) && \ - (__GNUC__ > (major) || \ - (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))) - - -#if defined(__GNUC__) - -// The GenericCrc tables must be properly aligned. -// Penalty for misalignment? 50% performance degradation. -// For 128-bit SSE2, the penalty is access violation. -#define GCC_ALIGN_ATTRIBUTE(n) __attribute__((aligned(n))) - -#if GCC_VERSION_AVAILABLE(4, 4) -// If not marked as "omit frame pointer", -// GCC won't be able to find enough registers. -#define GCC_OMIT_FRAME_POINTER \ - __attribute__((__optimize__(2, "omit-frame-pointer"))) -#endif // GCC_VERSION_AVAILABLE(4, 4) - -#if !defined(__forceinline) -#define __forceinline __attribute__((__always_inline__)) inline -#endif // !defined(__forceinline) - -#if defined(__APPLE_CC__) -// The version of GCC used by Max OS X xCode v 5664 does not understand -// "movq xmm, r64" instruction and requires the use of "movd" (probably -// because of the bug in GCC which treats "movq/movd xmm,r64 or r64,xmm" -// the same). -// -// Leaving common sense aside, let's peek into Intel's instruction -// reference manual. That's what description of MOVD command says: -// MOVD xmm, r/m32 (opcode 66 0F 6E /r) -// MOVD r/m32, xmm (opcode 66 0F 7E /r) -// MOVQ xmm, r/m64 (opcode 66 REX.W 0F 6E /r) -// MOVQ r/m64, xmm (opcode 66 REX.W 0F 7E /r) -#define SSE2_MOVQ "movd" -#else -#define SSE2_MOVQ "movq" -#endif // defined(__APPLE_CC__) - -#endif // defined(__GNUC__) - - -// Define compiler-specific macros that were not set yet. -#if !defined(_MSC_VER) && !defined(__forceinline) -#define __forceinline inline -#endif // !defined(_MSC_VER) && !defined(__forceinline) - -#if !defined(GCC_OMIT_FRAME_POINTER) -#define GCC_OMIT_FRAME_POINTER -#endif // !defined(GCC_OMIT_FRAME_POINTER) - -#if !defined(GCC_ALIGN_ATTRIBUTE) -#define GCC_ALIGN_ATTRIBUTE(n) -#endif // !defined(GCC_ALIGN_ATTRIBUTE) - - -#endif // CRCUTIL_PLATFORM_H_ diff --git a/rapidyenc/rapidyenc/crcutil-1.0/code/protected_crc.h b/rapidyenc/rapidyenc/crcutil-1.0/code/protected_crc.h deleted file mode 100644 index 762fced..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/code/protected_crc.h +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Protects CRC tables with its own CRC. -// CRC tables get corrupted too, and if corruption is -// not caught, data poisoning becomes a reality. - -#ifndef CRCUTIL_PROTECTED_CRC_H_ -#define CRCUTIL_PROTECTED_CRC_H_ - -namespace crcutil { - -#pragma pack(push, 16) - -// Class CrcImplementation should not have virtual functions: -// vptr is stored as the very first field, vptr value is defined -// at runtime, so it is impossible to CRC(*this) once and -// guarantee that this value will not change from run to run. -// -template class ProtectedCrc - : public CrcImplementation { - public: - typedef typename CrcImplementation::Crc Crc; - - // Returns check value that the caller should compare - // against pre-computed, trusted constant. - // - // Computing SelfCheckValue() after CRC initialization, - // storing it in memory, and periodically checking against - // stored value may not work: if CRC tables were initialized - // incorrectly and/or had been corrupted during initialization, - // CheckValue() will return garbage. Garbage in, garbage out. - // Consequitive checks will not detect a problem, the application - // will happily produce and save the data with corrupt CRC. - // - // The application should call SelfCheckValue() regularly: - // 1. First and foremost, on every CRC mismatch. - // 2. After CRC'ing the data but before sending it out or writing it. - // 3. Worst case, every Nth CRC'ed byte or every Nth call to CRC. - // - Crc SelfCheckValue() const { - return CrcDefault(this, sizeof(*this), 0); - } -} GCC_ALIGN_ATTRIBUTE(16); - -#pragma pack(pop) - -} // namespace crcutil - -#endif // CRCUTIL_PROTECTED_CRC_H_ diff --git a/rapidyenc/rapidyenc/crcutil-1.0/code/rolling_crc.h b/rapidyenc/rapidyenc/crcutil-1.0/code/rolling_crc.h deleted file mode 100644 index ad4a947..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/code/rolling_crc.h +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Implements rolling CRC (e.g. for Rabin fingerprinting). - -#ifndef CRCUTIL_ROLLING_CRC_H_ -#define CRCUTIL_ROLLING_CRC_H_ - -#include "base_types.h" // size_t, uint8 -#include "crc_casts.h" // TO_BYTE - -namespace crcutil { - -#pragma pack(push, 16) - -// CrcImplementation should provide: -// - typename Crc -// - typename TableEntry -// - typename Word -// - Crc CrcDefault(const void *data, size_t bytes, const Crc &start) -// - const GfUtil &Base() const -template class RollingCrc { - public: - typedef typename CrcImplementation::Crc Crc; - typedef typename CrcImplementation::TableEntry TableEntry; - typedef typename CrcImplementation::Word Word; - - RollingCrc() {} - - // Initializes internal data structures. - // Retains reference to "crc" instance -- it is used by Start(). - RollingCrc(const CrcImplementation &crc, - size_t roll_window_bytes, - const Crc &start_value) { - Init(crc, roll_window_bytes, start_value); - } - - // Computes crc of "roll_window_bytes" using - // "start_value" of "crc" (see Init()). - Crc Start(const void *data) const { - return crc_->CrcDefault(data, roll_window_bytes_, start_value_); - } - - // Computes CRC of "roll_window_bytes" starting in next position. - Crc Roll(const Crc &old_crc, size_t byte_out, size_t byte_in) const { - return (old_crc >> 8) ^ in_[TO_BYTE(old_crc) ^ byte_in] ^ out_[byte_out]; - } - - // Initializes internal data structures. - // Retains reference to "crc" instance -- it is used by Start(). - void Init(const CrcImplementation &crc, - size_t roll_window_bytes, - const Crc &start_value) { - crc_ = &crc; - roll_window_bytes_ = roll_window_bytes; - start_value_ = start_value; - - Crc add = crc.Base().Canonize() ^ start_value; - add = crc.Base().Multiply(add, crc.Base().Xpow8N(roll_window_bytes)); - add ^= crc.Base().Canonize(); - Crc mul = crc.Base().One() ^ crc.Base().Xpow8N(1); - add = crc.Base().Multiply(add, mul); - - mul = crc.Base().XpowN(8 * roll_window_bytes + crc.Base().Degree()); - for (size_t i = 0; i < 256; ++i) { - out_[i] = static_cast( - crc.Base().MultiplyUnnormalized( - static_cast(i), 8, mul) ^ add); - } - for (size_t i = 0; i < 256; ++i) { - in_[i] = crc.crc_word_[sizeof(Word) - 1][i]; - } - } - - // Returns start value. - Crc StartValue() const { return start_value_; } - - // Returns length of roll window. - size_t WindowBytes() const { return roll_window_bytes_; } - - protected: - TableEntry in_[256]; - TableEntry out_[256]; - - // Used only by Start(). - Crc start_value_; - const CrcImplementation *crc_; - size_t roll_window_bytes_; -} GCC_ALIGN_ATTRIBUTE(16); - -#pragma pack(pop) - -} // namespace crcutil - -#endif // CRCUTIL_ROLLING_CRC_H_ diff --git a/rapidyenc/rapidyenc/crcutil-1.0/code/std_headers.h b/rapidyenc/rapidyenc/crcutil-1.0/code/std_headers.h deleted file mode 100644 index 5c7b6a3..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/code/std_headers.h +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Includes some standard C headers for size_t, memset, etc. -// -// Also, permanently disables a number of warnings produced -// by Microsoft's compiler when it includes standard headers -// (surprisingly, also by Microsoft). - -#ifndef CRCUTIL_STD_HEADERS_H_ -#define CRCUTIL_STD_HEADERS_H_ - -#if defined(_MSC_VER) -// '4' bytes padding added after data member ... -#pragma warning(disable:4820) - -// unreferenced inline function has been removed ... -#pragma warning(disable:4514) - -// conditional expression is constant -#pragma warning(disable: 4127) - -// function ... not inlined -#pragma warning(disable: 4710) - -// function ... selected for automatic inline expansion -#pragma warning(disable: 4711) - -#define _CRT_SECURE_NO_WARNINGS - -#endif // defined(_MSC_VER) - -// #define _CSTDLIB_ -#include // always handy -#include // memset -#include // size_t, _rotl/_rotl64(MSC) -#include // ptrdiff_t (GNUC) -#include // va_list - -#endif // CRCUTIL_STD_HEADERS_H_ diff --git a/rapidyenc/rapidyenc/crcutil-1.0/code/uint128_sse2.h b/rapidyenc/rapidyenc/crcutil-1.0/code/uint128_sse2.h deleted file mode 100644 index 6365135..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/code/uint128_sse2.h +++ /dev/null @@ -1,312 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Implements a limited set of 128-bit arithmetic operations -// (the ones that are used by CRC) using SSE2 intrinsics. - -#ifndef CRCUTIL_UINT128_SSE2_H_ -#define CRCUTIL_UINT128_SSE2_H_ - -#include "base_types.h" -#include "crc_casts.h" // Downcast, CrcFromUint64, Uint64FromCrc -#include "platform.h" - -#if HAVE_SSE2 - -namespace crcutil { - -// Specialized functions handling __m128i. -template<> __forceinline uint64 Downcast(const __m128i &value) { -#if HAVE_AMD64 && defined(__GNUC__) - // GCC 4.4.x is too smart and, instead of MOVQ, generates SSE4 PEXTRQ - // instruction when the code is compiled with -mmsse4. - // Fixed in 4.5 which generates conversion through memory (why?). - // And -- yes, it makes quite measurable difference. - uint64 temp; - asm(SSE2_MOVQ " %[i128], %[u64]\n" : [u64] "=r" (temp) : [i128] "x" (value)); - return temp; -#elif HAVE_AMD64 && (!defined(_MSC_FULL_VER) || _MSC_FULL_VER > 150030729) - return static_cast(_mm_cvtsi128_si64(value)); -#else - // 64-bit CL 15.00.30729.1 -O2 generates incorrect code (tests fail). - // _mm_cvtsi128_si64() is not available on i386. - uint64 temp; - _mm_storel_epi64(reinterpret_cast<__m128i *>(&temp), value); - return temp; -#endif -} - - -class uint128_sse2 { - public: - uint128_sse2() {} - ~uint128_sse2() {} - - // Default casts to uint128_sse2 and assignment operator. - __forceinline void operator =(uint64 value) { -#if HAVE_AMD64 && defined(__GNUC__) && !GCC_VERSION_AVAILABLE(4, 5) - // Prevent generation of SSE4 pinsrq insruction when - // compiling with GCC 4.4.x with -msse4 flag. - asm(SSE2_MOVQ " %[u64], %[i128]\n" : [i128] "=x" (x_) : [u64] "r" (value)); -#elif HAVE_AMD64 - x_ = _mm_cvtsi64_si128(static_cast(value)); -#else - x_ = _mm_loadl_epi64(reinterpret_cast(&value)); -#endif - } - __forceinline uint128_sse2(uint64 x) { - *this = x; - } - __forceinline uint128_sse2(const __m128i x) : x_(x) { - } - __forceinline uint128_sse2(const uint128_sse2& x) : x_(x.x_) { - } - __forceinline operator __m128i() const { - return x_; - } - __forceinline void operator =(const uint128_sse2 &x) { - x_ = x.x_; - } - - // Extracts 64 less significant bits. - __forceinline uint64 to_uint64() const { - return Downcast<__m128i, uint64>(x_); - } - - // Comparisons. - __forceinline bool operator ==(const uint128_sse2 &y) const { - union { - __m128i i128; - uint64 u64[2]; - } t; - t.i128 = _mm_xor_si128(x_, y.x_); - return (t.u64[0] | t.u64[1]) == 0; - } - __forceinline bool operator ==(uint64 value) const { - union { - __m128i i128; - uint64 u64[2]; - } t; - t.i128 = x_; - return (t.u64[0] == value && t.u64[1] == 0); - } - __forceinline bool operator !=(const uint128_sse2 &y) const { - union { - __m128i i128; - uint64 u64[2]; - } t; - t.i128 = _mm_xor_si128(x_, y.x_); - return (t.u64[0] | t.u64[1]) != 0; - } - __forceinline bool operator !=(uint64 value) const { - union { - __m128i i128; - uint64 u64[2]; - } t; - t.i128 = x_; - return (t.u64[0] != value || t.u64[1] != 0); - } - - __forceinline bool operator <(const uint128_sse2 &y) const { - union { - __m128i i128; - uint64 u64[2]; - } xx, yy; - xx.i128 = x_; - yy.i128 = y.x_; - return (xx.u64[0] < yy.u64[0] || - (xx.u64[0] == yy.u64[0] && xx.u64[1] < yy.u64[1])); - } - - // Bitwise logic operators. - __forceinline uint128_sse2 operator ^(const uint128_sse2 &y) const { - return _mm_xor_si128(x_, y.x_); - } - __forceinline uint128_sse2 operator &(const uint128_sse2 &y) const { - return _mm_and_si128(x_, y.x_); - } - __forceinline uint128_sse2 operator |(const uint128_sse2 &y) const { - return _mm_or_si128(x_, y.x_); - } - - __forceinline void operator ^=(const uint128_sse2 &y) { - *this = *this ^ y.x_; - } - __forceinline void operator &=(const uint128_sse2 &y) { - *this = *this & y.x_; - } - __forceinline void operator |=(const uint128_sse2 &y) { - *this = *this | y.x_; - } - - // Arithmetic operators. - __forceinline uint128_sse2 operator +(uint64 y) const { - union { - __m128i i128; - uint64 u64[2]; - } temp; - temp.i128 = x_; - // a + b >= 2**64 iff - // a + b > (2**64 - 1) iff - // a > (2**64 - 1) - b iff - // a > ~b - if (temp.u64[0] > ~y) { - temp.u64[1] += 1; - } - temp.u64[0] += y; - return temp.i128; - } - __forceinline void operator +=(uint64 x) { - *this = *this + x; - } - __forceinline uint128_sse2 operator -(uint64 y) const { - union { - __m128i i128; - uint64 u64[2]; - } temp; - temp.i128 = x_; - if (temp.u64[0] < y) { - temp.u64[1] -= 1; - } - temp.u64[0] -= y; - return temp.i128; - } - __forceinline void operator -=(uint64 x) { - *this = *this - x; - } - - // Bitwise logical shifts. - __forceinline uint128_sse2 operator >>(const int bits) const { - if (bits == 8) { - return _mm_srli_si128(x_, 1); - } else if (bits == 16) { - return _mm_srli_si128(x_, 2); - } else if (bits == 32) { - return _mm_srli_si128(x_, 4); - } else if (bits == 64) { - return _mm_srli_si128(x_, 8); - } else { - return long_shift_right(bits); - } - } - __forceinline uint128_sse2 operator >>(const size_t bits) const { - return *this >> static_cast(bits); - } - __forceinline void operator >>=(const int bits) { - *this = *this >> bits; - } - __forceinline void operator >>=(const size_t bits) { - *this = *this >> static_cast(bits); - } - - __forceinline uint128_sse2 operator <<(int bits) const { - if (bits == 8) { - return _mm_slli_si128(x_, 1); - } else if (bits == 16) { - return _mm_slli_si128(x_, 2); - } else if (bits == 32) { - return _mm_slli_si128(x_, 4); - } else if (bits == 64) { - return _mm_slli_si128(x_, 8); - } else { - return long_shift_left(bits); - } - } - __forceinline uint128_sse2 operator <<(size_t bits) const { - return *this << static_cast(bits); - } - __forceinline void operator <<=(int bits) { - *this = *this << bits; - } - __forceinline void operator <<=(size_t bits) { - *this = *this << static_cast(bits); - } - - protected: - __forceinline uint128_sse2 long_shift_right(int bits) const { - union { - __m128i i128; - uint64 u64[2]; - } x; - x.i128 = x_; - for (; bits > 0; --bits) { - x.u64[0] >>= 1; - if (x.u64[1] & 1) { - x.u64[0] |= static_cast(1) << 63; - } - x.u64[1] >>= 1; - } - return x.i128; - } - - __forceinline uint128_sse2 long_shift_left(int bits) const { - union { - __m128i i128; - int64 i64[2]; - } x; - x.i128 = x_; - for (; bits > 0; --bits) { - x.i64[1] <<= 1; - if (x.i64[0] < 0) { - x.i64[1] |= 1; - } - x.i64[0] <<= 1; - } - return x.i128; - } - - __m128i x_; -} GCC_ALIGN_ATTRIBUTE(16); - - -// Specialized versions. -template<> __forceinline uint64 Downcast(const uint128_sse2 &x) { - return x.to_uint64(); -} -template<> __forceinline uint32 Downcast(const uint128_sse2 &x) { - return static_cast(x.to_uint64()); -} -template<> __forceinline uint16 Downcast(const uint128_sse2 &x) { - return static_cast(x.to_uint64()); -} -template<> __forceinline uint8 Downcast(const uint128_sse2 &x) { - return static_cast(x.to_uint64()); -} - -template<> __forceinline uint128_sse2 CrcFromUint64(uint64 lo, uint64 hi) { - union { - __m128i i128; - uint64 u64[2]; - } temp; - temp.u64[0] = lo; - temp.u64[1] = hi; - return temp.i128; -} - -template<> __forceinline void Uint64FromCrc(const uint128_sse2 &crc, - uint64 *lo, uint64 *hi) { - union { - __m128i i128; - uint64 u64[2]; - } temp; - temp.i128 = crc; - *lo = temp.u64[0]; - *hi = temp.u64[1]; -} - -} // namespace crcutil - -#endif // HAVE_SSE2 - -#endif // CRCUTIL_UINT128_SSE2_H_ diff --git a/rapidyenc/rapidyenc/crcutil-1.0/examples/interface.cc b/rapidyenc/rapidyenc/crcutil-1.0/examples/interface.cc deleted file mode 100644 index 6c8a34f..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/examples/interface.cc +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// This is the only file where all details of CRC implementation are buried. - -#include "interface.h" - -#include "aligned_alloc.h" -#include "crc32c_sse4.h" -#include "generic_crc.h" -#include "protected_crc.h" -#include "rolling_crc.h" - -// Align all CRC tables on kAlign boundary. -// Shall be exact power of 2. -static size_t kAlign = 4 * 1024; - -using namespace crcutil; - -#if defined(__GNUC__) -// Suppress 'invalid access to non-static data member ... of NULL object' -#undef offsetof -#define offsetof(TYPE, MEMBER) (reinterpret_cast \ - ((&reinterpret_cast ( \ - reinterpret_cast (1)->MEMBER))) - 1) -#endif // defined(__GNUC__) - -namespace crcutil_interface { - -template - class Implementation : public CRC { - public: - typedef typename CrcImplementation::Crc Crc; - typedef Implementation Self; - - Implementation(const Crc &poly, - size_t degree, - bool canonical, - const Crc &roll_start_value, - size_t roll_length) - : crc_(poly, degree, canonical), - rolling_crc_(crc_, roll_length, roll_start_value) { - } - - static Self *Create(const Crc &poly, - size_t degree, - bool canonical, - const Crc &roll_start_value, - size_t roll_length, - const void **allocated_memory) { - void *memory = AlignedAlloc(sizeof(Self), - offsetof(Self, crc_), - kAlign, - allocated_memory); - return new(memory) Self(poly, - degree, - canonical, - roll_start_value, - roll_length); - } - - virtual void Delete() { - AlignedFree(this); - } - - void *operator new(size_t, void *p) { - return p; - } - - virtual void GeneratingPolynomial(/* OUT */ UINT64 *lo, - /* OUT */ UINT64 *hi = NULL) const { - SetValue(crc_.Base().GeneratingPolynomial(), lo, hi); - } - - virtual size_t Degree() const { - return crc_.Base().Degree(); - } - - virtual void CanonizeValue(/* OUT */ UINT64 *lo, - /* OUT */ UINT64 *hi = NULL) const { - SetValue(crc_.Base().Canonize(), lo, hi); - } - - virtual void RollStartValue(/* OUT */ UINT64 *lo, - /* OUT */ UINT64 *hi = NULL) const { - SetValue(rolling_crc_.StartValue(), lo, hi); - } - - virtual size_t RollWindowBytes() const { - return rolling_crc_.WindowBytes(); - } - - virtual void SelfCheckValue(/* OUT */ UINT64 *lo, - /* OUT */ UINT64 *hi = NULL) const { - Crc crc = crc_.CrcDefault(&crc_, sizeof(crc_), 0); - crc = crc_.CrcDefault(&rolling_crc_, sizeof(rolling_crc_), crc); - SetValue(crc, lo, hi); - } - - virtual void Compute(const void *data, - size_t bytes, - /* INOUT */ UINT64 *lo, - /* INOUT */ UINT64 *hi = NULL) const { - SetValue(crc_.CrcDefault(data, bytes, GetValue(lo, hi)), lo, hi); - } - - virtual void RollStart(const void *data, - /* INOUT */ UINT64 *lo, - /* INOUT */ UINT64 *hi = NULL) const { - SetValue(rolling_crc_.Start(data), lo, hi); - } - - virtual void Roll(size_t byte_out, - size_t byte_in, - /* INOUT */ UINT64 *lo, - /* INOUT */ UINT64 *hi = NULL) const { - SetValue(rolling_crc_.Roll(GetValue(lo, hi), byte_out, byte_in), lo, hi); - } - - virtual void CrcOfZeroes(UINT64 bytes, - /* INOUT */ UINT64 *lo, - /* INOUT */ UINT64 *hi = NULL) const { - SetValue(crc_.Base().CrcOfZeroes(bytes, GetValue(lo, hi)), lo, hi); - } - - virtual void ChangeStartValue( - UINT64 start_old_lo, UINT64 start_old_hi, - UINT64 start_new_lo, UINT64 start_new_hi, - UINT64 bytes, - /* INOUT */ UINT64 *lo, - /* INOUT */ UINT64 *hi = NULL) const { - SetValue(crc_.Base().ChangeStartValue( - GetValue(lo, hi), - bytes, - GetValue(start_old_lo, start_old_hi), - GetValue(start_new_lo, start_new_hi)), - lo, - hi); - } - - virtual void Concatenate(UINT64 crcB_lo, UINT64 crcB_hi, - UINT64 bytes_B, - /* INOUT */ UINT64* crcA_lo, - /* INOUT */ UINT64* crcA_hi = NULL) const { - SetValue(crc_.Base().Concatenate(GetValue(crcA_lo, crcA_hi), - GetValue(crcB_lo, crcB_hi), - bytes_B), - crcA_lo, - crcA_hi); - } - - virtual size_t StoreComplementaryCrc( - void *dst, - UINT64 message_crc_lo, UINT64 message_crc_hi, - UINT64 result_crc_lo, UINT64 result_crc_hi = 0) const { - return crc_.Base().StoreComplementaryCrc( - dst, - GetValue(message_crc_lo, message_crc_hi), - GetValue(result_crc_lo, result_crc_hi)); - } - - virtual size_t StoreCrc(void *dst, - UINT64 lo, - UINT64 hi = 0) const { - return crc_.Base().StoreCrc(dst, GetValue(lo, hi)); - } - - virtual void CrcOfCrc(/* OUT */ UINT64 *lo, - /* OUT */ UINT64 *hi = NULL) const { - SetValue(crc_.Base().CrcOfCrc(), lo, hi); - } - - private: - static Crc GetValue(UINT64 *lo, UINT64 *hi) { - if (sizeof(Crc) <= sizeof(*lo)) { - return CrcFromUint64(*lo); - } else { - return CrcFromUint64(*lo, *hi); - } - } - - static Crc GetValue(UINT64 lo, UINT64 hi) { - return CrcFromUint64(lo, hi); - } - - static void SetValue(const Crc &crc, UINT64 *lo, UINT64 *hi) { - Uint64FromCrc(crc, - reinterpret_cast(lo), - reinterpret_cast(hi)); - } - - const CrcImplementation crc_; - const RollingCrcImplementation rolling_crc_; - - const Self &operator =(const Self &) {} -}; - -#if defined(_MSC_VER) -// 'use_sse4_2' : unreferenced formal parameter -#pragma warning(disable: 4100) -#endif // defined(_MSC_VER) - -bool CRC::IsSSE42Available() { -#if HAVE_AMD64 || HAVE_I386 - return Crc32cSSE4::IsSSE42Available(); -#else - return false; -#endif // HAVE_AMD64 || HAVE_I386 -} - -CRC::~CRC() {} -CRC::CRC() {} - -CRC *CRC::Create(UINT64 poly_lo, - UINT64 poly_hi, - size_t degree, - bool canonical, - UINT64 roll_start_value_lo, - UINT64 roll_start_value_hi, - size_t roll_length, - bool use_sse4_2, - const void **allocated_memory) { - if (degree == 0) { - return NULL; - } - - if (degree > 64) { -#if !HAVE_SSE2 - return NULL; -#else - if (degree > 128) { - return NULL; - } - uint128_sse2 poly = CrcFromUint64(poly_lo, poly_hi); - if (degree != 128 && (poly >> degree) != 0) { - return NULL; - } - uint128_sse2 roll_start_value = - CrcFromUint64(roll_start_value_lo, roll_start_value_hi); - if (degree != 128 && (roll_start_value >> degree) != 0) { - return NULL; - } -#if HAVE_I386 - typedef GenericCrc Crc128; -#elif defined(__GNUC__) && GCC_VERSION_AVAILABLE(4, 5) - typedef GenericCrc Crc128; -#else - typedef GenericCrc Crc128; -#endif // HAVE_I386 - return Implementation >::Create( - poly, - degree, - canonical, - roll_start_value, - roll_length, - allocated_memory); -#endif // !HAVE_SSE2 - } - -#if CRCUTIL_USE_MM_CRC32 && (HAVE_I386 || HAVE_AMD64) - if (use_sse4_2 && - degree == Crc32cSSE4::FixedDegree() && - poly_lo == Crc32cSSE4::FixedGeneratingPolynomial() && - poly_hi == 0) { - if (roll_start_value_hi != 0 || (roll_start_value_lo >> 32) != 0) { - return NULL; - } - return Implementation::Create( - static_cast(poly_lo), - degree, - canonical, - static_cast(roll_start_value_lo), - static_cast(roll_length), - allocated_memory); - } -#endif // CRCUTIL_USE_MM_CRC32 && (HAVE_I386 || HAVE_AMD64) - - if (poly_hi != 0 || (degree != 64 && (poly_lo >> degree) != 0)) { - return NULL; - } - if (roll_start_value_hi != 0 || - (degree != 64 && (roll_start_value_lo >> degree) != 0)) { - return NULL; - } - typedef GenericCrc - Crc64; - return Implementation >::Create( - poly_lo, - degree, - canonical, - roll_start_value_lo, - roll_length, - allocated_memory); -} - -} // namespace crcutil_interface diff --git a/rapidyenc/rapidyenc/crcutil-1.0/examples/interface.h b/rapidyenc/rapidyenc/crcutil-1.0/examples/interface.h deleted file mode 100644 index 2b3e2ee..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/examples/interface.h +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Example how to use CRC implementation via the interface which -// hides details of implementation. -// -// The raw implementation is not indended to be used in a project -// directly because: -// - Implementation lives in the header files because that is the -// only way to use templates efficiently. -// - Header files are quite "dirty" -- they define and use a -// lot of macros. Bringing these macros to all files in -// a project is not particularly good idea. -// - The code takes forever to compile with GCC (e.g. GCC -// 4.4.3 and 4.5.0 compile the unittest for about 30 seconds). -// -// Solution: -// - Create your own, clean interface. -// - Do not expose interface internals in a header file. -// - Proxy all calls to your interface to CRC implementation. -// - Keep only one copy of actual implementation. - -#ifndef CRCUTIL_INTERFACE_H_ -#define CRCUTIL_INTERFACE_H_ - -#include "std_headers.h" // size_t - -namespace crcutil_interface { - -// Many projects define their own uint64. Do it here. -typedef unsigned long long UINT64; - -class CRC { - public: - // Creates new instance of CRC class. - // If arguments are illegal (e.g. provided generating polynomial - // has more bits than provided degree), returns NULL. - // - // poly_* - generating polynomial (reversed bit format). - // degree - degree of generating polynomial. - // canonical - if true, input CRC value will be XOR'ed with - // (inverted) before and after CRC computation. - // roll_start_value - starting value of rolling CRC. - // roll_window_bytes - length of rolling CRC window in bytes. - // If roll_length is 0, roll_start_value - // shall be 0. - // use_sse4_2 - if true, use SSE4.2 crc32 instruction to compute - // CRC when generating polynomial is CRC32C (Castagnoli) - // allocated_memory - optional (may be NULL) address of a variable - // to store the address of actually allocated memory. - static CRC *Create(UINT64 poly_lo, - UINT64 poly_hi, - size_t degree, - bool canonical, - UINT64 roll_start_value_lo, - UINT64 roll_start_value_hi, - size_t roll_window_bytes, - bool use_sse4_2, - const void **allocated_memory); - - // Deletes the instance of CRC class. - virtual void Delete() = 0; - - // Returns true if SSE4.2 is available. - static bool IsSSE42Available(); - - // Returns generating polynomial. - virtual void GeneratingPolynomial(/* OUT */ UINT64 *lo, - /* OUT */ UINT64 *hi = NULL) const = 0; - - // Returns degree of generating polynomial. - virtual size_t Degree() const = 0; - - // Returns canonization constant used to XOR crc value - // before and after CRC computation. - virtual void CanonizeValue(/* OUT */ UINT64 *lo, - /* OUT */ UINT64 *hi = NULL) const = 0; - - // Returns rolling CRC starting value. - virtual void RollStartValue(/* OUT */ UINT64 *lo, - /* OUT */ UINT64 *hi = NULL) const = 0; - - // Returns length of rolling CRC window. - virtual size_t RollWindowBytes() const = 0; - - // Returns CRC of CRC tables to enable verification - // of integrity of CRC function itself by comparing - // the result with pre-computed value. - virtual void SelfCheckValue(/* OUT */ UINT64 *lo, - /* OUT */ UINT64 *hi = NULL) const = 0; - - // Given CRC value of previous chunk of data, - // extends it to new chunk, retuning the result in-place. - // - // If degree of CRC polynomial is 64 or less, - // (*hi) will not be touched. - virtual void Compute(const void *data, - size_t bytes, - /* INOUT */ UINT64 *lo, - /* INOUT */ UINT64 *hi = NULL) const = 0; - - // Starts rolling CRC by computing CRC of first - // "roll_length" bytes of "data", using "roll_start_value" - // as starting value (see Create()). - // Should not be called if the value of "roll_value" was 0. - virtual void RollStart(const void *data, - /* OUT */ UINT64 *lo, - /* OUT */ UINT64 *hi = NULL) const = 0; - - // Rolls CRC by 1 byte, given the bytes leaving and - // entering the window of "roll_length" bytes. - // RollStart() should be called before "Roll". - // Should not be called if the value of "roll_value" was 0. - virtual void Roll(size_t byte_out, - size_t byte_in, - /* INOUT */ UINT64 *lo, - /* INOUT */ UINT64 *hi = NULL) const = 0; - - // Computes CRC of sequence of zeroes -- without touching the data. - virtual void CrcOfZeroes(UINT64 bytes, - /* INOUT */ UINT64 *lo, - /* INOUT */ UINT64 *hi = NULL) const = 0; - - // Computes value of CRC(A, bytes, start_new) given known - // crc=CRC(A, bytes, start_old) -- without touching the data. - virtual void ChangeStartValue( - UINT64 start_old_lo, UINT64 start_old_hi, - UINT64 start_new_lo, UINT64 start_new_hi, - UINT64 bytes, - /* INOUT */ UINT64 *lo, - /* INOUT */ UINT64 *hi = NULL) const = 0; - - // Returns CRC of concatenation of blocks A and B when CRCs - // of blocks A and B are known -- without touching the data. - // - // To be precise, given CRC(A, |A|, startA) and CRC(B, |B|, 0), - // returns CRC(AB, |AB|, startA). - virtual void Concatenate(UINT64 crcB_lo, UINT64 crcB_hi, - UINT64 bytes_B, - /* INOUT */ UINT64* crcA_lo, - /* INOUT */ UINT64* crcA_hi = NULL) const = 0; - - // Given CRC of a message, stores extra (degree + 7)/8 bytes after - // the message so that CRC(message+extra, start) = result. - // Does not change CRC start value (use ChangeStartValue for that). - // Returns number of stored bytes. - virtual size_t StoreComplementaryCrc( - void *dst, - UINT64 message_crc_lo, UINT64 message_crc_hi, - UINT64 result_crc_lo, UINT64 result_crc_hi = 0) const = 0; - - // Stores given CRC of a message as (degree + 7)/8 bytes filled - // with 0s to the right. Returns number of stored bytes. - // CRC of the message and stored CRC is a constant value returned - // by CrcOfCrc() -- it does not depend on contents of the message. - virtual size_t StoreCrc(/* OUT */ void *dst, - UINT64 lo, - UINT64 hi = 0) const = 0; - - // Computes expected CRC value of CRC(Message,CRC(Message)) - // when CRC is stored after the message. This value is fixed - // and does not depend on the message or CRC start value. - virtual void CrcOfCrc(/* OUT */ UINT64 *lo, - /* OUT */ UINT64 *hi = NULL) const = 0; - - protected: - // CRC instance should be created only once (most of the time): - // - Creation and initializion is relatively expensive. - // - CRC is fully defined by its generating polynomials - // (well, and few more parameters). - // - CRC instances are pure constants. There is no - // reason to have 2 instances of the same CRC. - // - There are not too many generating polynomials that are - // used on practice. It is hard to imagine a project - // which uses 50 different generating polynomials. - // Thus, a handful of CRC instances is sufficient - // to cover the needs of even very large project. - // - Finally and most importantly, CRC tables should be - // aligned properly. No, the instances of CRC class - // are not created by blind "new" -- they use placement - // "new" and, in absense of placement "delete", - // should be deleted by calling explicit Delete() method. - virtual ~CRC(); - - // Cannot instantiate the class -- instances may be created - // by CRC::Create() only. - CRC(); -}; - -} // namespace crcutil_interface - - -#endif // CRCUTIL_INTERFACE_H_ diff --git a/rapidyenc/rapidyenc/crcutil-1.0/tests/aligned_alloc.h b/rapidyenc/rapidyenc/crcutil-1.0/tests/aligned_alloc.h deleted file mode 100644 index 37cefba..0000000 --- a/rapidyenc/rapidyenc/crcutil-1.0/tests/aligned_alloc.h +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2010 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Poor man's platform-independent implementation of aligned memory allocator. - -#ifndef CRCUTIL_ALIGNED_ALLOC_H_ -#define CRCUTIL_ALIGNED_ALLOC_H_ - -#include "std_headers.h" // size_t, ptrdiff_t - -namespace crcutil { - -// Allocates a block of memory of "size" bytes so that a field -// at "field_offset" is aligned on "align" boundary. -// -// NB #1: "align" shall be exact power of two. -// -// NB #2: memory allocated by AlignedAlloc should be release by AlignedFree(). -// -inline void *AlignedAlloc(size_t size, - size_t field_offset, - size_t align, - const void **allocated_mem) { - if (align == 0 || (align & (align - 1)) != 0 || align < sizeof(char *)) { - align = sizeof(*allocated_mem); - } - size += align - 1 + sizeof(*allocated_mem); - char *allocated_memory = new char[size]; - char *aligned_memory = allocated_memory + sizeof(*allocated_mem); - field_offset &= align - 1; - size_t actual_alignment = - reinterpret_cast(aligned_memory + field_offset) & (align - 1); - if (actual_alignment != 0) { - aligned_memory += align - actual_alignment; - } - reinterpret_cast(aligned_memory)[-1] = allocated_memory; - - if (allocated_mem != NULL) { - *allocated_mem = allocated_memory; - } - - return aligned_memory; -} - -// Frees memory allocated by AlignedAlloc(). -inline void AlignedFree(void *aligned_memory) { - if (aligned_memory != NULL) { - char *allocated_memory = reinterpret_cast(aligned_memory)[-1]; - delete[] allocated_memory; - } -} - -} // namespace crcutil - -#endif // CRCUTIL_ALIGNED_ALLOC_H_ diff --git a/rapidyenc/rapidyenc/rapidyenc.cc b/rapidyenc/rapidyenc/rapidyenc.cc deleted file mode 100644 index 596be89..0000000 --- a/rapidyenc/rapidyenc/rapidyenc.cc +++ /dev/null @@ -1,126 +0,0 @@ -#ifdef BUILD_SHARED -# ifdef _MSC_VER -# define RAPIDYENC_API __declspec(dllexport) -# else -# define RAPIDYENC_API __attribute__((visibility("default"))) -# endif -#endif - -#include "rapidyenc.h" - -int rapidyenc_version(void) { - return RAPIDYENC_VERSION; -} - -#ifndef RAPIDYENC_DISABLE_ENCODE - -#include "src/encoder.h" -void rapidyenc_encode_init(void) { - static int done = 0; - if(done) return; - done = 1; - RapidYenc::encoder_init(); -} - -size_t rapidyenc_encode(const void* __restrict src, void* __restrict dest, size_t src_length) { - return rapidyenc_encode_ex(128, NULL, src, dest, src_length, 1); -} - -size_t rapidyenc_encode_ex(int line_size, int* column, const void* __restrict src, void* __restrict dest, size_t src_length, int is_end) { - int unusedColumn = 0; - if(!column) column = &unusedColumn; - return RapidYenc::encode(line_size, column, src, dest, src_length, is_end); -} - -int rapidyenc_encode_kernel() { - return RapidYenc::encode_isa_level(); -} - -#endif // !defined(RAPIDYENC_DISABLE_ENCODE) - -size_t rapidyenc_encode_max_length(size_t length, int line_size) { - size_t ret = length * 2 /* all characters escaped */ - + 2 /* allocation for offset and that a newline may occur early */ -#if !defined(YENC_DISABLE_AVX256) - + 64 /* allocation for YMM overflowing */ -#else - + 32 /* allocation for XMM overflowing */ -#endif - ; - /* add newlines, considering the possibility of all chars escaped */ - if(line_size == 128) // optimize common case - return ret + 2 * (length >> 6); - return ret + 2 * ((length*2) / line_size); -} - - -#ifndef RAPIDYENC_DISABLE_DECODE - -#include "src/decoder.h" -void rapidyenc_decode_init(void) { - static int done = 0; - if(done) return; - done = 1; - RapidYenc::decoder_init(); -} - -size_t rapidyenc_decode(const void* src, void* dest, size_t src_length) { - return rapidyenc_decode_ex(1, src, dest, src_length, NULL); -} - -size_t rapidyenc_decode_ex(int is_raw, const void* src, void* dest, size_t src_length, RapidYencDecoderState* state) { - RapidYencDecoderState unusedState = RYDEC_STATE_CRLF; - if(!state) state = &unusedState; - return RapidYenc::decode(is_raw, src, dest, src_length, (RapidYenc::YencDecoderState*)state); -} - -RapidYencDecoderEnd rapidyenc_decode_incremental(const void** src, void** dest, size_t src_length, RapidYencDecoderState* state) { - RapidYencDecoderState unusedState = RYDEC_STATE_CRLF; - if(!state) state = &unusedState; - return (RapidYencDecoderEnd)RapidYenc::decode_end(src, dest, src_length, (RapidYenc::YencDecoderState*)state); -} - -int rapidyenc_decode_kernel() { - return RapidYenc::decode_isa_level(); -} - -#endif // !defined(RAPIDYENC_DISABLE_DECODE) - -#ifndef RAPIDYENC_DISABLE_CRC - -#include "src/crc.h" -void rapidyenc_crc_init(void) { - static int done = 0; - if(done) return; - done = 1; - RapidYenc::crc32_init(); -} - -uint32_t rapidyenc_crc(const void* src, size_t src_length, uint32_t init_crc) { - return RapidYenc::crc32(src, src_length, init_crc); -} -uint32_t rapidyenc_crc_combine(uint32_t crc1, const uint32_t crc2, uint64_t length2) { - return RapidYenc::crc32_combine(crc1, crc2, length2); -} -uint32_t rapidyenc_crc_zeros(uint32_t init_crc, uint64_t length) { - return RapidYenc::crc32_zeros(init_crc, length); -} -uint32_t rapidyenc_crc_unzero(uint32_t init_crc, uint64_t length) { - return RapidYenc::crc32_unzero(init_crc, length); -} -uint32_t rapidyenc_crc_multiply(uint32_t a, uint32_t b) { - return RapidYenc::crc32_multiply(a, b); -} -uint32_t rapidyenc_crc_2pow(int64_t n) { - return RapidYenc::crc32_2pow(n); -} -uint32_t rapidyenc_crc_256pow(uint64_t n) { - return RapidYenc::crc32_256pow(n); -} - -int rapidyenc_crc_kernel() { - return RapidYenc::crc32_isa_level(); -} - -#endif // !defined(RAPIDYENC_DISABLE_CRC) - diff --git a/rapidyenc/rapidyenc/rapidyenc.h b/rapidyenc/rapidyenc/rapidyenc.h deleted file mode 100644 index 3f5cbbe..0000000 --- a/rapidyenc/rapidyenc/rapidyenc.h +++ /dev/null @@ -1,225 +0,0 @@ -#ifndef __RAPIDYENC_H -#define __RAPIDYENC_H -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include - -#ifndef RAPIDYENC_API -# define RAPIDYENC_API -#endif - -/** - * Version, in 0xMMmmpp format, where MM=major version, mm=minor version, pp=patch version - */ -#define RAPIDYENC_VERSION 0x010101 -RAPIDYENC_API int rapidyenc_version(void); // returns RAPIDYENC_VERSION - -/** - * For determining which kernel was selected for the current CPU - * Note that if this was compiled with the BUILD_NATIVE option set, values may not correspond with any of those below - */ -#define RYKERN_GENERIC 0 // generic kernel chosen -// x86 specific encode/decode kernels -#define RYKERN_SSE2 0x100 -#define RYKERN_SSSE3 0x200 -#define RYKERN_AVX 0x381 -#define RYKERN_AVX2 0x403 -#define RYKERN_VBMI2 0x603 -// ARM specific encode/decode kernels -#define RYKERN_NEON 0x1000 -// RISC-V specific encode/decode kernels -#define RYKERN_RVV 0x10000 -// x86 specific CRC32 kernels -#define RYKERN_PCLMUL 0x340 -#define RYKERN_VPCLMUL 0x440 -// ARM specific CRC32 kernels -#define RYKERN_ARMCRC 8 -#define RYKERN_ARMPMULL 0x48 -// RISC-V specific CRC32 kernels -#define RYKERN_ZBC 16 - - -/***** ENCODE *****/ -#ifndef RAPIDYENC_DISABLE_ENCODE -/** - * Initialise global state of the encoder (sets up lookup tables and performs CPU detection). - * As it alters global state, this function only needs to be called once, and is not thread-safe (subsequent calls to this will do nothing). - * This must be called before any other rapidyenc_encode* functions are called. - */ -RAPIDYENC_API void rapidyenc_encode_init(void); - - -/** - * yEnc encode the buffer at `src` (of length `src_length`) and write it to `dest` - * Returns the number of bytes written to `dest` - * `dest` is assumed to be large enough to hold the output - use `rapidyenc_encode_max_length` to compute the necessary size of `dest` - * - * This is effectively an alias for `rapidyenc_encode_ex(128, NULL, src, dest, src_length, 1)` - */ -RAPIDYENC_API size_t rapidyenc_encode(const void* __restrict src, void* __restrict dest, size_t src_length); - -/** - * Like `rapidyenc_encode` but provide the ability to perform incremental processing - * This is done by keeping track of the column position, and you'll need to indicate if this is the last chunk of an article. - * - * - line_size: the target number of bytes for each line. 128 is commonly used - * - column [in/out]: the column in the line to start at. This will be updated with the column position after encoding. Articles will typically start at 0. Pass in NULL to not track the column. - * - src: the source data to encode - * - dest: where to write the encoded data to. This cannot alias the source data - * - src_length: the length of the source data to encode. Note that the length of the output buffer is assumed to be large enough (see `rapidyenc_encode_max_length`) - * - is_end: if not 0, this is the final chunk of the article. Setting this ensures that trailing whitespace is properly escaped - */ -RAPIDYENC_API size_t rapidyenc_encode_ex(int line_size, int* column, const void* __restrict src, void* __restrict dest, size_t src_length, int is_end); - -/** - * Returns the kernel/ISA level used for encoding - * Values correspond with RYKERN_* definitions above - */ -RAPIDYENC_API int rapidyenc_encode_kernel(); - -#endif // !defined(RAPIDYENC_DISABLE_ENCODE) - -/** - * Returns the maximum possible length of yEnc encoded output, given an input of `length` bytes - * This function does also include additional padding needed by rapidyenc's implementation. - * Note that this function doesn't require `rapidyenc_encode_init` to be called beforehand - */ -RAPIDYENC_API size_t rapidyenc_encode_max_length(size_t length, int line_size); - - - -/***** DECODE *****/ -#ifndef RAPIDYENC_DISABLE_DECODE -/** - * Current decoder state, for incremental decoding - * The values here refer to the previously seen characters in the stream, which influence how some sequences need to be handled - * The shorthands represent: - * CR (\r), LF (\n), EQ (=), DT (.) - */ -typedef enum { - RYDEC_STATE_CRLF, // default - RYDEC_STATE_EQ, - RYDEC_STATE_CR, - RYDEC_STATE_NONE, - RYDEC_STATE_CRLFDT, - RYDEC_STATE_CRLFDTCR, - RYDEC_STATE_CRLFEQ // may actually be "\r\n.=" in raw decoder -} RapidYencDecoderState; - -/** - * End state for incremental decoding (whether the end of the yEnc data was reached) - */ -typedef enum { - RYDEC_END_NONE, // end not reached - RYDEC_END_CONTROL, // \r\n=y sequence found, src points to byte after 'y' - RYDEC_END_ARTICLE // \r\n.\r\n sequence found, src points to byte after last '\n' -} RapidYencDecoderEnd; - -/** - * Initialise global state of the decoder (sets up lookup tables and performs CPU detection). - * As it alters global state, this function only needs to be called once, and is not thread-safe (subsequent calls to this will do nothing). - * This must be called before any other rapidyenc_decode* functions are called. - */ -RAPIDYENC_API void rapidyenc_decode_init(void); - -/** - * yEnc decode the buffer at `src` (of length `src_length`) and write it to `dest` - * Returns the number of bytes written to `dest` - * - * This is effectively an alias for `rapidyenc_decode_ex(1, src, dest, src_length, NULL)` - */ -RAPIDYENC_API size_t rapidyenc_decode(const void* src, void* dest, size_t src_length); - -/** - * yEnc decode the buffer at `src` (of length `src_length`) and write it to `dest` - * Returns the number of bytes written to `dest` - * - * If `is_raw` is non-zero, will also handle NNTP dot unstuffing - * `state` can be used to track the decoder state, if incremental decoding is desired. Set to NULL if tracking is not needed - * `src` and `dest` are allowed to point to the same location for in-situ decoding, otherwise `dest` is assumed to be at least `src_length` in size - */ -RAPIDYENC_API size_t rapidyenc_decode_ex(int is_raw, const void* src, void* dest, size_t src_length, RapidYencDecoderState* state); - -/** - * Like `rapidyenc_decode`, but stops decoding when a yEnc/NNTP end sequence is found - * Returns whether such an end sequence was found - * Note that the `is_raw` parameter in `rapidyenc_decode` is assumed to be True here - * - * `src` and `dest` are pointers of pointers here, as they'll both be updated to the positions after decoding - * The length of the written data can thus be derived from the post-decode `dest` minus the pre-decode `dest` - * Whilst `src` and `dest` can point to the same memory, the pointers themselves should be different. In other words, `**src == **dest` is fine, but `*src == *dest` is not - */ -RAPIDYENC_API RapidYencDecoderEnd rapidyenc_decode_incremental(const void** src, void** dest, size_t src_length, RapidYencDecoderState* state); - -/** - * Returns the kernel/ISA level used for decoding - * Values correspond with RYKERN_* definitions above - */ -RAPIDYENC_API int rapidyenc_decode_kernel(); - -#endif // !defined(RAPIDYENC_DISABLE_DECODE) - - -/***** CRC32 *****/ -#ifndef RAPIDYENC_DISABLE_CRC -/** - * Initialise global state for CRC32 computation (performs CPU detection). - * As it alters global state, this function only needs to be called once, and is not thread-safe (subsequent calls to this will do nothing). - * This must be called before any other rapidyenc_crc* functions are called. - */ -RAPIDYENC_API void rapidyenc_crc_init(void); - -/** - * Returns the CRC32 hash of `src` (of length `src_length`), with initial CRC32 value `init_crc` - * The initial value should be 0 unless this is a subsequent call during incremental hashing - */ -RAPIDYENC_API uint32_t rapidyenc_crc(const void* src, size_t src_length, uint32_t init_crc); - -/** - * Given `crc1 = CRC32(data1)` and `crc2 = CRC32(data2)`, returns CRC32(data1 + data2) - * `length2` refers to the length of 'data2' - */ -RAPIDYENC_API uint32_t rapidyenc_crc_combine(uint32_t crc1, const uint32_t crc2, uint64_t length2); - -/** - * Returns `rapidyenc_crc(src, length, init_crc)` where 'src' is all zeroes - */ -RAPIDYENC_API uint32_t rapidyenc_crc_zeros(uint32_t init_crc, uint64_t length); - -/** - * Performs the inverse of `rapidyenc_crc_zeros`: - * Given `init_crc = CRC32(data + [0]*length)`, returns `CRC32(data)` - */ -RAPIDYENC_API uint32_t rapidyenc_crc_unzero(uint32_t init_crc, uint64_t length); - -/** - * Returns the product of `a` and `b` in the CRC32 field - */ -RAPIDYENC_API uint32_t rapidyenc_crc_multiply(uint32_t a, uint32_t b); - -/** - * Returns 2**n in the CRC32 field. n can be negative - */ -RAPIDYENC_API uint32_t rapidyenc_crc_2pow(int64_t n); - -/** - * Returns 2**(8n) in the CRC32 field - * Similar to `rapidyenc_crc_2pow(8*n)`, but avoids overflow and n cannot be negative - */ -RAPIDYENC_API uint32_t rapidyenc_crc_256pow(uint64_t n); - -/** - * Returns the kernel/ISA level used for CRC32 computation - * Values correspond with RYKERN_* definitions above - */ -RAPIDYENC_API int rapidyenc_crc_kernel(); - -#endif // !defined(RAPIDYENC_DISABLE_CRC) - -#ifdef __cplusplus -} -#endif -#endif /* __RAPIDYENC_H */ diff --git a/rapidyenc/rapidyenc/src/common.h b/rapidyenc/rapidyenc/src/common.h deleted file mode 100644 index 37dcaef..0000000 --- a/rapidyenc/rapidyenc/src/common.h +++ /dev/null @@ -1,364 +0,0 @@ -#ifndef __YENC_COMMON -#define __YENC_COMMON - -#include "hedley.h" - -#if defined(__x86_64__) || \ - defined(__amd64__ ) || \ - defined(__LP64 ) || \ - defined(_M_X64 ) || \ - defined(_M_AMD64 ) || \ - (defined(_WIN64) && !defined(_M_ARM64)) - #define PLATFORM_AMD64 1 -#endif -#if defined(PLATFORM_AMD64) || \ - defined(__i386__ ) || \ - defined(__i486__ ) || \ - defined(__i586__ ) || \ - defined(__i686__ ) || \ - defined(_M_I86 ) || \ - defined(_M_IX86 ) || \ - (defined(_WIN32) && !defined(_M_ARM) && !defined(_M_ARM64)) - #define PLATFORM_X86 1 -#endif -#if defined(__aarch64__) || \ - defined(__armv7__ ) || \ - defined(__arm__ ) || \ - defined(_M_ARM64 ) || \ - defined(_M_ARM ) || \ - defined(__ARM_ARCH_6__ ) || \ - defined(__ARM_ARCH_7__ ) || \ - defined(__ARM_ARCH_7A__) || \ - defined(__ARM_ARCH_8A__) || \ - (defined(__ARM_ARCH ) && __ARM_ARCH >= 6) - #define PLATFORM_ARM 1 -#endif - - -#include -#if defined(_MSC_VER) || defined(__MINGW32__) || defined(__MINGW64__) - // MSVC doesn't support C11 aligned_alloc: https://stackoverflow.com/a/62963007 - #define ALIGN_ALLOC(buf, len, align) *(void**)&(buf) = _aligned_malloc((len), align) - #define ALIGN_FREE _aligned_free -#elif defined(_ISOC11_SOURCE) - // C11 method - // len needs to be a multiple of alignment, although it sometimes works if it isn't... - #define ALIGN_ALLOC(buf, len, align) *(void**)&(buf) = aligned_alloc(align, ((len) + (align)-1) & ~((align)-1)) - #define ALIGN_FREE free -#elif defined(__cplusplus) && __cplusplus >= 201700 - // C++17 method - #include - #define ALIGN_ALLOC(buf, len, align) *(void**)&(buf) = std::aligned_alloc(align, ((len) + (align)-1) & ~((align)-1)) - #define ALIGN_FREE free -#else - #define ALIGN_ALLOC(buf, len, align) if(posix_memalign((void**)&(buf), align, (len))) (buf) = NULL - #define ALIGN_FREE free -#endif - - -// MSVC compatibility -#if ((defined(_M_IX86_FP) && _M_IX86_FP == 2) || defined(_M_X64)) && defined(_MSC_VER) && !defined(__clang__) - #define __SSE2__ 1 - #define __SSSE3__ 1 - #define __SSE4_1__ 1 - #if _MSC_VER >= 1600 && defined(__SSE2__) - #define __POPCNT__ 1 - #define __LZCNT__ 1 - #endif - #if !defined(__AVX__) && (_MSC_VER >= 1700 && defined(__SSE2__)) - #define __AVX__ 1 - #endif - #if !defined(__AVX2__) && (_MSC_VER >= 1800 && defined(__AVX__)) - #define __AVX2__ 1 - #define __BMI2__ 1 - #endif - /* AVX512 requires VS 15.3 */ - #if !defined(__AVX512F__) && (_MSC_VER >= 1911 && defined(__AVX__)) - #define __AVX512BW__ 1 - #define __AVX512F__ 1 - #endif - /* AVX512VL not available until VS 15.5 */ - #if defined(__AVX512F__) && _MSC_VER >= 1912 - #define __AVX512VL__ 1 - #endif - #if defined(__AVX512F__) && _MSC_VER >= 1920 - #define __AVX512VBMI__ 1 - #define __AVX512VBMI2__ 1 - #endif -#endif -#if defined(_M_ARM64) - #define __aarch64__ 1 - #define __ARM_NEON 1 -#endif -#if defined(_M_ARM) - #define __ARM_NEON 1 -#endif -#ifdef _MSC_VER -# ifndef __BYTE_ORDER__ -# define __BYTE_ORDER__ 1234 -# endif -# ifndef __ORDER_BIG_ENDIAN__ -# define __ORDER_BIG_ENDIAN__ 4321 -# endif -# include -#endif - - -// combine two 8-bit ints into a 16-bit one -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -#define UINT16_PACK(a, b) (((a) << 8) | (b)) -#define UINT32_PACK(a, b, c, d) (((a) << 24) | ((b) << 16) | ((c) << 8) | (d)) -#define UINT32_16_PACK(a, b) (((a) << 16) | (b)) -#else -#define UINT16_PACK(a, b) ((a) | ((b) << 8)) -#define UINT32_PACK(a, b, c, d) ((a) | ((b) << 8) | ((c) << 16) | ((d) << 24)) -#define UINT32_16_PACK(a, b) ((a) | ((b) << 16)) -#endif - -#ifdef __SSE2__ -#include -#define XMM_SIZE 16 /*== (signed int)sizeof(__m128i)*/ - -#ifdef __SSSE3__ -#include -#endif -#ifdef __POPCNT__ -#include -// POPCNT can never return a negative result, but GCC doesn't seem to realise this, so typecast it to hint it better -#define popcnt32 (unsigned int)_mm_popcnt_u32 -#endif - -#if defined(__AVX2__) || defined(__AVX512F__) -#include -#endif - - -#if defined(__tune_core2__) || defined(__tune_atom__) -/* on older Intel CPUs, plus first gen Atom, it is faster to store XMM registers in half */ -# define STOREU_XMM(dest, xmm) \ - _mm_storel_epi64((__m128i*)(dest), xmm); \ - _mm_storeh_pi(((__m64*)(dest) +1), _mm_castsi128_ps(xmm)) -#else -# define STOREU_XMM(dest, xmm) \ - _mm_storeu_si128((__m128i*)(dest), xmm) -#endif - -#endif - -#if defined(__ARM_NEON) && defined(__has_include) -# if !__has_include() -# undef __ARM_NEON -HEDLEY_WARNING("NEON has been disabled due to missing arm_neon.h"); -# endif -#endif - -#ifdef __ARM_NEON -# include - -// ARM provides no standard way to inline define a vector :( -static HEDLEY_ALWAYS_INLINE uint8x8_t vmake_u8( - uint8_t a, uint8_t b, uint8_t c, uint8_t d, uint8_t e, uint8_t f, uint8_t g, uint8_t h -) { -# if defined(_MSC_VER) - uint8_t t[] = {a,b,c,d,e,f,g,h}; - return vld1_u8(t); -# else - return (uint8x8_t){a,b,c,d,e,f,g,h}; -# endif -} -static HEDLEY_ALWAYS_INLINE uint8x16_t vmakeq_u8( - uint8_t a, uint8_t b, uint8_t c, uint8_t d, uint8_t e, uint8_t f, uint8_t g, uint8_t h, - uint8_t i, uint8_t j, uint8_t k, uint8_t l, uint8_t m, uint8_t n, uint8_t o, uint8_t p -) { -# if defined(_MSC_VER) - uint8_t t[] = {a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p}; - return vld1q_u8(t); -# else - return (uint8x16_t){a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p}; -# endif -} -static HEDLEY_ALWAYS_INLINE int8x16_t vmakeq_s8( - int8_t a, int8_t b, int8_t c, int8_t d, int8_t e, int8_t f, int8_t g, int8_t h, - int8_t i, int8_t j, int8_t k, int8_t l, int8_t m, int8_t n, int8_t o, int8_t p -) { -# if defined(_MSC_VER) - int8_t t[] = {a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p}; - return vld1q_s8(t); -# else - return (int8x16_t){a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p}; -# endif -} - -# ifdef _MSC_VER -# define _CREATE_TUPLE(type, ...) type{{ __VA_ARGS__ }} -# else -# define _CREATE_TUPLE(type, ...) (type){{ __VA_ARGS__ }} -# endif -static HEDLEY_ALWAYS_INLINE uint8x16x2_t vcreate2_u8(uint8x16_t a, uint8x16_t b) { - return _CREATE_TUPLE(uint8x16x2_t, a, b); -} -static HEDLEY_ALWAYS_INLINE int8x16x2_t vcreate2_s8(int8x16_t a, int8x16_t b) { - return _CREATE_TUPLE(int8x16x2_t, a, b); -} -static HEDLEY_ALWAYS_INLINE uint8x16x3_t vcreate3_u8(uint8x16_t a, uint8x16_t b, uint8x16_t c) { - return _CREATE_TUPLE(uint8x16x3_t, a, b, c); -} -static HEDLEY_ALWAYS_INLINE uint8x16x4_t vcreate4_u8(uint8x16_t a, uint8x16_t b, uint8x16_t c, uint8x16_t d) { - return _CREATE_TUPLE(uint8x16x4_t, a, b, c, d); -} -# undef _CREATE_TUPLE -#endif -#ifdef PLATFORM_ARM -namespace RapidYenc { - bool cpu_supports_neon(); -} -#endif - -#ifdef _MSC_VER -#define ALIGN_TO(a, v) __declspec(align(a)) v -#else -#define ALIGN_TO(a, v) v __attribute__((aligned(a))) -#endif - - -#ifdef PLATFORM_X86 -enum YEncDecIsaLevel { - ISA_GENERIC = 0, - ISA_FEATURE_POPCNT = 0x1, - ISA_FEATURE_LZCNT = 0x2, - ISA_FEATURE_EVEX512 = 0x4, // AVX512 support - ISA_LEVEL_SSE2 = 0x100, - ISA_LEVEL_SSSE3 = 0x200, - ISA_LEVEL_SSE41 = 0x300, - ISA_LEVEL_SSE4_POPCNT = 0x301, - ISA_LEVEL_PCLMUL = 0x340, - ISA_LEVEL_AVX = 0x381, // same as above, just used as a differentiator for `cpu_supports_isa` - ISA_LEVEL_AVX2 = 0x403, // also includes BMI1/2 and LZCNT - ISA_LEVEL_VPCLMUL = 0x440, - ISA_LEVEL_AVX3 = 0x507, // SKX variant; AVX512VL + AVX512BW - ISA_LEVEL_VBMI2 = 0x603 // ICL, AVX10 -}; -#elif defined(PLATFORM_ARM) -enum YEncDecIsaLevel { - ISA_GENERIC = 0, - ISA_FEATURE_CRC = 8, - ISA_FEATURE_PMULL = 0x40, - ISA_LEVEL_NEON = 0x1000 -}; -#elif defined(__riscv) -enum YEncDecIsaLevel { - ISA_GENERIC = 0, - ISA_FEATURE_ZBC = 16, - ISA_LEVEL_RVV = 0x10000 -}; -#else -enum YEncDecIsaLevel { - ISA_GENERIC = 0 -}; -#endif -#ifdef PLATFORM_X86 -#ifdef _MSC_VER -// native tuning not supported in MSVC -# define ISA_NATIVE ISA_LEVEL_SSE2 -#else -# if defined(__AVX512VBMI2__) -# define _ISA_NATIVE ISA_LEVEL_VBMI2 -# elif defined(__AVX512BW__) -# define _ISA_NATIVE ISA_LEVEL_AVX3 -# elif defined(__AVX2__) -# define _ISA_NATIVE ISA_LEVEL_AVX2 -# elif defined(__SSE4_1__) -# define _ISA_NATIVE ISA_LEVEL_SSE41 -# elif defined(__SSSE3__) -# define _ISA_NATIVE ISA_LEVEL_SSSE3 -# else -# define _ISA_NATIVE ISA_LEVEL_SSE2 -# endif -# if defined(__POPCNT__) -# if defined(__LZCNT__) -# define ISA_NATIVE (enum YEncDecIsaLevel)(_ISA_NATIVE | ISA_FEATURE_POPCNT | ISA_FEATURE_LZCNT) -# else -# define ISA_NATIVE (enum YEncDecIsaLevel)(_ISA_NATIVE | ISA_FEATURE_POPCNT) -# endif -# else -# define ISA_NATIVE _ISA_NATIVE -# endif -#endif - -namespace RapidYenc { - int cpu_supports_isa(); - int cpu_supports_crc_isa(); -} -#endif // PLATFORM_X86 - - -#ifdef __riscv -namespace RapidYenc { - bool cpu_supports_rvv(); -} -#endif -#if defined(__riscv_vector) && defined(HEDLEY_GCC_VERSION) && !HEDLEY_GCC_VERSION_CHECK(13,0,0) -// GCC added RVV intrinsics in GCC13 -# undef __riscv_vector -#elif defined(__riscv_vector) && defined(HEDLEY_GCC_VERSION) && !HEDLEY_GCC_VERSION_CHECK(14,0,0) -// ...however, GCC13 lacks necessary mask<>vector vreinterpret casts, and it crashes on type punning, so I can't be bothered trying to make it work -# undef __riscv_vector -#endif -#ifdef __riscv_vector -# include -# ifdef __riscv_v_intrinsic -# define RV(f) __riscv_##f -# else -# define RV(f) f -# endif -# if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 12000 -# define RV_MASK_CAST(masksz, vecsz, vec) RV(vreinterpret_v_u##vecsz##m1_b##masksz)(vec) -# define RV_VEC_U8MF4_CAST(vec) RV(vlmul_trunc_v_u8m1_u8mf4)(RV(vreinterpret_v_b4_u8m1)(vec)) -# else -# define RV_MASK_CAST(masksz, vecsz, vec) *(vbool##masksz##_t*)(&(vec)) -# define RV_VEC_U8MF4_CAST(vec) *(vuint8mf4_t*)(&(vec)) -# endif -#endif - -#include -#if !defined(_MSC_VER) || defined(_STDINT) || _MSC_VER >= 1900 -# include -# include -#else -/* Workaround for older MSVC not supporting stdint.h - just pull it from V8 */ -# if defined(NODE_GYP_MODULE_NAME) || defined(V8_DEPRECATION_WARNINGS) -# include -# else -# include "stdint.h" -# endif -#endif - - -// GCC 8/9/10(dev) fails to optimize cases where KNOT should be used, so use intrinsic explicitly; Clang 6+ has no issue, but Clang 6/7 doesn't have the intrinsic; MSVC 2019 also fails and lacks the intrinsic -#if (defined(__GNUC__) && __GNUC__ >= 7) || (defined(_MSC_VER) && _MSC_VER >= 1924) -# define KNOT16 _knot_mask16 -# define KNOT32 _knot_mask32 -#else -# define KNOT16(x) ((__mmask16)~(x)) -# define KNOT32(x) ((__mmask32)~(x)) -#endif - -// weird thing with Apple's Clang; doesn't seem to always occur, so assume that Clang >= 9 is fine: https://github.com/animetosho/node-yencode/issues/8#issuecomment-583385864 -// seems that Clang < 3.6 also uses the old name -#if defined(__clang__) && ((defined(__APPLE__) && __clang_major__ < 9) || __clang_major__ < 3 || (__clang_major__ == 3 && __clang_minor__ < 6)) -# define _lzcnt_u32 __lzcnt32 -#endif - - - -#ifdef __GNUC__ -# if __GNUC__ >= 9 -# define LIKELIHOOD(p, c) (HEDLEY_PREDICT(!!(c), 1, p)) -# else -# define LIKELIHOOD(p, c) (p>0.3 && p<0.7 ? HEDLEY_UNPREDICTABLE(!!(c)) : __builtin_expect(!!(c), (p >= 0.5))) -# endif -#else -# define LIKELIHOOD(p, c) (c) -#endif - -#endif /* __YENC_COMMON */ diff --git a/rapidyenc/rapidyenc/src/crc.cc b/rapidyenc/rapidyenc/src/crc.cc deleted file mode 100644 index 6d854df..0000000 --- a/rapidyenc/rapidyenc/src/crc.cc +++ /dev/null @@ -1,297 +0,0 @@ -#include "crc_common.h" - -#if defined(PLATFORM_X86) && !defined(__ILP32__) && !defined(YENC_DISABLE_CRCUTIL) -// Use crcutil for computing CRC32 (generic implementation) - -#include "interface.h" -crcutil_interface::CRC* crc = NULL; -#define GENERIC_CRC_INIT crc = crcutil_interface::CRC::Create(0xEDB88320, 0, 32, true, 0, 0, 0, 0, NULL) -// instance never deleted... oh well... - -static uint32_t do_crc32_incremental_generic(const void* data, size_t length, uint32_t init) { - // use optimised ASM on x86 platforms - crcutil_interface::UINT64 tmp = init; - crc->Compute(data, length, &tmp); - return (uint32_t)tmp; -} - -#else -// don't use crcutil - -static uint32_t* HEDLEY_RESTRICT crc_slice_table; -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -# if defined(__GNUC__) || defined(__clang__) -# define bswap32 __builtin_bswap32 -# else -static inline uint32_t bswap32(uint32_t x) { - return (x >> 24) | ((x >> 8) & 0x0000FF00) | ((x << 8) & 0x00FF0000) | (x << 24); -} -# endif -#endif - -#define CRC32_GENERIC_CHAINS 4 // newer processors may prefer 8 -static uint32_t do_crc32_incremental_generic(const void* data, size_t length, uint32_t init) { - const uint32_t* crc_base_table = crc_slice_table + 4*256; // this also seems to help MSVC's optimiser, which otherwise keeps trying to add to crc_slice_table every time it's referenced - uint32_t crc[CRC32_GENERIC_CHAINS]; // Clang seems to be more spill happy with an array over individual variables :( - crc[0] = ~init; - uint8_t* current8 = (uint8_t*)data; - - // align to multiple of 4 - if(((uintptr_t)current8 & 1) && length >= 1) { - crc[0] = (crc[0] >> 8) ^ crc_base_table[(crc[0] & 0xFF) ^ *current8++]; - length--; - } - if(((uintptr_t)current8 & 2) && length >= 2) { - crc[0] = (crc[0] >> 8) ^ crc_base_table[(crc[0] & 0xFF) ^ *current8++]; - crc[0] = (crc[0] >> 8) ^ crc_base_table[(crc[0] & 0xFF) ^ *current8++]; - length -= 2; - } - - uint8_t* end8 = current8 + length; - uint32_t* current = (uint32_t*)current8; - if(length >= 8*CRC32_GENERIC_CHAINS-4) { - size_t lenMain = ((length-(CRC32_GENERIC_CHAINS-1)*4) / 4); - uint32_t* end = current + (lenMain / CRC32_GENERIC_CHAINS) * CRC32_GENERIC_CHAINS; - for(int c=1; c> 24] ^ crc_slice_table[0x100L + ((v >> 16) & 0xff)] ^ crc_slice_table[0x200L + ((v >> 8) & 0xff)] ^ crc_slice_table[0x300L + (v & 0xff)] -#else - #define CRC_PROC4(v, in) \ - v ^= (in); \ - v = crc_slice_table[v >> 24] ^ crc_slice_table[0x100L + ((v >> 16) & 0xff)] ^ crc_slice_table[0x200L + ((v >> 8) & 0xff)] ^ crc_slice_table[0x300L + (v & 0xff)] -#endif - for(int c=0; c> 8) ^ crc_base_table[(crc[c] & 0xff) ^ *current8++]; - crc[(c+1) & ~CRC32_GENERIC_CHAINS] ^= crc[c]; - } - } else - #endif - #undef CRC_PROC4 - for(int c=1; c> 8) ^ crc_base_table[(crc[0] & 0xff) ^ *current8++]; - crc[0] ^= crc[c]; - } - } - - // tail loop - while(current8 != end8) { - crc[0] = (crc[0] >> 8) ^ crc_base_table[(crc[0] & 0xFF) ^ *current8++]; - } - return ~crc[0]; -} -static void generate_crc32_slice_table() { - crc_slice_table = (uint32_t*)malloc(5*256*sizeof(uint32_t)); - // generate standard byte-by-byte table - uint32_t* crc_base_table = crc_slice_table + 4*256; - for(int v=0; v<256; v++) { - uint32_t crc = v; - for(int j = 0; j < 8; j++) { - crc = (crc >> 1) ^ (-(int32_t)(crc & 1) & 0xEDB88320); - } - crc_base_table[v] = crc; - } - - // generate slice-by-4 shifted across for X independent chains - for(int v=0; v<256; v++) { - uint32_t crc = crc_base_table[v]; - #if CRC32_GENERIC_CHAINS > 1 - for(int i=0; i<4*CRC32_GENERIC_CHAINS-5; i++) - crc = (crc >> 8) ^ crc_base_table[crc & 0xff]; - for(int i=0; i<4; i++) { - crc = (crc >> 8) ^ crc_base_table[crc & 0xff]; - crc_slice_table[i*256 + v] = crc; - } - #else - for(int i=0; i<4; i++) { - crc_slice_table[i*256 + v] = crc; - crc = (crc >> 8) ^ crc_base_table[crc & 0xff]; - } - #endif - } -} - -#define GENERIC_CRC_INIT generate_crc32_slice_table() -#endif - - -namespace RapidYenc { - -// workaround MSVC complaining "unary minus operator applied to unsigned type, result still unsigned" -#define NEGATE(n) (uint32_t)(-((int32_t)(n))) -uint32_t crc32_multiply_generic(uint32_t a, uint32_t b) { - uint32_t res = 0; - for(int i=0; i<31; i++) { - res ^= NEGATE(b>>31) & a; - a = ((a >> 1) ^ (0xEDB88320 & NEGATE(a&1))); - b <<= 1; - } - res ^= NEGATE(b>>31) & a; - return res; -} -#undef NEGATE - -const uint32_t crc_power[32] = { // pre-computed 2^(2^n) - 0x40000000, 0x20000000, 0x08000000, 0x00800000, 0x00008000, 0xedb88320, 0xb1e6b092, 0xa06a2517, - 0xed627dae, 0x88d14467, 0xd7bbfe6a, 0xec447f11, 0x8e7ea170, 0x6427800e, 0x4d47bae0, 0x09fe548f, - 0x83852d0f, 0x30362f1a, 0x7b5a9cc3, 0x31fec169, 0x9fec022a, 0x6c8dedc4, 0x15d6874d, 0x5fde7a4e, - 0xbad90e37, 0x2e4e5eef, 0x4eaba214, 0xa8a472c0, 0x429a969e, 0x148d302a, 0xc40ba6d0, 0xc4e22c3c -}; - -uint32_t crc32_shift_generic(uint32_t crc1, uint32_t n) { - uint32_t result = crc1; -#ifdef __GNUC__ - while(n) { - result = crc32_multiply_generic(result, crc_power[__builtin_ctz(n)]); - n &= n-1; - } -#elif defined(_MSC_VER) - unsigned long power; - while(_BitScanForward(&power, n)) { - result = crc32_multiply_generic(result, crc_power[power]); - n &= n-1; - } -#else - unsigned power = 0; - while(n) { - if(n & 1) { - result = crc32_multiply_generic(result, crc_power[power]); - } - n >>= 1; - power++; - } -#endif - return result; -} -} // namespace - - -namespace RapidYenc { - crc_func _do_crc32_incremental = &do_crc32_incremental_generic; - crc_mul_func _crc32_shift = &crc32_shift_generic; - crc_mul_func _crc32_multiply = &crc32_multiply_generic; - int _crc32_isa = ISA_GENERIC; -} - - - -#if defined(PLATFORM_ARM) && defined(_WIN32) -# define WIN32_LEAN_AND_MEAN -# include -#endif -#ifdef PLATFORM_ARM -# ifdef __ANDROID__ -# include -# elif defined(__APPLE__) -# include -# include -# elif defined(__has_include) -# if __has_include() -# include -# if defined(__FreeBSD__) || defined(__OpenBSD__) -static unsigned long getauxval(unsigned long cap) { - unsigned long ret; - elf_aux_info(cap, &ret, sizeof(ret)); - return ret; -} -# endif -# if __has_include() -# include -# endif -# endif -# endif -#endif -#if defined(__riscv) && defined(__has_include) -# if __has_include() -# include -# include -# include -# endif -#endif - -void RapidYenc::crc32_init() { - GENERIC_CRC_INIT; - -#ifdef PLATFORM_X86 - int support = cpu_supports_crc_isa(); - if(support == 2) - crc_clmul256_set_funcs(); - else if(support == 1) - crc_clmul_set_funcs(); -#endif -#ifdef PLATFORM_ARM -# ifdef __APPLE__ - int supports_crc = 0; - int supports_pmull = 0; - size_t len = sizeof(supports_crc); - if(sysctlbyname("hw.optional.armv8_crc32", &supports_crc, &len, NULL, 0)) - supports_crc = 0; - if(sysctlbyname("hw.optional.arm.FEAT_PMULL", &supports_pmull, &len, NULL, 0)) - supports_pmull = 0; -# else - bool supports_crc = false; - bool supports_pmull = false; -# if defined(AT_HWCAP2) && defined(HWCAP2_CRC32) - supports_crc = getauxval(AT_HWCAP2) & HWCAP2_CRC32; -# elif defined(AT_HWCAP) && defined(HWCAP_CRC32) - supports_crc = getauxval(AT_HWCAP) & HWCAP_CRC32; -# elif defined(ANDROID_CPU_FAMILY_ARM) && defined(__aarch64__) - supports_crc = android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_CRC32; - supports_pmull = android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_PMULL; -# elif defined(ANDROID_CPU_FAMILY_ARM) /* aarch32 */ - supports_crc = android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_CRC32; - supports_pmull = android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_PMULL; -# elif defined(_WIN32) - supports_crc = IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE); - supports_pmull = IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE); -# else - #ifdef __ARM_FEATURE_CRC32 - supports_crc = true; /* assume available if compiled as such */ - #endif - #ifdef __ARM_FEATURE_CRYPTO - supports_pmull = true; - #endif -# endif -# if defined(AT_HWCAP2) && defined(HWCAP2_PMULL) - supports_pmull = getauxval(AT_HWCAP2) & HWCAP2_PMULL; -# elif defined(AT_HWCAP) && defined(HWCAP_PMULL) - supports_pmull = getauxval(AT_HWCAP) & HWCAP_PMULL; -# endif -# endif - - if(supports_crc) { - crc_arm_set_funcs(); - if(supports_pmull) crc_pmull_set_funcs(); - } -#endif -#ifdef __riscv -# if defined(RISCV_HWPROBE_KEY_IMA_EXT_0) && defined(__NR_riscv_hwprobe) - const int rv_hwprobe_ext_zbc = 1 << 7, rv_hwprobe_ext_zbkc = 1 << 9; - struct riscv_hwprobe p; - p.key = RISCV_HWPROBE_KEY_IMA_EXT_0; - if(!syscall(__NR_riscv_hwprobe, &p, 1, 0, NULL, 0)) { - if(p.value & (rv_hwprobe_ext_zbc | rv_hwprobe_ext_zbkc)) { - crc_riscv_set_funcs(); - } - } -# endif -#endif -} diff --git a/rapidyenc/rapidyenc/src/crc.h b/rapidyenc/rapidyenc/src/crc.h deleted file mode 100644 index ae9f6d0..0000000 --- a/rapidyenc/rapidyenc/src/crc.h +++ /dev/null @@ -1,91 +0,0 @@ -#ifndef __YENC_CRC_H -#define __YENC_CRC_H -#include // for llabs - -#if !defined(__GNUC__) && defined(_MSC_VER) -# include -#endif - -namespace RapidYenc { - - -typedef uint32_t (*crc_func)(const void*, size_t, uint32_t); -extern crc_func _do_crc32_incremental; - -extern int _crc32_isa; -static inline uint32_t crc32(const void* data, size_t length, uint32_t init) { - return (*_do_crc32_incremental)(data, length, init); -} -static inline int crc32_isa_level() { - return _crc32_isa; -} - - -// computes `n % 0xffffffff` (well, almost), using some bit-hacks -static inline uint32_t crc32_powmod(uint64_t n) { -#if defined(__GNUC__) && (__GNUC__ >= 5 || (defined(__clang__) && (__clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ > 3)))) - unsigned res; - unsigned carry = __builtin_uadd_overflow(n >> 32, n, &res); - res += carry; - return res; -#elif defined(_MSC_VER) && defined(PLATFORM_X86) - unsigned res; - unsigned char carry = _addcarry_u32(0, n >> 32, n, &res); - _addcarry_u32(carry, res, 0, &res); - return res; -#else - n = (n >> 32) + (n & 0xffffffff); - n += n >> 32; - return n; -#endif -} -// computes `crc32_powmod(n*8)` avoiding overflow -static inline uint32_t crc32_bytepow(uint64_t n) { -#if defined(__GNUC__) || defined(_MSC_VER) - unsigned res = crc32_powmod(n); -# ifdef _MSC_VER - return _rotl(res, 3); -# else - return (res << 3) | (res >> 29); -# endif -#else - n = (n >> 32) + (n & 0xffffffff); - n <<= 3; - n += n >> 32; - return n; -#endif -} - -typedef uint32_t (*crc_mul_func)(uint32_t, uint32_t); -extern crc_mul_func _crc32_shift; -extern crc_mul_func _crc32_multiply; -static inline uint32_t crc32_shift(uint32_t a, uint32_t b) { - return (*_crc32_shift)(a, b); -} -static inline uint32_t crc32_multiply(uint32_t a, uint32_t b) { - return (*_crc32_multiply)(a, b); -} - -static inline uint32_t crc32_combine(uint32_t crc1, uint32_t crc2, uint64_t len2) { - return crc32_shift(crc1, crc32_bytepow(len2)) ^ crc2; -} -static inline uint32_t crc32_zeros(uint32_t crc1, uint64_t len) { - return ~crc32_shift(~crc1, crc32_bytepow(len)); -} -static inline uint32_t crc32_unzero(uint32_t crc1, uint64_t len) { - return ~crc32_shift(~crc1, ~crc32_bytepow(len)); -} -static inline uint32_t crc32_2pow(int64_t n) { - uint32_t sign = (uint32_t)(n >> 63); - return crc32_shift(0x80000000, crc32_powmod(llabs(n)) ^ sign); -} -static inline uint32_t crc32_256pow(uint64_t n) { - return crc32_shift(0x80000000, crc32_bytepow(n)); -} - -void crc32_init(); - - - -} // namespace -#endif // defined(__YENC_CRC_H) diff --git a/rapidyenc/rapidyenc/src/crc_arm.cc b/rapidyenc/rapidyenc/src/crc_arm.cc deleted file mode 100644 index 6eb6815..0000000 --- a/rapidyenc/rapidyenc/src/crc_arm.cc +++ /dev/null @@ -1,226 +0,0 @@ -#include "crc_common.h" - -#if defined(PLATFORM_ARM) && defined(_MSC_VER) && defined(__clang__) && !defined(__ARM_FEATURE_CRC32) -// I don't think GYP provides a nice way to detect whether MSVC or clang-cl is being used, but it doesn't use clang-cl by default, so a warning here is probably sufficient -HEDLEY_WARNING("CRC32 acceleration is not been enabled under ARM clang-cl by default; add `-march=armv8-a+crc` to additional compiler arguments to enable"); -#endif - -// disable CRC on GCC versions with broken arm_acle.h -#if defined(__ARM_FEATURE_CRC32) && defined(HEDLEY_GCC_VERSION) -# if !defined(__aarch64__) && HEDLEY_GCC_VERSION_CHECK(7,0,0) && !HEDLEY_GCC_VERSION_CHECK(8,1,1) -# undef __ARM_FEATURE_CRC32 -HEDLEY_WARNING("CRC32 acceleration has been disabled due to broken arm_acle.h shipped in GCC 7.0 - 8.1 [https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81497]. If you need this feature, please use a different compiler or version of GCC"); -# endif -# if defined(__aarch64__) && HEDLEY_GCC_VERSION_CHECK(9,4,0) && !HEDLEY_GCC_VERSION_CHECK(9,5,0) -# undef __ARM_FEATURE_CRC32 -HEDLEY_WARNING("CRC32 acceleration has been disabled due to broken arm_acle.h shipped in GCC 9.4 [https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100985]. If you need this feature, please use a different compiler or version of GCC"); -# endif -#endif -#if defined(__ARM_FEATURE_CRC32) && defined(__has_include) -# if !__has_include() -# undef __ARM_FEATURE_CRC32 -HEDLEY_WARNING("CRC32 acceleration has been disabled due to missing arm_acle.h"); -# endif -#endif - -#if defined(__ARM_FEATURE_CRC32) || (defined(_M_ARM64) && !defined(__clang__)) // MSVC doesn't support CRC for ARM32 - -/* ARMv8 accelerated CRC */ -#if defined(_MSC_VER) && !defined(__clang__) -#include -#else -#include -#endif - - -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -# ifdef __GNUC__ -# define _LE16 __builtin_bswap16 -# define _LE32 __builtin_bswap32 -# define _LE64 __builtin_bswap64 -# else -// currently not supported -# error No endian swap intrinsic defined -# endif -#else -# define _LE16(x) (x) -# define _LE32(x) (x) -# define _LE64(x) (x) -#endif - -#ifdef __aarch64__ -# define WORD_T uint64_t -# define WORDSIZE_LOG 3 // sizeof(WORD_T) == 1<> 63); - b64 += b64; - a64 >>= 1; - } - // reduction via CRC - res = __crc32w(0, res) ^ (res >> 32); - return res; -} -#endif -// regular multiply is probably better for AArch32 - - -// exploit CPU pipelining during CRC computation; unfortunately I haven't been able to measure any benefit -// - Neoverse N1: no noticeable difference -// - Cortex A53: actually runs a bit slower -//#define ENABLE_PIPELINE_OPT 1 - -#ifdef ENABLE_PIPELINE_OPT -#ifndef __aarch64__ -# define crc32_multiply_arm RapidYenc::crc32_multiply_generic -#endif -#endif - - - -// inspired/stolen off https://github.com/jocover/crc32_armv8/blob/master/crc32_armv8.c -static uint32_t arm_crc_calc(uint32_t crc, const unsigned char *src, long len) { - - // initial alignment - if (len >= 16) { // 16 is an arbitrary number; it just needs to be >=8 - if ((uintptr_t)src & sizeof(uint8_t)) { - crc = __crc32b(crc, *src); - src++; - len--; - } - if ((uintptr_t)src & sizeof(uint16_t)) { - crc = __crc32h(crc, _LE16(*((uint16_t *)src))); - src += sizeof(uint16_t); - len -= sizeof(uint16_t); - } -#ifdef __aarch64__ - if ((uintptr_t)src & sizeof(uint32_t)) { - crc = __crc32w(crc, _LE32(*((uint32_t *)src))); - src += sizeof(uint32_t); - len -= sizeof(uint32_t); - } -#endif - } - - const WORD_T* srcW = (const WORD_T*)src; - -#ifdef ENABLE_PIPELINE_OPT - // uses ideas from https://github.com/komrad36/crc#option-13-golden - // (this is a slightly less efficient, but much simpler implementation of the idea) - const unsigned SPLIT_WORDS_LOG = 10; // make sure it's at least 2 - const unsigned SPLIT_WORDS = 1<= (long)(sizeof(WORD_T)*SPLIT_WORDS*2)) { - // compute 2x CRCs concurrently to leverage piplining - uint32_t crc2 = 0; - for(unsigned i=0; i= 0) { - crc = CRC_WORD(crc, *(srcW++)); - crc = CRC_WORD(crc, *(srcW++)); - crc = CRC_WORD(crc, *(srcW++)); - crc = CRC_WORD(crc, *(srcW++)); - crc = CRC_WORD(crc, *(srcW++)); - crc = CRC_WORD(crc, *(srcW++)); - crc = CRC_WORD(crc, *(srcW++)); - crc = CRC_WORD(crc, *(srcW++)); - } - if (len & sizeof(WORD_T)*4) { - crc = CRC_WORD(crc, *(srcW++)); - crc = CRC_WORD(crc, *(srcW++)); - crc = CRC_WORD(crc, *(srcW++)); - crc = CRC_WORD(crc, *(srcW++)); - } - if (len & sizeof(WORD_T)*2) { - crc = CRC_WORD(crc, *(srcW++)); - crc = CRC_WORD(crc, *(srcW++)); - } - if (len & sizeof(WORD_T)) { - crc = CRC_WORD(crc, *(srcW++)); - } - src = (const unsigned char*)srcW; - -#ifdef __aarch64__ - if (len & sizeof(uint32_t)) { - crc = __crc32w(crc, _LE32(*((uint32_t *)src))); - src += sizeof(uint32_t); - } -#endif - if (len & sizeof(uint16_t)) { - crc = __crc32h(crc, _LE16(*((uint16_t *)src))); - src += sizeof(uint16_t); - } - if (len & sizeof(uint8_t)) - crc = __crc32b(crc, *src); - - return crc; -} - -static uint32_t do_crc32_incremental_arm(const void* data, size_t length, uint32_t init) { - return ~arm_crc_calc(~init, (const unsigned char*)data, (long)length); -} - - -#if defined(__aarch64__) && (defined(__GNUC__) || defined(_MSC_VER)) -static uint32_t crc32_shift_arm(uint32_t crc1, uint32_t n) { - uint32_t result = crc1; - uint64_t prod = result; - prod <<= 32 - (n&31); - result = __crc32w(0, prod) ^ (prod >> 32); - n &= ~31; - - while(n) { - result = crc32_multiply_arm(result, RapidYenc::crc_power[ctz32(n)]); - n &= n-1; - } - return result; -} -#endif - - -void RapidYenc::crc_arm_set_funcs() { - _do_crc32_incremental = &do_crc32_incremental_arm; -#ifdef __aarch64__ - _crc32_multiply = &crc32_multiply_arm; -# if defined(__GNUC__) || defined(_MSC_VER) - _crc32_shift = &crc32_shift_arm; -# endif -#endif - _crc32_isa = ISA_FEATURE_CRC; -} -#else -void RapidYenc::crc_arm_set_funcs() {} -#endif diff --git a/rapidyenc/rapidyenc/src/crc_arm_pmull.cc b/rapidyenc/rapidyenc/src/crc_arm_pmull.cc deleted file mode 100644 index 2691ccf..0000000 --- a/rapidyenc/rapidyenc/src/crc_arm_pmull.cc +++ /dev/null @@ -1,215 +0,0 @@ -#include "crc_common.h" - -// exclude broken/missing arm_acle.h -#if defined(__ARM_FEATURE_CRYPTO) && defined(HEDLEY_GCC_VERSION) -# if !defined(__aarch64__) && HEDLEY_GCC_VERSION_CHECK(7,0,0) && !HEDLEY_GCC_VERSION_CHECK(8,1,1) -# undef __ARM_FEATURE_CRYPTO -# endif -# if defined(__aarch64__) && HEDLEY_GCC_VERSION_CHECK(9,4,0) && !HEDLEY_GCC_VERSION_CHECK(9,5,0) -# undef __ARM_FEATURE_CRYPTO -# endif -#endif -#if defined(__ARM_FEATURE_CRYPTO) && defined(__has_include) -# if !__has_include() -# undef __ARM_FEATURE_CRYPTO -# endif -#endif - -// ARM's intrinsics guide seems to suggest that vmull_p64 is available on A32, but neither Clang/GCC seem to support it on AArch32 -#if (defined(__ARM_FEATURE_CRYPTO) && defined(__ARM_FEATURE_CRC32) && defined(__aarch64__)) || (defined(_M_ARM64) && !defined(__clang__)) - -#include -#if defined(_MSC_VER) && !defined(__clang__) -# include - -# ifdef _M_ARM64 -// MSVC may detect this pattern: https://devblogs.microsoft.com/cppblog/a-tour-of-4-msvc-backend-improvements/#byteswap-identification -static HEDLEY_ALWAYS_INLINE uint64_t rbit64(uint64_t x) { - x = _byteswap_uint64(x); - x = (x & 0xaaaaaaaaaaaaaaaa) >> 1 | (x & 0x5555555555555555) << 1; - x = (x & 0xcccccccccccccccc) >> 2 | (x & 0x3333333333333333) << 2; - x = (x & 0xf0f0f0f0f0f0f0f0) >> 4 | (x & 0x0f0f0f0f0f0f0f0f) << 4; - return x; -} -// ...whilst this seems to work best for 32-bit RBIT -static HEDLEY_ALWAYS_INLINE uint32_t rbit32(uint32_t x) { - uint64_t r = rbit64(x); - return r >> 32; -} -# else -# define rbit32 _arm_rbit -# endif -#else -# include -// __rbit not present before GCC 11.4.0 or 12.2.0; for ARM32, requires GCC 14 -# if defined(HEDLEY_GCC_VERSION) && !HEDLEY_GCC_VERSION_CHECK(14,0,0) && (!defined(__aarch64__) || !HEDLEY_GCC_VERSION_CHECK(11,3,0) || (HEDLEY_GCC_VERSION_CHECK(12,0,0) && !HEDLEY_GCC_VERSION_CHECK(12,2,0))) -# ifdef __aarch64__ -static HEDLEY_ALWAYS_INLINE uint64_t rbit64(uint64_t x) { - uint64_t r; - __asm__ ("rbit %0,%1\n" - : "=r"(r) : "r"(x) - : /* No clobbers */); - return r; -} -# endif -static HEDLEY_ALWAYS_INLINE uint32_t rbit32(uint32_t x) { - uint32_t r; - __asm__ ( -# ifdef __aarch64__ - "rbit %w0,%w1\n" -# else - "rbit %0,%1\n" -# endif - : "=r"(r) : "r"(x) - : /* No clobbers */); - return r; -} -# else -# define rbit32 __rbit -# define rbit64 __rbitll -# endif -#endif - - -// MSVC doesn't have poly64/poly128 types, so always use uint64 instead - -#ifdef __aarch64__ -# if defined(__GNUC__) || defined(__clang__) -static HEDLEY_ALWAYS_INLINE uint64x2_t pmull_low(uint64x1_t a, uint64x1_t b) { - uint64x2_t result; - __asm__ ("pmull %0.1q,%1.1d,%2.1d" - : "=w"(result) - : "w"(a), "w"(b) - : /* No clobbers */); - return result; -} -static HEDLEY_ALWAYS_INLINE uint64x2_t pmull_high(uint64x2_t a, uint64x2_t b) { - uint64x2_t result; - __asm__ ("pmull2 %0.1q,%1.2d,%2.2d" - : "=w"(result) - : "w"(a), "w"(b) - : /* No clobbers */); - return result; -} -# elif defined(_MSC_VER) && !defined(__clang__) -# define pmull_low vmull_p64 -# define pmull_high vmull_high_p64 -# else -# define pmull_low(x, y) vreinterpretq_u64_p128(vmull_p64(vreinterpret_p64_u64(x), vreinterpret_p64_u64(y))) -# define pmull_high(x, y) vreinterpretq_u64_p128(vmull_high_p64(vreinterpretq_p64_u64(x), vreinterpretq_p64_u64(y))) -# endif -#else -# if defined(_MSC_VER) && !defined(__clang__) -# define pmull_low vmull_p64 -# define pmull_high(x, y) vmull_p64(vget_high_u64(x), vget_high_u64(y)) -# else -# define pmull_low(x, y) vreinterpretq_u64_p128(vmull_p64(x, y)) -# define pmull_high(x, y) vreinterpretq_u64_p128(vmull_p64(vget_high_p64(vreinterpretq_p64_u64(x)), vget_high_p64(vreinterpretq_p64_u64(y)))) -# endif -#endif - - -static uint32_t crc32_multiply_pmull(uint32_t a, uint32_t b) { - uint64x1_t prod = vget_low_u64(pmull_low( - vreinterpret_u64_u32(vset_lane_u32(a, vdup_n_u32(0), 0)), - vreinterpret_u64_u32(vset_lane_u32(b, vdup_n_u32(0), 0)) - )); - #ifdef __aarch64__ - uint64_t p = vget_lane_u64(prod, 0); - return __crc32w(0, p+p) ^ (p >> 31); - #else - prod = vadd_u64(prod, prod); - uint32x2_t prod32 = vreinterpret_u32_u64(prod); - return __crc32w(0, vget_lane_u32(prod32, 0)) ^ vget_lane_u32(prod32, 1); - #endif -} - - - -static const uint32_t crc_power_rev[32] = { // bit-reversed crc_power - 0x00000002, 0x00000004, 0x00000010, 0x00000100, 0x00010000, 0x04c11db7, 0x490d678d, 0xe8a45605, - 0x75be46b7, 0xe6228b11, 0x567fddeb, 0x88fe2237, 0x0e857e71, 0x7001e426, 0x075de2b2, 0xf12a7f90, - 0xf0b4a1c1, 0x58f46c0c, 0xc3395ade, 0x96837f8c, 0x544037f9, 0x23b7b136, 0xb2e16ba8, 0x725e7bfa, - 0xec709b5d, 0xf77a7274, 0x2845d572, 0x034e2515, 0x79695942, 0x540cb128, 0x0b65d023, 0x3c344723 -}; - - -static HEDLEY_ALWAYS_INLINE uint64x1_t crc32_shift_pmull_mulred(uint64x1_t a, uint64x1_t b) { - uint64x2_t r = pmull_low(a, b); - uint64x2_t h = pmull_high(r, vdupq_n_u64(0x490d678d)); - return veor_u64(vget_low_u64(r), vget_low_u64(h)); -} - - -static uint32_t crc32_shift_pmull(uint32_t crc1, uint32_t n) { - crc1 = rbit32(crc1); - - uint64x1_t res; - #ifdef __aarch64__ - uint64_t crc = (uint64_t)crc1 << (n & 31); - res = vset_lane_u64(crc, vdup_n_u64(0), 0); - #else - res = vreinterpret_u64_u32(vset_lane_u32(crc1, vdup_n_u32(0), 0)); - res = vshl_u64(res, vdup_n_u64(n&31)); - #endif - n &= ~31; - - if(n) { - #define LOAD_NEXT_POWER vreinterpret_u64_u32(vset_lane_u32(crc_power_rev[ctz32(n)], vdup_n_u32(0), 0)) - uint64x1_t res2 = LOAD_NEXT_POWER; - n &= n-1; - - if(n) { - // first multiply doesn't need reduction - res2 = vget_low_u64(pmull_low(res2, LOAD_NEXT_POWER)); - n &= n-1; - - while(n) { - res = crc32_shift_pmull_mulred(res, LOAD_NEXT_POWER); - n &= n-1; - - if(n) { - res2 = crc32_shift_pmull_mulred(res2, LOAD_NEXT_POWER); - n &= n-1; - } - } - } - #undef LOAD_NEXT_POWER - - // merge two results - uint64x2_t prod = pmull_low(res, res2); - // weirdly, vrbitq_u8 is missing in ARM32 MSVC - prod = vreinterpretq_u64_u8(vrev64q_u8(vrbitq_u8(vreinterpretq_u8_u64(prod)))); - #ifdef __aarch64__ - crc = __crc32d(0, vgetq_lane_u64(prod, 1)); - uint64_t rem = vgetq_lane_u64(prod, 0); - crc = __crc32w(rem, crc) ^ (rem >> 32); - #else - uint32x4_t prod32 = vreinterpretq_u32_u64(prod); - uint32_t crc = __crc32w(0, vgetq_lane_u32(prod32, 2)); - crc = __crc32w(vgetq_lane_u32(prod32, 3), crc); - crc = __crc32w(vgetq_lane_u32(prod32, 0), crc) ^ vgetq_lane_u32(prod32, 1); - #endif - return crc; - } else { - #ifdef __aarch64__ - crc = rbit64(crc); - crc = __crc32w(0, crc) ^ (crc >> 32); - return crc; - #else - uint32x2_t r = vreinterpret_u32_u64(res); - return __crc32w(0, rbit32(vget_lane_u32(r, 1))) ^ rbit32(vget_lane_u32(r, 0)); - #endif - } -} - - -void RapidYenc::crc_pmull_set_funcs() { - _crc32_multiply = &crc32_multiply_pmull; - _crc32_shift = &crc32_shift_pmull; - _crc32_isa |= ISA_FEATURE_PMULL; -} - -#else -void RapidYenc::crc_pmull_set_funcs() {} -#endif /* defined(__ARM_FEATURE_CRYPTO) && defined(__ARM_FEATURE_CRC32) */ diff --git a/rapidyenc/rapidyenc/src/crc_common.h b/rapidyenc/rapidyenc/src/crc_common.h deleted file mode 100644 index e653d37..0000000 --- a/rapidyenc/rapidyenc/src/crc_common.h +++ /dev/null @@ -1,26 +0,0 @@ -#include "common.h" -#include // for size_t -#include "crc.h" - -#ifdef __GNUC__ -# define ctz32 __builtin_ctz -#elif defined(_MSC_VER) -static HEDLEY_ALWAYS_INLINE unsigned ctz32(uint32_t n) { - unsigned long r; - _BitScanForward(&r, n); - return r; -} -#endif - -namespace RapidYenc { - void crc_clmul_set_funcs(); - void crc_clmul256_set_funcs(); - void crc_arm_set_funcs(); - void crc_pmull_set_funcs(); - void crc_riscv_set_funcs(); - - extern const uint32_t crc_power[32]; - uint32_t crc32_multiply_generic(uint32_t a, uint32_t b); - uint32_t crc32_shift_generic(uint32_t crc1, uint32_t n); - -} \ No newline at end of file diff --git a/rapidyenc/rapidyenc/src/crc_folding.cc b/rapidyenc/rapidyenc/src/crc_folding.cc deleted file mode 100644 index 1af8b14..0000000 --- a/rapidyenc/rapidyenc/src/crc_folding.cc +++ /dev/null @@ -1,513 +0,0 @@ -// taken from zlib-ng / Intel's zlib patch, modified to remove zlib dependencies -/* - * Compute the CRC32 using a parallelized folding approach with the PCLMULQDQ - * instruction. - * - * A white paper describing this algorithm can be found at: - * http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf - * - * Copyright (C) 2013 Intel Corporation. All rights reserved. - * Authors: - * Wajdi Feghali - * Jim Guilford - * Vinodh Gopal - * Erdinc Ozturk - * Jim Kukunas - * - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -#include "crc_common.h" - -#if (defined(__PCLMUL__) && defined(__SSSE3__) && defined(__SSE4_1__)) || (defined(_MSC_VER) && _MSC_VER >= 1600 && defined(PLATFORM_X86) && !defined(__clang__)) -#include -#include -#include - - -#if defined(__AVX512VL__) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0 -# define ENABLE_AVX512 1 -#endif - - -// interestingly, MSVC seems to generate better code if using VXORPS over VPXOR -// original Intel code uses XORPS for many XOR operations, but PXOR is pretty much always better (more port freedom on Intel CPUs). The only advantage of XORPS is that it's 1 byte shorter, an advantage which disappears under AVX as both instructions have the same length -#if defined(__AVX__) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0 -# define fold_xor _mm_xor_si128 -#else -static __m128i fold_xor(__m128i a, __m128i b) { - return _mm_castps_si128(_mm_xor_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))); -} -#endif - -#ifdef ENABLE_AVX512 -static __m128i do_one_fold_merge(__m128i src, __m128i data) { - const __m128i xmm_fold4 = _mm_set_epi32( - 0x00000001, 0x54442bd4, - 0x00000001, 0xc6e41596); - return _mm_ternarylogic_epi32( - _mm_clmulepi64_si128(src, xmm_fold4, 0x01), - _mm_clmulepi64_si128(src, xmm_fold4, 0x10), - data, - 0x96 - ); -} -#else -static __m128i do_one_fold(__m128i src) { - const __m128i xmm_fold4 = _mm_set_epi32( - 0x00000001, 0x54442bd4, - 0x00000001, 0xc6e41596); - return fold_xor( - _mm_clmulepi64_si128(src, xmm_fold4, 0x01), - _mm_clmulepi64_si128(src, xmm_fold4, 0x10) - ); -} -#endif - -ALIGN_TO(32, static const unsigned pshufb_shf_table[60]) = { - 0x84838281, 0x88878685, 0x8c8b8a89, 0x008f8e8d, /* shl 15 (16 - 1)/shr1 */ - 0x85848382, 0x89888786, 0x8d8c8b8a, 0x01008f8e, /* shl 14 (16 - 3)/shr2 */ - 0x86858483, 0x8a898887, 0x8e8d8c8b, 0x0201008f, /* shl 13 (16 - 4)/shr3 */ - 0x87868584, 0x8b8a8988, 0x8f8e8d8c, 0x03020100, /* shl 12 (16 - 4)/shr4 */ - 0x88878685, 0x8c8b8a89, 0x008f8e8d, 0x04030201, /* shl 11 (16 - 5)/shr5 */ - 0x89888786, 0x8d8c8b8a, 0x01008f8e, 0x05040302, /* shl 10 (16 - 6)/shr6 */ - 0x8a898887, 0x8e8d8c8b, 0x0201008f, 0x06050403, /* shl 9 (16 - 7)/shr7 */ - 0x8b8a8988, 0x8f8e8d8c, 0x03020100, 0x07060504, /* shl 8 (16 - 8)/shr8 */ - 0x8c8b8a89, 0x008f8e8d, 0x04030201, 0x08070605, /* shl 7 (16 - 9)/shr9 */ - 0x8d8c8b8a, 0x01008f8e, 0x05040302, 0x09080706, /* shl 6 (16 -10)/shr10*/ - 0x8e8d8c8b, 0x0201008f, 0x06050403, 0x0a090807, /* shl 5 (16 -11)/shr11*/ - 0x8f8e8d8c, 0x03020100, 0x07060504, 0x0b0a0908, /* shl 4 (16 -12)/shr12*/ - 0x008f8e8d, 0x04030201, 0x08070605, 0x0c0b0a09, /* shl 3 (16 -13)/shr13*/ - 0x01008f8e, 0x05040302, 0x09080706, 0x0d0c0b0a, /* shl 2 (16 -14)/shr14*/ - 0x0201008f, 0x06050403, 0x0a090807, 0x0e0d0c0b /* shl 1 (16 -15)/shr15*/ -}; - -static void partial_fold(const size_t len, __m128i *xmm_crc0, __m128i *xmm_crc1, - __m128i *xmm_crc2, __m128i *xmm_crc3, __m128i *xmm_crc_part) { - - const __m128i xmm_mask3 = _mm_set1_epi32(0x80808080); - - __m128i xmm_shl, xmm_shr, xmm_tmp1, xmm_tmp2, xmm_tmp3; - __m128i xmm_a0_0; - - xmm_shl = _mm_load_si128((__m128i *)pshufb_shf_table + (len - 1)); - xmm_shr = xmm_shl; - xmm_shr = _mm_xor_si128(xmm_shr, xmm_mask3); - - xmm_a0_0 = _mm_shuffle_epi8(*xmm_crc0, xmm_shl); - - *xmm_crc0 = _mm_shuffle_epi8(*xmm_crc0, xmm_shr); - xmm_tmp1 = _mm_shuffle_epi8(*xmm_crc1, xmm_shl); - *xmm_crc0 = _mm_or_si128(*xmm_crc0, xmm_tmp1); - - *xmm_crc1 = _mm_shuffle_epi8(*xmm_crc1, xmm_shr); - xmm_tmp2 = _mm_shuffle_epi8(*xmm_crc2, xmm_shl); - *xmm_crc1 = _mm_or_si128(*xmm_crc1, xmm_tmp2); - - *xmm_crc2 = _mm_shuffle_epi8(*xmm_crc2, xmm_shr); - xmm_tmp3 = _mm_shuffle_epi8(*xmm_crc3, xmm_shl); - *xmm_crc2 = _mm_or_si128(*xmm_crc2, xmm_tmp3); - - *xmm_crc3 = _mm_shuffle_epi8(*xmm_crc3, xmm_shr); - *xmm_crc_part = _mm_shuffle_epi8(*xmm_crc_part, xmm_shl); - *xmm_crc3 = _mm_or_si128(*xmm_crc3, *xmm_crc_part); - -#ifdef ENABLE_AVX512 - *xmm_crc3 = do_one_fold_merge(xmm_a0_0, *xmm_crc3); -#else - *xmm_crc3 = fold_xor( - do_one_fold(xmm_a0_0), - *xmm_crc3 - ); -#endif -} - -ALIGN_TO(16, static const unsigned crc_k[]) = { - 0xccaa009e, 0x00000000, /* rk1 */ - 0x751997d0, 0x00000001, /* rk2 */ - 0xccaa009e, 0x00000000, /* rk5 */ - 0x63cd6124, 0x00000001, /* rk6 */ - 0xf7011641, 0x00000000, /* rk7 */ - 0xdb710640, 0x00000001 /* rk8 */ -}; - -ALIGN_TO(16, static const unsigned crc_mask[4]) = { - 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF -}; - - -static uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) { - unsigned long algn_diff; - __m128i xmm_t0, xmm_t1, xmm_t2, xmm_t3; - - // since the initial value will immediately be multiplied by around 2^512, we need to roll it backwards - // this is done by dividing the initial value by 2^480 - // the constant used here is reverse(2^-480)<<1 == 0xdfded7ec - __m128i xmm_crc0 = _mm_clmulepi64_si128(_mm_cvtsi32_si128(~initial), _mm_cvtsi32_si128(0xdfded7ec), 0); - - __m128i xmm_crc1 = _mm_setzero_si128(); - __m128i xmm_crc2 = _mm_setzero_si128(); - __m128i xmm_crc3 = _mm_setzero_si128(); - __m128i xmm_crc_part; - - if (len < 16) { - if (len == 0) - return initial; - xmm_crc_part = _mm_setzero_si128(); - memcpy(&xmm_crc_part, src, len); - goto partial; - } - - algn_diff = (0 - (uintptr_t)src) & 0xF; - if (algn_diff) { - xmm_crc_part = _mm_loadu_si128((__m128i *)src); - - src += algn_diff; - len -= algn_diff; - - partial_fold(algn_diff, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, - &xmm_crc_part); - } - - while (len >= 64) { - xmm_t0 = _mm_load_si128((__m128i *)src); - xmm_t1 = _mm_load_si128((__m128i *)src + 1); - xmm_t2 = _mm_load_si128((__m128i *)src + 2); - xmm_t3 = _mm_load_si128((__m128i *)src + 3); - -#ifdef ENABLE_AVX512 - xmm_crc0 = do_one_fold_merge(xmm_crc0, xmm_t0); - xmm_crc1 = do_one_fold_merge(xmm_crc1, xmm_t1); - xmm_crc2 = do_one_fold_merge(xmm_crc2, xmm_t2); - xmm_crc3 = do_one_fold_merge(xmm_crc3, xmm_t3); -#else - // nesting do_one_fold() in _mm_xor_si128() seems to cause MSVC to generate horrible code, so separate it out - xmm_crc0 = do_one_fold(xmm_crc0); - xmm_crc1 = do_one_fold(xmm_crc1); - xmm_crc2 = do_one_fold(xmm_crc2); - xmm_crc3 = do_one_fold(xmm_crc3); - xmm_crc0 = _mm_xor_si128(xmm_crc0, xmm_t0); - xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t1); - xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t2); - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t3); -#endif - - src += 64; - len -= 64; - } - - if (len >= 48) { - len -= 48; - - xmm_t0 = _mm_load_si128((__m128i *)src); - xmm_t1 = _mm_load_si128((__m128i *)src + 1); - xmm_t2 = _mm_load_si128((__m128i *)src + 2); - - xmm_t3 = xmm_crc3; -#ifdef ENABLE_AVX512 - xmm_crc3 = do_one_fold_merge(xmm_crc2, xmm_t2); - xmm_crc2 = do_one_fold_merge(xmm_crc1, xmm_t1); - xmm_crc1 = do_one_fold_merge(xmm_crc0, xmm_t0); -#else - xmm_crc3 = do_one_fold(xmm_crc2); - xmm_crc2 = do_one_fold(xmm_crc1); - xmm_crc1 = do_one_fold(xmm_crc0); - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t2); - xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t1); - xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t0); -#endif - xmm_crc0 = xmm_t3; - - if (len == 0) - goto done; - - xmm_crc_part = _mm_load_si128((__m128i *)src + 3); - } else if (len >= 32) { - len -= 32; - - xmm_t0 = _mm_load_si128((__m128i *)src); - xmm_t1 = _mm_load_si128((__m128i *)src + 1); - - xmm_t2 = xmm_crc2; - xmm_t3 = xmm_crc3; -#ifdef ENABLE_AVX512 - xmm_crc3 = do_one_fold_merge(xmm_crc1, xmm_t1); - xmm_crc2 = do_one_fold_merge(xmm_crc0, xmm_t0); -#else - xmm_crc3 = do_one_fold(xmm_crc1); - xmm_crc2 = do_one_fold(xmm_crc0); - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t1); - xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t0); -#endif - xmm_crc1 = xmm_t3; - xmm_crc0 = xmm_t2; - - if (len == 0) - goto done; - - xmm_crc_part = _mm_load_si128((__m128i *)src + 2); - } else if (len >= 16) { - len -= 16; - - xmm_t0 = _mm_load_si128((__m128i *)src); - - xmm_t3 = xmm_crc3; -#ifdef ENABLE_AVX512 - xmm_crc3 = do_one_fold_merge(xmm_crc0, xmm_t0); -#else - xmm_crc3 = _mm_xor_si128(do_one_fold(xmm_crc0), xmm_t0); -#endif - xmm_crc0 = xmm_crc1; - xmm_crc1 = xmm_crc2; - xmm_crc2 = xmm_t3; - - if (len == 0) - goto done; - - xmm_crc_part = _mm_load_si128((__m128i *)src + 1); - } else { - if (len == 0) - goto done; - xmm_crc_part = _mm_load_si128((__m128i *)src); - } - -partial: - partial_fold(len, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, - &xmm_crc_part); -done: -{ - const __m128i xmm_mask = _mm_load_si128((__m128i *)crc_mask); - __m128i x_tmp0, x_tmp1, x_tmp2, crc_fold; - - /* - * k1 - */ - crc_fold = _mm_load_si128((__m128i *)crc_k); - - x_tmp0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x10); - xmm_crc0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x01); -#ifdef ENABLE_AVX512 - xmm_crc1 = _mm_ternarylogic_epi32(xmm_crc1, x_tmp0, xmm_crc0, 0x96); -#else - xmm_crc1 = _mm_xor_si128(xmm_crc1, x_tmp0); - xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_crc0); -#endif - - x_tmp1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x10); - xmm_crc1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x01); -#ifdef ENABLE_AVX512 - xmm_crc2 = _mm_ternarylogic_epi32(xmm_crc2, x_tmp1, xmm_crc1, 0x96); -#else - xmm_crc2 = _mm_xor_si128(xmm_crc2, x_tmp1); - xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_crc1); -#endif - - x_tmp2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x10); - xmm_crc2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x01); -#ifdef ENABLE_AVX512 - xmm_crc3 = _mm_ternarylogic_epi32(xmm_crc3, x_tmp2, xmm_crc2, 0x96); -#else - xmm_crc3 = _mm_xor_si128(xmm_crc3, x_tmp2); - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); -#endif - - /* - * k5 - */ - crc_fold = _mm_load_si128((__m128i *)crc_k + 1); - - xmm_crc0 = xmm_crc3; - xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0); - xmm_crc0 = _mm_srli_si128(xmm_crc0, 8); - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0); - - xmm_crc0 = xmm_crc3; - xmm_crc3 = _mm_slli_si128(xmm_crc3, 4); - xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10); -#ifdef ENABLE_AVX512 - //xmm_crc3 = _mm_maskz_xor_epi32(14, xmm_crc3, xmm_crc0); - xmm_crc3 = _mm_ternarylogic_epi32(xmm_crc3, xmm_crc0, xmm_mask, 0x28); -#else - xmm_crc0 = _mm_and_si128(xmm_crc0, xmm_mask); - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0); -#endif - - /* - * k7 - */ - xmm_crc1 = xmm_crc3; - crc_fold = _mm_load_si128((__m128i *)crc_k + 2); - - xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0); - xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10); -#ifdef ENABLE_AVX512 - xmm_crc3 = _mm_ternarylogic_epi32(xmm_crc3, xmm_crc1, xmm_crc1, 0xC3); // NOT(xmm_crc3 ^ xmm_crc1) -#else - xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_mask); - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc1); -#endif - return _mm_extract_epi32(xmm_crc3, 2); -} - -} - -static uint32_t do_crc32_incremental_clmul(const void* data, size_t length, uint32_t init) { - return crc_fold((const unsigned char*)data, (long)length, init); -} - - -static HEDLEY_ALWAYS_INLINE __m128i crc32_reduce(__m128i prod) { - // do Barrett reduction back into 32-bit field - const __m128i reduction_const = _mm_load_si128((__m128i*)crc_k + 2); - __m128i t = _mm_clmulepi64_si128(prod, reduction_const, 0); - t = _mm_clmulepi64_si128(t, reduction_const, 0x10); - t = _mm_xor_si128(t, prod); - return t; -} - -static uint32_t crc32_multiply_clmul(uint32_t a, uint32_t b) { - // do the actual multiply - __m128i prod = _mm_clmulepi64_si128(_mm_cvtsi32_si128(a), _mm_cvtsi32_si128(b), 0); - - // prepare product for reduction - prod = _mm_add_epi64(prod, prod); // bit alignment fix, due to CRC32 being bit-reversal - prod = _mm_slli_si128(prod, 4); // straddle low/high halves across 64-bit boundary - this provides automatic truncation during reduction - - prod = crc32_reduce(prod); - return _mm_extract_epi32(prod, 2); -} - -#if defined(__GNUC__) || defined(_MSC_VER) -static HEDLEY_ALWAYS_INLINE __m128i reverse_bits_epi8(__m128i src) { -#if defined(__GFNI__) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0 - return _mm_gf2p8affine_epi64_epi8(src, _mm_set_epi32( - 0x80402010, 0x08040201, - 0x80402010, 0x08040201 - ), 0); -/* -#elif defined(ENABLE_AVX512) - // !! this only processes the bottom 32 bits !! - src = _mm_maskz_mov_epi32(1, src); - src = _mm_ternarylogic_epi32(src, _mm_slli_epi64(src, 28), _mm_set1_epi8(0xf), 0xa8); // (a|b)&c - src = _mm_shuffle_epi8(_mm_set_epi8( - -16, 112, -80, 48, -48, 80, -112, 16, -32, 96, -96, 32, -64, 64, -128, 0 - ), src); - return _mm_maskz_or_epi32(1, src, _mm_srli_epi64(src, 36)); -*/ -#else - __m128i xmm_t0 = _mm_and_si128(src, _mm_set1_epi8(0x0f)); - __m128i xmm_t1 = _mm_and_si128(_mm_srli_epi16(src, 4), _mm_set1_epi8(0x0f)); - xmm_t0 = _mm_shuffle_epi8(_mm_set_epi8( - -16, 112, -80, 48, -48, 80, -112, 16, -32, 96, -96, 32, -64, 64, -128, 0 - //0xf0, 0x70, 0xb0, 0x30, 0xd0, 0x50, 0x90, 0x10, 0xe0, 0x60, 0xa0, 0x20, 0xc0, 0x40, 0x80, 0 - ), xmm_t0); - xmm_t1 = _mm_shuffle_epi8(_mm_set_epi8( - 15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0 - ), xmm_t1); - return _mm_or_si128(xmm_t0, xmm_t1); -#endif -} - -#ifdef _MSC_VER -// because MSVC doesn't use BSWAP unless you specifically tell it to... -# include -# define BSWAP32 _byteswap_ulong -#else -# define BSWAP32(n) ((((n)&0xff)<<24) | (((n)&0xff00)<<8) | (((n)&0xff0000)>>8) | (((n)&0xff000000)>>24)) -#endif - - - -static const uint32_t crc_power_rev[32] = { // bit-reversed crc_power - 0x00000002, 0x00000004, 0x00000010, 0x00000100, 0x00010000, 0x04c11db7, 0x490d678d, 0xe8a45605, - 0x75be46b7, 0xe6228b11, 0x567fddeb, 0x88fe2237, 0x0e857e71, 0x7001e426, 0x075de2b2, 0xf12a7f90, - 0xf0b4a1c1, 0x58f46c0c, 0xc3395ade, 0x96837f8c, 0x544037f9, 0x23b7b136, 0xb2e16ba8, 0x725e7bfa, - 0xec709b5d, 0xf77a7274, 0x2845d572, 0x034e2515, 0x79695942, 0x540cb128, 0x0b65d023, 0x3c344723 -}; - -static HEDLEY_ALWAYS_INLINE __m128i crc32_shift_clmul_mulred(unsigned pos, __m128i prod) { - // this multiplies a 64-bit `prod` with a 32-bit CRC power - // compared with crc32_multiply_clmul, this only reduces the result to 64-bit, saving a multiply - __m128i coeff = _mm_cvtsi32_si128(crc_power_rev[pos]); - - const __m128i fold_const = _mm_set_epi32(0, 0x490d678d, 0, 0xf200aa66); - prod = _mm_clmulepi64_si128(prod, coeff, 0); - __m128i hi = _mm_clmulepi64_si128(prod, fold_const, 0x11); - return _mm_xor_si128(hi, prod); -} - -static uint32_t crc32_shift_clmul(uint32_t crc1, uint32_t n) { - if(!n) return crc1; - - __m128i result = _mm_cvtsi32_si128(BSWAP32(crc1)); - result = reverse_bits_epi8(result); - - // handle n < 32 with a shift - result = _mm_sll_epi64(result, _mm_cvtsi32_si128(n & 31)); - n &= ~31; - - __m128i t; - if(n) { - // use a second accumulator to leverage some IPC from slow CLMUL - __m128i result2 = _mm_cvtsi32_si128(crc_power_rev[ctz32(n)]); - n &= n-1; - - if(n) { - // first multiply doesn't need reduction - result2 = _mm_clmulepi64_si128(result2, _mm_cvtsi32_si128(crc_power_rev[ctz32(n)]), 0); - n &= n-1; - - while(n) { - result = crc32_shift_clmul_mulred(ctz32(n), result); - n &= n-1; - - if(n) { - result2 = crc32_shift_clmul_mulred(ctz32(n), result2); - n &= n-1; - } - } - } - - const __m128i fold_const = _mm_set_epi32(0, 0x490d678d, 0, 0xf200aa66); - - // merge two results - result = _mm_clmulepi64_si128(result, result2, 0); - - // do 128b reduction - t = _mm_unpackhi_epi32(result, _mm_setzero_si128()); - // fold [127:96] -> [63:0] - __m128i hi = _mm_clmulepi64_si128(t, fold_const, 1); - // fold [95:64] -> [63:0] - __m128i lo = _mm_clmulepi64_si128(t, fold_const, 0x10); -#ifdef ENABLE_AVX512 - result = _mm_ternarylogic_epi32(result, hi, lo, 0x96); -#else - result = _mm_xor_si128(result, hi); - result = _mm_xor_si128(result, lo); -#endif - } - - // do Barrett reduction back into 32-bit field - const __m128i reduction_const = _mm_set_epi32(0, 0x04c11db7, 1, 0x04d101df); - t = _mm_clmulepi64_si128(_mm_blend_epi16(_mm_setzero_si128(), result, 0x3c), reduction_const, 0); - t = _mm_clmulepi64_si128(t, reduction_const, 0x11); - result = _mm_xor_si128(t, result); - - result = reverse_bits_epi8(result); - return BSWAP32(_mm_cvtsi128_si32(result)); -} -#endif - - -void RapidYenc::crc_clmul_set_funcs() { - _do_crc32_incremental = &do_crc32_incremental_clmul; - _crc32_multiply = &crc32_multiply_clmul; -#if defined(__GNUC__) || defined(_MSC_VER) - _crc32_shift = &crc32_shift_clmul; -#endif - _crc32_isa = ISA_LEVEL_PCLMUL; -} -#else -void RapidYenc::crc_clmul_set_funcs() {} -#endif - diff --git a/rapidyenc/rapidyenc/src/crc_folding_256.cc b/rapidyenc/rapidyenc/src/crc_folding_256.cc deleted file mode 100644 index 28b0757..0000000 --- a/rapidyenc/rapidyenc/src/crc_folding_256.cc +++ /dev/null @@ -1,223 +0,0 @@ -// 256-bit version of crc_folding - -#include "crc_common.h" - -#if !defined(YENC_DISABLE_AVX256) && ((defined(__VPCLMULQDQ__) && defined(__AVX2__) && defined(__PCLMUL__)) || (defined(_MSC_VER) && _MSC_VER >= 1920 && defined(PLATFORM_X86) && !defined(__clang__))) -#include -#include - - -#if defined(__AVX512VL__) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0 -# define ENABLE_AVX512 1 -#endif - -static __m256i do_one_fold(__m256i src, __m256i data) { - const __m256i fold4 = _mm256_set_epi32( - 0x00000001, 0x54442bd4, - 0x00000001, 0xc6e41596, - 0x00000001, 0x54442bd4, - 0x00000001, 0xc6e41596 - ); -#ifdef ENABLE_AVX512 - return _mm256_ternarylogic_epi32( - _mm256_clmulepi64_epi128(src, fold4, 0x01), - _mm256_clmulepi64_epi128(src, fold4, 0x10), - data, - 0x96 - ); -#else - return _mm256_xor_si256(_mm256_xor_si256( - data, _mm256_clmulepi64_epi128(src, fold4, 0x01) - ), _mm256_clmulepi64_epi128(src, fold4, 0x10)); -#endif -} - -ALIGN_TO(32, static const uint8_t pshufb_rot_table[]) = { - 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, - 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 -}; -// _mm256_castsi128_si256, but upper is defined to be 0 -#if (defined(__clang__) && __clang_major__ >= 5 && (!defined(__APPLE__) || __clang_major__ >= 7)) || (defined(__GNUC__) && __GNUC__ >= 10) || (defined(_MSC_VER) && _MSC_VER >= 1910) -// intrinsic unsupported in GCC 9 and MSVC < 2017 -# define zext128_256 _mm256_zextsi128_si256 -#else -// technically a cast is incorrect, due to upper 128 bits being undefined, but should usually work fine -// alternative may be `_mm256_set_m128i(_mm_setzero_si128(), v)` but unsupported on GCC < 7, and most compilers generate a VINSERTF128 instruction for it -# ifdef __OPTIMIZE__ -# define zext128_256 _mm256_castsi128_si256 -# else -# define zext128_256(x) _mm256_inserti128_si256(_mm256_setzero_si256(), x, 0) -# endif -#endif - -#ifdef ENABLE_AVX512 -# define MM256_BLENDV(a, b, m) _mm256_ternarylogic_epi32(a, b, m, 0xd8) -# define MM_2XOR(a, b, c) _mm_ternarylogic_epi32(a, b, c, 0x96) -#else -# define MM256_BLENDV _mm256_blendv_epi8 -# define MM_2XOR(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) -#endif - -static void partial_fold(const size_t len, __m256i *crc0, __m256i *crc1, __m256i crc_part) { - __m256i shuf = _mm256_broadcastsi128_si256(_mm_loadu_si128((__m128i*)(pshufb_rot_table + (len&15)))); - __m256i mask = _mm256_cmpgt_epi8(shuf, _mm256_set1_epi8(15)); - - *crc0 = _mm256_shuffle_epi8(*crc0, shuf); - *crc1 = _mm256_shuffle_epi8(*crc1, shuf); - crc_part = _mm256_shuffle_epi8(crc_part, shuf); - - __m256i crc_out = _mm256_permute2x128_si256(*crc0, *crc0, 0x08); // move bottom->top - __m256i crc01, crc1p; - if(len >= 16) { - crc_out = MM256_BLENDV(crc_out, *crc0, mask); - crc01 = *crc1; - crc1p = crc_part; - *crc0 = _mm256_permute2x128_si256(*crc0, *crc1, 0x21); - *crc1 = _mm256_permute2x128_si256(*crc1, crc_part, 0x21); - crc_part = zext128_256(_mm256_extracti128_si256(crc_part, 1)); - } else { - crc_out = _mm256_and_si256(crc_out, mask); - crc01 = _mm256_permute2x128_si256(*crc0, *crc1, 0x21); - crc1p = _mm256_permute2x128_si256(*crc1, crc_part, 0x21); - } - - *crc0 = MM256_BLENDV(*crc0, crc01, mask); - *crc1 = MM256_BLENDV(*crc1, crc1p, mask); - - *crc1 = do_one_fold(crc_out, *crc1); -} - - -ALIGN_TO(16, static const unsigned crc_k[]) = { - 0xccaa009e, 0x00000000, /* rk1 */ - 0x751997d0, 0x00000001, /* rk2 */ - 0xccaa009e, 0x00000000, /* rk5 */ - 0x63cd6124, 0x00000001, /* rk6 */ - 0xf7011641, 0x00000000, /* rk7 */ - 0xdb710640, 0x00000001 /* rk8 */ -}; - - -static uint32_t crc_fold(const unsigned char *src, long len, uint32_t initial) { - __m128i xmm_t0 = _mm_clmulepi64_si128( - _mm_cvtsi32_si128(~initial), - _mm_cvtsi32_si128(0xdfded7ec), - 0 - ); - - __m256i crc0 = zext128_256(xmm_t0); - __m256i crc1 = _mm256_setzero_si256(); - - if (len < 32) { - if (len == 0) - return initial; - __m256i crc_part = _mm256_setzero_si256(); - memcpy(&crc_part, src, len); - partial_fold(len, &crc0, &crc1, crc_part); - } else { - uintptr_t algn_diff = (0 - (uintptr_t)src) & 0x1F; - if (algn_diff) { - partial_fold(algn_diff, &crc0, &crc1, _mm256_loadu_si256((__m256i *)src)); - src += algn_diff; - len -= algn_diff; - } - - while (len >= 64) { - crc0 = do_one_fold(crc0, _mm256_load_si256((__m256i*)src)); - crc1 = do_one_fold(crc1, _mm256_load_si256((__m256i*)src + 1)); - src += 64; - len -= 64; - } - - if (len >= 32) { - __m256i old = crc1; - crc1 = do_one_fold(crc0, _mm256_load_si256((__m256i*)src)); - crc0 = old; - - len -= 32; - src += 32; - } - - if(len != 0) { - partial_fold(len, &crc0, &crc1, _mm256_load_si256((__m256i *)src)); - } - } - - const __m128i xmm_mask = _mm_set_epi32(-1,-1,-1,0); - __m128i x_tmp0, x_tmp1, x_tmp2, crc_fold; - - __m128i xmm_crc0 = _mm256_castsi256_si128(crc0); - __m128i xmm_crc1 = _mm256_extracti128_si256(crc0, 1); - __m128i xmm_crc2 = _mm256_castsi256_si128(crc1); - __m128i xmm_crc3 = _mm256_extracti128_si256(crc1, 1); - - /* - * k1 - */ - crc_fold = _mm_load_si128((__m128i *)crc_k); - - x_tmp0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x10); - xmm_crc0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x01); - xmm_crc1 = MM_2XOR(xmm_crc1, x_tmp0, xmm_crc0); - - x_tmp1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x10); - xmm_crc1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x01); - xmm_crc2 = MM_2XOR(xmm_crc2, x_tmp1, xmm_crc1); - - x_tmp2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x10); - xmm_crc2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x01); - xmm_crc3 = MM_2XOR(xmm_crc3, x_tmp2, xmm_crc2); - - /* - * k5 - */ - crc_fold = _mm_load_si128((__m128i *)crc_k + 1); - - xmm_crc0 = xmm_crc3; - xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0); - xmm_crc0 = _mm_srli_si128(xmm_crc0, 8); - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0); - - xmm_crc0 = xmm_crc3; - xmm_crc3 = _mm_slli_si128(xmm_crc3, 4); - xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10); -#ifdef ENABLE_AVX512 - //xmm_crc3 = _mm_maskz_xor_epi32(14, xmm_crc3, xmm_crc0); - xmm_crc3 = _mm_ternarylogic_epi32(xmm_crc3, xmm_crc0, xmm_mask, 0x28); -#else - xmm_crc0 = _mm_and_si128(xmm_crc0, xmm_mask); - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0); -#endif - - /* - * k7 - */ - xmm_crc1 = xmm_crc3; - crc_fold = _mm_load_si128((__m128i *)crc_k + 2); - - xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0); - xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10); -#ifdef ENABLE_AVX512 - xmm_crc3 = _mm_ternarylogic_epi32(xmm_crc3, xmm_crc1, xmm_crc1, 0xC3); // NOT(xmm_crc3 ^ xmm_crc1) -#else - xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_mask); - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc1); -#endif - return _mm_extract_epi32(xmm_crc3, 2); -} - -static uint32_t do_crc32_incremental_clmul(const void* data, size_t length, uint32_t init) { - return crc_fold((const unsigned char*)data, (long)length, init); -} - -void RapidYenc::crc_clmul256_set_funcs() { - crc_clmul_set_funcs(); // set multiply/shift function - _do_crc32_incremental = &do_crc32_incremental_clmul; - _crc32_isa = ISA_LEVEL_VPCLMUL; -} -#else -void RapidYenc::crc_clmul256_set_funcs() { - crc_clmul_set_funcs(); -} -#endif - diff --git a/rapidyenc/rapidyenc/src/crc_riscv.cc b/rapidyenc/rapidyenc/src/crc_riscv.cc deleted file mode 100644 index 34ad2bf..0000000 --- a/rapidyenc/rapidyenc/src/crc_riscv.cc +++ /dev/null @@ -1,251 +0,0 @@ -#include "crc_common.h" - -#if defined(__riscv) && defined(__GNUC__) && (defined(__riscv_zbkc) || defined(__riscv_zbc)) - -#if __has_include() -# include -# if __riscv_xlen == 64 -# define rv_clmul __riscv_clmul_64 -# define rv_clmulh __riscv_clmulh_64 -# else -# define rv_clmul __riscv_clmul_32 -# define rv_clmulh __riscv_clmulh_32 -# endif -#else -static HEDLEY_ALWAYS_INLINE uintptr_t rv_clmul(uintptr_t x, uintptr_t y) { - uintptr_t r; - __asm__("clmul %0, %1, %2\n" - : "=r"(r) - : "r"(x), "r"(y) - :); - return r; -} -static HEDLEY_ALWAYS_INLINE uintptr_t rv_clmulh(uintptr_t x, uintptr_t y) { - uintptr_t r; - __asm__("clmulh %0, %1, %2\n" - : "=r"(r) - : "r"(x), "r"(y) - :); - return r; -} -#endif - -// TODO: test big-endian -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -# if __riscv_xlen == 64 -# define SWAP __builtin_bswap64 -# else -# define SWAP __builtin_bswap32 -# endif -#else -# define SWAP(d) (d) -#endif -static HEDLEY_ALWAYS_INLINE uintptr_t read_partial(const void* p, unsigned sz) { - uintptr_t data = 0; - memcpy(&data, p, sz); - return SWAP(data); -} -static HEDLEY_ALWAYS_INLINE uintptr_t read_full(const uintptr_t* p) { - return SWAP(*p); -} -#undef SWAP - -static uint32_t rv_crc_calc(uint32_t crc, const unsigned char *src, long len) { - uintptr_t accum[4] = {}; - - // note: constants here are bit-reflected and shifted left by 1 - // Zbc does also have clmulr to avoid the shift, but: - // - there's no clmulhr, so for XLEN=64, just shift the constant instead to get the same result - // - it's unavailable in Zbkc - // - for XLEN=32, 2x constants is likely worth it to avoid the additional XORs in the loop - -#if __riscv_xlen == 64 - const uint64_t MUL_HI = 0x15a546366 /*2^224*/, MUL_LO = 0xf1da05aa /*2^288*/; - #define CLMULL rv_clmul - #define CLMULH rv_clmulh - - accum[3] = rv_clmul(crc, 0xb66b1fa6); // 2^-32 -#elif __riscv_xlen == 32 - const uint64_t MUL_HI = 0x140d44a2e /*2^128*/, MUL_LO = 0x1751997d0 /*2^160*/; - #define CLMULL(x, k) rv_clmul(x, k & 0xffffffff) - #define CLMULH(x, k) (rv_clmulh(x, k & 0xffffffff) ^ (k > 0xffffffffULL ? (x) : 0)) - - accum[2] = rv_clmul(crc, 0xb66b1fa6); - accum[3] = rv_clmulh(crc, 0xb66b1fa6); -#else - #error "Unknown __riscv_xlen" -#endif - const size_t WS = sizeof(uintptr_t); - - // if src isn't word-aligned, process until it is so - long initial_alignment = ((uintptr_t)src & (WS-1)); - long initial_process = WS - initial_alignment; - if(initial_alignment && len >= initial_process) { - unsigned shl = initial_alignment * 8, shr = initial_process * 8; -#if __riscv_xlen == 64 - accum[2] = accum[3] << shl; -#else - accum[1] = accum[2] << shl; - accum[2] = (accum[3] << shl) | (accum[2] >> shr); -#endif - accum[3] = (read_partial(src, initial_process) << shl) | (accum[3] >> shr); - src += initial_process; - len -= initial_process; - } - - // main processing loop - const uintptr_t* srcW = (const uintptr_t*)src; - while((len -= WS*4) >= 0) { - uintptr_t tmpHi, tmpLo; - tmpLo = CLMULL(accum[0], MUL_LO) ^ CLMULL(accum[1], MUL_HI); - tmpHi = CLMULH(accum[0], MUL_LO) ^ CLMULH(accum[1], MUL_HI); - accum[0] = tmpLo ^ read_full(srcW++); - accum[1] = tmpHi ^ read_full(srcW++); - - tmpLo = CLMULL(accum[2], MUL_LO) ^ CLMULL(accum[3], MUL_HI); - tmpHi = CLMULH(accum[2], MUL_LO) ^ CLMULH(accum[3], MUL_HI); - accum[2] = tmpLo ^ read_full(srcW++); - accum[3] = tmpHi ^ read_full(srcW++); - } - - // process trailing bytes - if(len & (WS*2)) { - uintptr_t tmpLo = CLMULL(accum[0], MUL_LO) ^ CLMULL(accum[1], MUL_HI); - uintptr_t tmpHi = CLMULH(accum[0], MUL_LO) ^ CLMULH(accum[1], MUL_HI); - accum[0] = accum[2]; - accum[1] = accum[3]; - accum[2] = tmpLo ^ read_full(srcW++); - accum[3] = tmpHi ^ read_full(srcW++); - } - if(len & WS) { - uintptr_t tmpLo = CLMULL(accum[0], MUL_HI); - uintptr_t tmpHi = CLMULH(accum[0], MUL_HI); - accum[0] = accum[1]; - accum[1] = accum[2]; - accum[2] = accum[3] ^ tmpLo; - accum[3] = tmpHi ^ read_full(srcW++); - } - - size_t tail = len & (WS-1); - if(tail) { - unsigned shl = ((WS - tail) * 8), shr = tail * 8; - uintptr_t tmp = accum[0] << shl; - uintptr_t tmpLo = CLMULL(tmp, MUL_HI); - uintptr_t tmpHi = CLMULH(tmp, MUL_HI); - accum[0] = (accum[0] >> shr) | (accum[1] << shl); - accum[1] = (accum[1] >> shr) | (accum[2] << shl); - accum[2] = (accum[2] >> shr) | (accum[3] << shl); - accum[3] = (accum[3] >> shr) | (read_partial(srcW, tail) << shl); - accum[2] ^= tmpLo; - accum[3] ^= tmpHi; - } - - - // done processing: fold everything down -#if __riscv_xlen == 64 - // fold 0,1 -> 2,3 - accum[2] ^= rv_clmul(accum[0], 0x1751997d0) ^ rv_clmul(accum[1], 0xccaa009e); - accum[3] ^= rv_clmulh(accum[0], 0x1751997d0) ^ rv_clmulh(accum[1], 0xccaa009e); - - // fold 2->3 - accum[0] = rv_clmulh(accum[2], 0xccaa009e); - accum[3] ^= rv_clmul(accum[2], 0xccaa009e); - - // fold 64b->32b - accum[1] = rv_clmul(accum[3] & 0xffffffff, 0x163cd6124); - accum[0] ^= accum[1] >> 32; - accum[3] = accum[1] ^ (accum[3] >> 32); - accum[3] <<= 32; -#else - // fold 0,1 -> 2,3 - accum[2] ^= rv_clmul(accum[0], 0xccaa009e) ^ CLMULL(accum[1], 0x163cd6124); - accum[3] ^= rv_clmulh(accum[0], 0xccaa009e) ^ CLMULH(accum[1], 0x163cd6124); - - // fold 2->3 - accum[0] = CLMULH(accum[2], 0x163cd6124); - accum[3] ^= CLMULL(accum[2], 0x163cd6124); -#endif - - // reduction - accum[3] = CLMULL(accum[3], 0xf7011641); - accum[3] = CLMULH(accum[3], 0x1db710640); // maybe consider clmulr for XLEN=32 - crc = accum[0] ^ accum[3]; - return crc; - #undef CLMULL - #undef CLMULH -} - -static uint32_t do_crc32_incremental_rv_zbc(const void* data, size_t length, uint32_t init) { - return ~rv_crc_calc(~init, (const unsigned char*)data, (long)length); -} - - -#if __riscv_xlen == 64 - // note that prod is shifted by 1 place to the right, due to bit-reflection -static uint32_t crc32_reduce_rv_zbc(uint64_t prod) { - uint64_t t = rv_clmul(prod << 33, 0xf7011641); - t = rv_clmulh(t, 0x1db710640); - t ^= prod >> 31; - return t; -} -#endif -static uint32_t crc32_multiply_rv_zbc(uint32_t a, uint32_t b) { -#if __riscv_xlen == 64 - uint64_t t = crc32_reduce_rv_zbc(rv_clmul(a, b)); -#else - uint32_t prodLo = rv_clmul(a, b); - uint32_t prodHi = rv_clmulh(a, b); - - // fix prodHi for bit-reflection (clmulr would be ideal here) - prodHi += prodHi; - prodHi |= prodLo >> 31; - prodLo += prodLo; - - uint32_t t = rv_clmul(prodLo, 0xf7011641); - t ^= rv_clmulh(t, 0xdb710640); - t ^= prodHi; -#endif - return t; -} - -#if defined(__GNUC__) || defined(_MSC_VER) -static uint32_t crc32_shift_rv_zbc(uint32_t crc1, uint32_t n) { - // TODO: require Zbb for ctz - uint32_t result = crc1; -#if __riscv_xlen == 64 - // for n<32, can shift directly - uint64_t prod = result; - prod <<= 31 ^ (n&31); - n &= ~31; - result = crc32_reduce_rv_zbc(prod); -#endif - if(!n) return result; - - uint32_t result2 = RapidYenc::crc_power[ctz32(n)]; - n &= n-1; - - while(n) { - result = crc32_multiply_rv_zbc(result, RapidYenc::crc_power[ctz32(n)]); - n &= n-1; - - if(n) { - result2 = crc32_multiply_rv_zbc(result2, RapidYenc::crc_power[ctz32(n)]); - n &= n-1; - } - } - return crc32_multiply_rv_zbc(result, result2); -} -#endif - - -void RapidYenc::crc_riscv_set_funcs() { - _do_crc32_incremental = &do_crc32_incremental_rv_zbc; - _crc32_multiply = &crc32_multiply_rv_zbc; -#if defined(__GNUC__) || defined(_MSC_VER) - _crc32_shift = &crc32_shift_rv_zbc; -#endif - _crc32_isa = ISA_FEATURE_ZBC; -} -#else -void RapidYenc::crc_riscv_set_funcs() {} -#endif diff --git a/rapidyenc/rapidyenc/src/decoder.cc b/rapidyenc/rapidyenc/src/decoder.cc deleted file mode 100644 index 6dda4ef..0000000 --- a/rapidyenc/rapidyenc/src/decoder.cc +++ /dev/null @@ -1,429 +0,0 @@ -#include "common.h" - -#include "decoder_common.h" -#include "decoder.h" - - - -// TODO: add branch probabilities - - -// state var: refers to the previous state - only used for incremental processing -template -static size_t do_decode_noend_scalar(const unsigned char* src, unsigned char* dest, size_t len, RapidYenc::YencDecoderState* state) { - using namespace RapidYenc; - - const unsigned char *es = src + len; // end source pointer - unsigned char *p = dest; // destination pointer - long i = -(long)len; // input position - unsigned char c; // input character - - if(len < 1) return 0; - - if(isRaw) { - - if(state) switch(*state) { - case YDEC_STATE_EQ: - c = es[i]; - *p++ = c - 42 - 64; - i++; - if(c == '\r') { - *state = YDEC_STATE_CR; - if(i >= 0) return 0; - } else { - *state = YDEC_STATE_NONE; - break; - } - // fall-thru - case YDEC_STATE_CR: - if(es[i] != '\n') break; - i++; - *state = YDEC_STATE_CRLF; - if(i >= 0) return 0; - // Else fall-thru - case YDEC_STATE_CRLF: - // skip past first dot - if(es[i] == '.') i++; - // fall-thru - default: break; // silence compiler warnings - } else // treat as YDEC_STATE_CRLF - if(es[i] == '.') i++; - - for(; i < -2; i++) { - c = es[i]; - switch(c) { - case '\r': - // skip past \r\n. sequences - //i += (es[i+1] == '\n' && es[i+2] == '.') << 1; - if(es[i+1] == '\n' && es[i+2] == '.') - i += 2; - // fall-thru - case '\n': - continue; - case '=': - c = es[i+1]; - *p++ = c - 42 - 64; - i += (c != '\r'); // if we have a \r, reprocess character to deal with \r\n. case - continue; - default: - *p++ = c - 42; - } - } - if(state) *state = YDEC_STATE_NONE; - - if(i == -2) { // 2nd last char - c = es[i]; - switch(c) { - case '\r': - if(state && es[i+1] == '\n') { - *state = YDEC_STATE_CRLF; - return p - dest; - } - // Else fall-thru - case '\n': - break; - case '=': - c = es[i+1]; - *p++ = c - 42 - 64; - i += (c != '\r'); - break; - default: - *p++ = c - 42; - } - i++; - } - - // do final char; we process this separately to prevent an overflow if the final char is '=' - if(i == -1) { - c = es[i]; - if(c != '\n' && c != '\r' && c != '=') { - *p++ = c - 42; - } else if(state) { - if(c == '=') *state = YDEC_STATE_EQ; - else if(c == '\r') *state = YDEC_STATE_CR; - else *state = YDEC_STATE_NONE; - } - } - - } else { - - if(state && *state == YDEC_STATE_EQ) { - *p++ = es[i] - 42 - 64; - i++; - *state = YDEC_STATE_NONE; - } - - /*for(i = 0; i < len - 1; i++) { - c = src[i]; - if(c == '\n' || c == '\r') continue; - unsigned char isEquals = (c == '='); - i += isEquals; - *p++ = src[i] - (42 + (isEquals << 6)); - }*/ - for(; i < -1; i++) { - c = es[i]; - switch(c) { - case '\n': case '\r': continue; - case '=': - i++; - c = es[i] - 64; - } - *p++ = c - 42; - } - if(state) *state = YDEC_STATE_NONE; - // do final char; we process this separately to prevent an overflow if the final char is '=' - if(i == -1) { - c = es[i]; - if(c != '\n' && c != '\r' && c != '=') { - *p++ = c - 42; - } else - if(state) *state = (c == '=' ? YDEC_STATE_EQ : YDEC_STATE_NONE); - } - - } - - return p - dest; -} - -template -static RapidYenc::YencDecoderEnd do_decode_end_scalar(const unsigned char** src, unsigned char** dest, size_t len, RapidYenc::YencDecoderState* state) { - using namespace RapidYenc; - - const unsigned char *es = (*src) + len; // end source pointer - unsigned char *p = *dest; // destination pointer - long i = -(long)len; // input position - unsigned char c; // input character - - if(len < 1) return YDEC_END_NONE; - -#define YDEC_CHECK_END(s) if(i == 0) { \ - *state = s; \ - *src = es; \ - *dest = p; \ - return YDEC_END_NONE; \ -} - if(state) switch(*state) { - case YDEC_STATE_CRLFEQ: do_decode_endable_scalar_ceq: - if(es[i] == 'y') { - *state = YDEC_STATE_NONE; - *src = es+i+1; - *dest = p; - return YDEC_END_CONTROL; - } // Else fall-thru - case YDEC_STATE_EQ: - c = es[i]; - *p++ = c - 42 - 64; - i++; - if(c != '\r') break; - YDEC_CHECK_END(YDEC_STATE_CR) - // fall-through - case YDEC_STATE_CR: - if(es[i] != '\n') break; - i++; - YDEC_CHECK_END(YDEC_STATE_CRLF) - // fall-through - case YDEC_STATE_CRLF: do_decode_endable_scalar_c0: - if(es[i] == '.' && isRaw) { - i++; - YDEC_CHECK_END(YDEC_STATE_CRLFDT) - } else if(es[i] == '=') { - i++; - YDEC_CHECK_END(YDEC_STATE_CRLFEQ) - goto do_decode_endable_scalar_ceq; - } else - break; - // fall-through - case YDEC_STATE_CRLFDT: - if(isRaw && es[i] == '\r') { - i++; - YDEC_CHECK_END(YDEC_STATE_CRLFDTCR) - } else if(isRaw && es[i] == '=') { // check for dot-stuffed ending: \r\n.=y - i++; - YDEC_CHECK_END(YDEC_STATE_CRLFEQ) - goto do_decode_endable_scalar_ceq; - } else - break; - // fall-through - case YDEC_STATE_CRLFDTCR: - if(es[i] == '\n') { - if(isRaw) { - *state = YDEC_STATE_CRLF; - *src = es + i + 1; - *dest = p; - return YDEC_END_ARTICLE; - } else { - i++; - YDEC_CHECK_END(YDEC_STATE_CRLF) - goto do_decode_endable_scalar_c0; // handle as CRLF - } - } else - break; - case YDEC_STATE_NONE: break; // silence compiler warning - } else // treat as YDEC_STATE_CRLF - goto do_decode_endable_scalar_c0; - - for(; i < -2; i++) { - c = es[i]; - switch(c) { - case '\r': if(es[i+1] == '\n') { - if(isRaw && es[i+2] == '.') { - // skip past \r\n. sequences - i += 3; - YDEC_CHECK_END(YDEC_STATE_CRLFDT) - // check for end - if(es[i] == '\r') { - i++; - YDEC_CHECK_END(YDEC_STATE_CRLFDTCR) - if(es[i] == '\n') { - *src = es + i + 1; - *dest = p; - *state = YDEC_STATE_CRLF; - return YDEC_END_ARTICLE; - } else i--; - } else if(es[i] == '=') { - i++; - YDEC_CHECK_END(YDEC_STATE_CRLFEQ) - if(es[i] == 'y') { - *src = es + i + 1; - *dest = p; - *state = YDEC_STATE_NONE; - return YDEC_END_CONTROL; - } else { - // escape char & continue - c = es[i]; - *p++ = c - 42 - 64; - i -= (c == '\r'); - } - } else i--; - } - else if(es[i+2] == '=') { - i += 3; - YDEC_CHECK_END(YDEC_STATE_CRLFEQ) - if(es[i] == 'y') { - // ended - *src = es + i + 1; - *dest = p; - *state = YDEC_STATE_NONE; - return YDEC_END_CONTROL; - } else { - // escape char & continue - c = es[i]; - *p++ = c - 42 - 64; - i -= (c == '\r'); - } - } - } // fall-thru - case '\n': - continue; - case '=': - c = es[i+1]; - *p++ = c - 42 - 64; - i += (c != '\r'); // if we have a \r, reprocess character to deal with \r\n. case - continue; - default: - *p++ = c - 42; - } - } - if(state) *state = YDEC_STATE_NONE; - - if(i == -2) { // 2nd last char - c = es[i]; - switch(c) { - case '\r': - if(state && es[i+1] == '\n') { - *state = YDEC_STATE_CRLF; - *src = es; - *dest = p; - return YDEC_END_NONE; - } - // Else fall-thru - case '\n': - break; - case '=': - c = es[i+1]; - *p++ = c - 42 - 64; - i += (c != '\r'); - break; - default: - *p++ = c - 42; - } - i++; - } - - // do final char; we process this separately to prevent an overflow if the final char is '=' - if(i == -1) { - c = es[i]; - if(c != '\n' && c != '\r' && c != '=') { - *p++ = c - 42; - } else if(state) { - if(c == '=') *state = YDEC_STATE_EQ; - else if(c == '\r') *state = YDEC_STATE_CR; - else *state = YDEC_STATE_NONE; - } - } -#undef YDEC_CHECK_END - - *src = es; - *dest = p; - return YDEC_END_NONE; -} - -template -RapidYenc::YencDecoderEnd RapidYenc::do_decode_scalar(const unsigned char** src, unsigned char** dest, size_t len, RapidYenc::YencDecoderState* state) { - if(searchEnd) - return do_decode_end_scalar(src, dest, len, state); - *dest += do_decode_noend_scalar(*src, *dest, len, state); - *src += len; - return YDEC_END_NONE; -} - - -namespace RapidYenc { - YencDecoderEnd (*_do_decode)(const unsigned char**, unsigned char**, size_t, YencDecoderState*) = &do_decode_scalar; - YencDecoderEnd (*_do_decode_raw)(const unsigned char**, unsigned char**, size_t, YencDecoderState*) = &do_decode_scalar; - YencDecoderEnd (*_do_decode_end_raw)(const unsigned char**, unsigned char**, size_t, YencDecoderState*) = &do_decode_end_scalar; - - int _decode_isa = ISA_GENERIC; - - template YencDecoderEnd do_decode_scalar(const unsigned char**, unsigned char**, size_t, YencDecoderState*); -} - - -#if defined(PLATFORM_X86) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0 -# if defined(__AVX2__) && !defined(YENC_DISABLE_AVX256) -# include "decoder_avx2_base.h" -static inline void decoder_set_native_funcs() { - ALIGN_ALLOC(lookups, sizeof(*lookups), 16); - using namespace RapidYenc; - decoder_init_lut(lookups->compact); - _do_decode = &do_decode_simd >; - _do_decode_raw = &do_decode_simd >; - _do_decode_end_raw = &do_decode_simd >; - _decode_isa = ISA_NATIVE; -} -# else -# include "decoder_sse_base.h" -static inline void decoder_set_native_funcs() { - using namespace RapidYenc; - decoder_sse_init(lookups); - decoder_init_lut(lookups->compact); - _do_decode = &do_decode_simd >; - _do_decode_raw = &do_decode_simd >; - _do_decode_end_raw = &do_decode_simd >; - _decode_isa = ISA_NATIVE; -} -# endif -#endif - - -#if defined(PLATFORM_X86) || defined(PLATFORM_ARM) -void RapidYenc::decoder_init_lut(void* compactLUT) { - #ifdef YENC_DEC_USE_THINTABLE - const int tableSize = 8; - #else - const int tableSize = 16; - #endif - for(int i=0; i<(tableSize==8?256:32768); i++) { - int k = i; - uint8_t* res = (uint8_t*)compactLUT + i*tableSize; - int p = 0; - for(int j=0; j>= 1; - } - for(; p= ISA_LEVEL_VBMI2 && (decoder_has_avx10 || (use_isa & ISA_FEATURE_EVEX512))) - decoder_set_vbmi2_funcs(); - else if(use_isa >= ISA_LEVEL_AVX2) - decoder_set_avx2_funcs(); - else if(use_isa >= ISA_LEVEL_AVX) - decoder_set_avx_funcs(); - else if(use_isa >= ISA_LEVEL_SSSE3) - decoder_set_ssse3_funcs(); - else - decoder_set_sse2_funcs(); -# endif -#endif -#ifdef PLATFORM_ARM - if(cpu_supports_neon()) - decoder_set_neon_funcs(); -#endif -#ifdef __riscv - if(cpu_supports_rvv()) - decoder_set_rvv_funcs(); -#endif -} diff --git a/rapidyenc/rapidyenc/src/decoder.h b/rapidyenc/rapidyenc/src/decoder.h deleted file mode 100644 index 60a4d25..0000000 --- a/rapidyenc/rapidyenc/src/decoder.h +++ /dev/null @@ -1,53 +0,0 @@ -#ifndef __YENC_DECODER_H -#define __YENC_DECODER_H - -#include "hedley.h" - -namespace RapidYenc { - - -// the last state that the decoder was in (i.e. last few characters processed) -// the state is needed for incremental decoders as its behavior is affected by what it processed last -// acronyms: CR = carriage return (\r), LF = line feed (\n), EQ = equals char, DT = dot char (.) -typedef enum { - YDEC_STATE_CRLF, // default - YDEC_STATE_EQ, - YDEC_STATE_CR, - YDEC_STATE_NONE, - YDEC_STATE_CRLFDT, - YDEC_STATE_CRLFDTCR, - YDEC_STATE_CRLFEQ // may actually be "\r\n.=" in raw decoder -} YencDecoderState; - -// end result for incremental processing (whether the end of the yEnc data was reached) -typedef enum { - YDEC_END_NONE, // end not reached - YDEC_END_CONTROL, // \r\n=y sequence found, src points to byte after 'y' - YDEC_END_ARTICLE // \r\n.\r\n sequence found, src points to byte after last '\n' -} YencDecoderEnd; - - -extern YencDecoderEnd (*_do_decode)(const unsigned char**, unsigned char**, size_t, YencDecoderState*); -extern YencDecoderEnd (*_do_decode_raw)(const unsigned char**, unsigned char**, size_t, YencDecoderState*); -extern YencDecoderEnd (*_do_decode_end_raw)(const unsigned char**, unsigned char**, size_t, YencDecoderState*); -extern int _decode_isa; - -static inline size_t decode(int isRaw, const void* src, void* dest, size_t len, YencDecoderState* state) { - unsigned char* ds = (unsigned char*)dest; - (*(isRaw ? _do_decode_raw : _do_decode))((const unsigned char**)&src, &ds, len, state); - return ds - (unsigned char*)dest; -} - -static inline YencDecoderEnd decode_end(const void** src, void** dest, size_t len, YencDecoderState* state) { - return _do_decode_end_raw((const unsigned char**)src, (unsigned char**)dest, len, state); -} - -void decoder_init(); - -static inline int decode_isa_level() { - return _decode_isa; -} - - -} // namespace -#endif // defined(__YENC_DECODER_H) diff --git a/rapidyenc/rapidyenc/src/decoder_avx.cc b/rapidyenc/rapidyenc/src/decoder_avx.cc deleted file mode 100644 index 443552a..0000000 --- a/rapidyenc/rapidyenc/src/decoder_avx.cc +++ /dev/null @@ -1,18 +0,0 @@ -#include "common.h" - -#include "decoder_common.h" -#if defined(__AVX__) && defined(__POPCNT__) -#include "decoder_sse_base.h" -void RapidYenc::decoder_set_avx_funcs() { - decoder_sse_init(lookups); - decoder_init_lut(lookups->compact); - _do_decode = &do_decode_simd >; - _do_decode_raw = &do_decode_simd >; - _do_decode_end_raw = &do_decode_simd >; - _decode_isa = ISA_LEVEL_AVX; -} -#else -void RapidYenc::decoder_set_avx_funcs() { - decoder_set_ssse3_funcs(); -} -#endif diff --git a/rapidyenc/rapidyenc/src/decoder_avx2.cc b/rapidyenc/rapidyenc/src/decoder_avx2.cc deleted file mode 100644 index 638bb12..0000000 --- a/rapidyenc/rapidyenc/src/decoder_avx2.cc +++ /dev/null @@ -1,18 +0,0 @@ -#include "common.h" - -#include "decoder_common.h" -#if defined(__AVX2__) && !defined(YENC_DISABLE_AVX256) -#include "decoder_avx2_base.h" -void RapidYenc::decoder_set_avx2_funcs() { - ALIGN_ALLOC(lookups, sizeof(*lookups), 16); - decoder_init_lut(lookups->compact); - RapidYenc::_do_decode = &do_decode_simd >; - RapidYenc::_do_decode_raw = &do_decode_simd >; - RapidYenc::_do_decode_end_raw = &do_decode_simd >; - RapidYenc::_decode_isa = ISA_LEVEL_AVX2; -} -#else -void RapidYenc::decoder_set_avx2_funcs() { - decoder_set_avx_funcs(); -} -#endif diff --git a/rapidyenc/rapidyenc/src/decoder_avx2_base.h b/rapidyenc/rapidyenc/src/decoder_avx2_base.h deleted file mode 100644 index ca46a27..0000000 --- a/rapidyenc/rapidyenc/src/decoder_avx2_base.h +++ /dev/null @@ -1,618 +0,0 @@ - -#ifdef __AVX2__ - -// GCC (ver 6-10(dev)) fails to optimize pure C version of mask testing, but has this intrinsic; Clang >= 7 optimizes C version fine; functions added in Clang 8 -#if (defined(__GNUC__) && __GNUC__ >= 7) || (defined(_MSC_VER) && _MSC_VER >= 1924) -# define KORTEST32(a, b) !_kortestz_mask32_u8((a), (b)) -# define KAND32(a, b) _kand_mask32((a), (b)) -# define KOR32(a, b) _kor_mask32((a), (b)) -#else -# define KORTEST32(a, b) ((a) | (b)) -# define KAND32(a, b) ((a) & (b)) -# define KOR32(a, b) ((a) | (b)) -#endif - -#pragma pack(16) -static struct { - /*align16*/ struct { char bytes[16]; } compact[32768]; -} * HEDLEY_RESTRICT lookups; -#pragma pack() - - -static HEDLEY_ALWAYS_INLINE __m256i force_align_read_256(const void* p) { -#ifdef _MSC_VER - // MSVC complains about casting away volatile - return *(__m256i *)(p); -#else - return *(volatile __m256i *)(p); -#endif -} - -// _mm256_castsi128_si256, but upper is defined to be 0 -#if (defined(__clang__) && __clang_major__ >= 5 && (!defined(__APPLE__) || __clang_major__ >= 7)) || (defined(__GNUC__) && __GNUC__ >= 10) || (defined(_MSC_VER) && _MSC_VER >= 1910) -// intrinsic unsupported in GCC 9 and MSVC < 2017 -# define zext128_256 _mm256_zextsi128_si256 -#else -// technically a cast is incorrect, due to upper 128 bits being undefined, but should usually work fine -// alternative may be `_mm256_set_m128i(_mm_setzero_si128(), v)` but unsupported on GCC < 7, and most compilers generate a VINSERTF128 instruction for it -# ifdef __OPTIMIZE__ -# define zext128_256 _mm256_castsi128_si256 -# else -# define zext128_256(x) _mm256_inserti128_si256(_mm256_setzero_si256(), x, 0) -# endif -#endif - -#if defined(__tune_icelake_client__) || defined(__tune_icelake_server__) || defined(__tune_tigerlake__) || defined(__tune_rocketlake__) || defined(__tune_alderlake__) || defined(__tune_sapphirerapids__) -# define COMPRESS_STORE _mm256_mask_compressstoreu_epi8 -#else -// avoid uCode on Zen4 -# define COMPRESS_STORE(dst, mask, vec) _mm256_storeu_si256((__m256i*)(dst), _mm256_maskz_compress_epi8(mask, vec)) -#endif - -namespace RapidYenc { - -template -HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* src, long& len, unsigned char*& p, unsigned char& _escFirst, uint16_t& _nextMask) { - HEDLEY_ASSUME(_escFirst == 0 || _escFirst == 1); - HEDLEY_ASSUME(_nextMask == 0 || _nextMask == 1 || _nextMask == 2); - uintptr_t escFirst = _escFirst; - __m256i yencOffset = escFirst ? _mm256_set_epi8( - -42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42, - -42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42-64 - ) : _mm256_set1_epi8(-42); - __m256i minMask = _mm256_set1_epi8('.'); - if(_nextMask && isRaw) { - minMask = _mm256_set_epi8( - '.','.','.','.','.','.','.','.','.','.','.','.','.','.','.','.', - '.','.','.','.','.','.','.','.','.','.','.','.','.','.',_nextMask==2?0:'.',_nextMask==1?0:'.' - ); - } - - decoder_set_nextMask(src, len, _nextMask); // set this before the loop because we can't check src after it's been overwritten - - // for some reason, MSVC Win32 seems to crash when trying to compile _mm256_mask_cmpeq_epi8_mask - // the crash can be fixed by switching the order of the last two arguments, but it seems to generate wrong code - // so just disable the optimisation as it seems to be problematic there -#if defined(__AVX512VL__) && defined(__AVX512BW__) -# if defined(_MSC_VER) && !defined(PLATFORM_AMD64) && !defined(__clang__) - const bool useAVX3MaskCmp = false; -# else - const bool useAVX3MaskCmp = (use_isa >= ISA_LEVEL_AVX3); -# endif -#endif - intptr_t i; - for(i = -len; i; i += sizeof(__m256i)*2) { - __m256i oDataA = _mm256_load_si256((__m256i *)(src+i)); - __m256i oDataB = _mm256_load_si256((__m256i *)(src+i) + 1); - - // search for special chars - __m256i cmpA = _mm256_cmpeq_epi8(oDataA, _mm256_shuffle_epi8( - _mm256_set_epi8( - -1,'=','\r',-1,-1,'\n',-1,-1,-1,-1,-1,-1,-1,-1,-1,'.', - -1,'=','\r',-1,-1,'\n',-1,-1,-1,-1,-1,-1,-1,-1,-1,'.' - ), - _mm256_min_epu8(oDataA, minMask) - )); - __m256i cmpB = _mm256_cmpeq_epi8(oDataB, _mm256_shuffle_epi8( - _mm256_set_epi8( - -1,'=','\r',-1,-1,'\n',-1,-1,-1,-1,-1,-1,-1,-1,-1,'.', - -1,'=','\r',-1,-1,'\n',-1,-1,-1,-1,-1,-1,-1,-1,-1,'.' - ), - _mm256_min_epu8(oDataB, _mm256_set1_epi8('.')) - )); - - // TODO: can OR the vectors together to save generating a mask, but may not be worth it - uint64_t mask = (uint32_t)_mm256_movemask_epi8(cmpB); // not the most accurate mask if we have invalid sequences; we fix this up later - mask = (mask << 32) | (uint32_t)_mm256_movemask_epi8(cmpA); - __m256i dataA, dataB; - if(use_isa >= ISA_LEVEL_AVX3) - dataA = _mm256_add_epi8(oDataA, yencOffset); - - if (mask != 0) { - __m256i cmpEqA = _mm256_cmpeq_epi8(oDataA, _mm256_set1_epi8('=')); - __m256i cmpEqB = _mm256_cmpeq_epi8(oDataB, _mm256_set1_epi8('=')); - uint64_t maskEq = (uint32_t)_mm256_movemask_epi8(cmpEqB); - maskEq = (maskEq << 32) | (uint32_t)_mm256_movemask_epi8(cmpEqA); - - // handle \r\n. sequences - // RFC3977 requires the first dot on a line to be stripped, due to dot-stuffing - if((isRaw || searchEnd) && LIKELIHOOD(0.45, mask != maskEq)) { -#if 0 - // prefer shuffling data over unaligned loads on Zen (unknown if worth it on Zen2/Excavator) - // unfortunately not beneficial, probably due to available register pressure; this is left here because it could be beneficial if we figure out how to use fewer registers - __m256i nextDataA, nextDataB; - if(searchEnd) { - nextDataA = _mm256_inserti128_si256( - _mm256_castsi128_si256(_mm256_extracti128_si256(oDataA, 1)), - _mm256_castsi256_si128(oDataB), - 1 - ); - nextDataB = _mm256_inserti128_si256( - _mm256_castsi128_si256(_mm256_extracti128_si256(oDataB, 1)), - _mm_load_si128((__m128i*)(src+i+sizeof(__m256i)*2)), - 1 - ); - } -# define SHIFT_DATA_A(offs) (searchEnd ? _mm256_alignr_epi8(nextDataA, oDataA, offs) : _mm256_loadu_si256((__m256i *)(src+i+offs))) -# define SHIFT_DATA_B(offs) (searchEnd ? _mm256_alignr_epi8(nextDataB, oDataB, offs) : _mm256_loadu_si256((__m256i *)(src+i+offs) + 1)) -#else -# define SHIFT_DATA_A(offs) _mm256_loadu_si256((__m256i *)(src+i+offs)) -# define SHIFT_DATA_B(offs) _mm256_loadu_si256((__m256i *)(src+i+offs) + 1) -#endif - __m256i tmpData2A = SHIFT_DATA_A(2); - __m256i tmpData2B = SHIFT_DATA_B(2); - __m256i match2EqA, match2EqB; -#if defined(__AVX512VL__) && defined(__AVX512BW__) - __mmask32 match2EqMaskA, match2EqMaskB; - __mmask32 match0CrMaskA, match0CrMaskB; - __mmask32 match2CrXDtMaskA, match2CrXDtMaskB; - if(useAVX3MaskCmp && searchEnd) { - match2EqMaskA = _mm256_cmpeq_epi8_mask(_mm256_set1_epi8('='), tmpData2A); - match2EqMaskB = _mm256_cmpeq_epi8_mask(_mm256_set1_epi8('='), tmpData2B); - } else -#endif - if(searchEnd) { - match2EqA = _mm256_cmpeq_epi8(_mm256_set1_epi8('='), tmpData2A); - match2EqB = _mm256_cmpeq_epi8(_mm256_set1_epi8('='), tmpData2B); - } - - int partialKillDotFound; - __m256i match2CrXDtA, match2CrXDtB; - if(isRaw) { - // find patterns of \r_. - -#if defined(__AVX512VL__) && defined(__AVX512BW__) - if(useAVX3MaskCmp) { - match0CrMaskA = _mm256_cmpeq_epi8_mask(oDataA, _mm256_set1_epi8('\r')); - match0CrMaskB = _mm256_cmpeq_epi8_mask(oDataB, _mm256_set1_epi8('\r')); - match2CrXDtMaskA = _mm256_mask_cmpeq_epi8_mask(match0CrMaskA, tmpData2A, _mm256_set1_epi8('.')); - match2CrXDtMaskB = _mm256_mask_cmpeq_epi8_mask(match0CrMaskB, tmpData2B, _mm256_set1_epi8('.')); - partialKillDotFound = KORTEST32(match2CrXDtMaskA, match2CrXDtMaskB); - } else -#endif - { - match2CrXDtA = _mm256_and_si256( - _mm256_cmpeq_epi8(oDataA, _mm256_set1_epi8('\r')), - _mm256_cmpeq_epi8(tmpData2A, _mm256_set1_epi8('.')) - ); - match2CrXDtB = _mm256_and_si256( - _mm256_cmpeq_epi8(oDataB, _mm256_set1_epi8('\r')), - _mm256_cmpeq_epi8(tmpData2B, _mm256_set1_epi8('.')) - ); - partialKillDotFound = _mm256_movemask_epi8(_mm256_or_si256( - match2CrXDtA, match2CrXDtB - )); - } - } - - if(isRaw && LIKELIHOOD(0.002, partialKillDotFound)) { - // merge matches for \r\n. - __m256i match2NlDotA, match1NlA; - __m256i match2NlDotB, match1NlB; -#if defined(__AVX512VL__) && defined(__AVX512BW__) - __mmask32 match1NlMaskA, match1NlMaskB; - __mmask32 match2NlDotMaskA, match2NlDotMaskB; - if(useAVX3MaskCmp) { - match1NlMaskA = _mm256_mask_cmpeq_epi8_mask( - match0CrMaskA, - _mm256_set1_epi8('\n'), - SHIFT_DATA_A(1) - ); - match1NlMaskB = _mm256_mask_cmpeq_epi8_mask( - match0CrMaskB, - _mm256_set1_epi8('\n'), - SHIFT_DATA_B(1) - ); - match2NlDotMaskA = KAND32(match2CrXDtMaskA, match1NlMaskA); - match2NlDotMaskB = KAND32(match2CrXDtMaskB, match1NlMaskB); - } else -#endif - { - __m256i match1LfA = _mm256_cmpeq_epi8( - _mm256_set1_epi8('\n'), - SHIFT_DATA_A(1) - ); - __m256i match1LfB = _mm256_cmpeq_epi8( - _mm256_set1_epi8('\n'), - SHIFT_DATA_B(1) - ); - // force re-computing these to avoid register spills elsewhere - match1NlA = _mm256_and_si256(match1LfA, _mm256_cmpeq_epi8(force_align_read_256(src+i), _mm256_set1_epi8('\r'))); - match1NlB = _mm256_and_si256(match1LfB, _mm256_cmpeq_epi8(force_align_read_256(src+i + sizeof(__m256i)), _mm256_set1_epi8('\r'))); - match2NlDotA = _mm256_and_si256(match2CrXDtA, match1NlA); - match2NlDotB = _mm256_and_si256(match2CrXDtB, match1NlB); - } - if(searchEnd) { - __m256i tmpData4A; -#if defined(__AVX512VL__) && defined(PLATFORM_AMD64) - if(use_isa >= ISA_LEVEL_AVX3) - // AVX512 with 32 registers shouldn't have any issue with holding onto oData* in registers - tmpData4A = _mm256_alignr_epi32(oDataB, oDataA, 1); - else -#endif - tmpData4A = SHIFT_DATA_A(4); - __m256i tmpData4B = SHIFT_DATA_B(4); - // match instances of \r\n.\r\n and \r\n.=y - __m256i match3CrA = _mm256_cmpeq_epi8( - _mm256_set1_epi8('\r'), - SHIFT_DATA_A(3) - ); - __m256i match3CrB = _mm256_cmpeq_epi8( - _mm256_set1_epi8('\r'), - SHIFT_DATA_B(3) - ); - __m256i match4LfA = _mm256_cmpeq_epi8(tmpData4A, _mm256_set1_epi8('\n')); - __m256i match4LfB = _mm256_cmpeq_epi8(tmpData4B, _mm256_set1_epi8('\n')); - __m256i match4EqYA = _mm256_cmpeq_epi16(tmpData4A, _mm256_set1_epi16(0x793d)); // =y - __m256i match4EqYB = _mm256_cmpeq_epi16(tmpData4B, _mm256_set1_epi16(0x793d)); // =y - - int matchEnd; -#if defined(__AVX512VL__) && defined(__AVX512BW__) - if(useAVX3MaskCmp) { - __mmask32 match3EqYMaskA = _mm256_mask_cmpeq_epi8_mask( - match2EqMaskA, - _mm256_set1_epi8('y'), - SHIFT_DATA_A(3) - ); - __mmask32 match3EqYMaskB = _mm256_mask_cmpeq_epi8_mask( - match2EqMaskB, - _mm256_set1_epi8('y'), - SHIFT_DATA_B(3) - ); - __m256i match34EqYA, match34EqYB; -# ifdef __AVX512VBMI2__ - if(use_isa >= ISA_LEVEL_VBMI2) { - match34EqYA = _mm256_shrdi_epi16(_mm256_movm_epi8(match3EqYMaskA), match4EqYA, 8); - match34EqYB = _mm256_shrdi_epi16(_mm256_movm_epi8(match3EqYMaskB), match4EqYB, 8); - } else -# endif - { - // (match4EqY & 0xff00) | (match3EqY >> 8) - match34EqYA = _mm256_mask_blend_epi8(match3EqYMaskA>>1, _mm256_and_si256(match4EqYA, _mm256_set1_epi16(-0xff)), _mm256_set1_epi8(-1)); - match34EqYB = _mm256_mask_blend_epi8(match3EqYMaskB>>1, _mm256_and_si256(match4EqYB, _mm256_set1_epi16(-0xff)), _mm256_set1_epi8(-1)); - } - // merge \r\n and =y matches for tmpData4 - __m256i match4EndA = _mm256_ternarylogic_epi32(match34EqYA, match3CrA, match4LfA, 0xF8); // (match3Cr & match4Lf) | match34EqY - __m256i match4EndB = _mm256_ternarylogic_epi32(match34EqYB, match3CrB, match4LfB, 0xF8); - // merge with \r\n. and combine - matchEnd = KORTEST32( - KOR32( - _mm256_mask_test_epi8_mask(match2NlDotMaskA, match4EndA, match4EndA), - KAND32(match3EqYMaskA, match1NlMaskA) - ), - KOR32( - _mm256_mask_test_epi8_mask(match2NlDotMaskB, match4EndB, match4EndB), - KAND32(match3EqYMaskB, match1NlMaskB) - ) - ); - } else -#endif - { - __m256i match3EqYA = _mm256_and_si256(match2EqA, _mm256_cmpeq_epi8( - _mm256_set1_epi8('y'), - SHIFT_DATA_A(3) - )); - __m256i match3EqYB = _mm256_and_si256(match2EqB, _mm256_cmpeq_epi8( - _mm256_set1_epi8('y'), - SHIFT_DATA_B(3) - )); - match4EqYA = _mm256_slli_epi16(match4EqYA, 8); // TODO: also consider using PBLENDVB here with shifted match3EqY instead - match4EqYB = _mm256_slli_epi16(match4EqYB, 8); - // merge \r\n and =y matches for tmpData4 - __m256i match4EndA = _mm256_or_si256( - _mm256_and_si256(match3CrA, match4LfA), - _mm256_or_si256(match4EqYA, _mm256_srli_epi16(match3EqYA, 8)) // _mm256_srli_si256 by 1 also works - ); - __m256i match4EndB = _mm256_or_si256( - _mm256_and_si256(match3CrB, match4LfB), - _mm256_or_si256(match4EqYB, _mm256_srli_epi16(match3EqYB, 8)) - ); - // merge with \r\n. - match4EndA = _mm256_and_si256(match4EndA, match2NlDotA); - match4EndB = _mm256_and_si256(match4EndB, match2NlDotB); - // match \r\n=y - __m256i match3EndA = _mm256_and_si256(match3EqYA, match1NlA); - __m256i match3EndB = _mm256_and_si256(match3EqYB, match1NlB); - // combine match sequences - matchEnd = _mm256_movemask_epi8(_mm256_or_si256( - _mm256_or_si256(match4EndA, match3EndA), - _mm256_or_si256(match4EndB, match3EndB) - )); - } - if(LIKELIHOOD(0.002, matchEnd)) { - // terminator found - // there's probably faster ways to do this, but reverting to scalar code should be good enough - len += (long)i; - _nextMask = decoder_set_nextMask(src+i, mask); - break; - } - } -#if defined(__AVX512VL__) && defined(__AVX512BW__) - if(useAVX3MaskCmp) { - mask |= (uint64_t)match2NlDotMaskA << 2; - mask |= (uint64_t)match2NlDotMaskB << 34; - minMask = _mm256_maskz_mov_epi8(~(match2NlDotMaskB>>30), _mm256_set1_epi8('.')); - } else -#endif - { - mask |= (uint64_t)((uint32_t)_mm256_movemask_epi8(match2NlDotA)) << 2; - mask |= (uint64_t)((uint32_t)_mm256_movemask_epi8(match2NlDotB)) << 34; - match2NlDotB = zext128_256(_mm_srli_si128(_mm256_extracti128_si256(match2NlDotB, 1), 14)); - minMask = _mm256_subs_epu8(_mm256_set1_epi8('.'), match2NlDotB); - } - } - else if(searchEnd) { - bool partialEndFound; - __m256i match3EqYA, match3EqYB; -#if defined(__AVX512VL__) && defined(__AVX512BW__) - __mmask32 match3EqYMaskA, match3EqYMaskB; - if(useAVX3MaskCmp) { - match3EqYMaskA = _mm256_mask_cmpeq_epi8_mask( - match2EqMaskA, - _mm256_set1_epi8('y'), - SHIFT_DATA_A(3) - ); - match3EqYMaskB = _mm256_mask_cmpeq_epi8_mask( - match2EqMaskB, - _mm256_set1_epi8('y'), - SHIFT_DATA_B(3) - ); - partialEndFound = KORTEST32(match3EqYMaskA, match3EqYMaskB); - } else -#endif - { - __m256i match3YA = _mm256_cmpeq_epi8( - _mm256_set1_epi8('y'), - SHIFT_DATA_A(3) - ); - __m256i match3YB = _mm256_cmpeq_epi8( - _mm256_set1_epi8('y'), - SHIFT_DATA_B(3) - ); - match3EqYA = _mm256_and_si256(match2EqA, match3YA); - match3EqYB = _mm256_and_si256(match2EqB, match3YB); - partialEndFound = _mm256_movemask_epi8(_mm256_or_si256(match3EqYA, match3EqYB)); - } - if(LIKELIHOOD(0.002, partialEndFound)) { - bool endFound; -#if defined(__AVX512VL__) && defined(__AVX512BW__) - if(useAVX3MaskCmp) { - __mmask32 match3LfEqYMaskA = _mm256_mask_cmpeq_epi8_mask( - match3EqYMaskA, - _mm256_set1_epi8('\n'), - SHIFT_DATA_A(1) - ); - __mmask32 match3LfEqYMaskB = _mm256_mask_cmpeq_epi8_mask( - match3EqYMaskB, - _mm256_set1_epi8('\n'), - SHIFT_DATA_B(1) - ); - - endFound = KORTEST32( - _mm256_mask_cmpeq_epi8_mask(match3LfEqYMaskA, oDataA, _mm256_set1_epi8('\r')), - _mm256_mask_cmpeq_epi8_mask(match3LfEqYMaskB, oDataB, _mm256_set1_epi8('\r')) - ); - } else -#endif - { - __m256i match1LfA = _mm256_cmpeq_epi8( - _mm256_set1_epi8('\n'), - SHIFT_DATA_A(1) - ); - __m256i match1LfB = _mm256_cmpeq_epi8( - _mm256_set1_epi8('\n'), - SHIFT_DATA_B(1) - ); - endFound = _mm256_movemask_epi8(_mm256_or_si256( - _mm256_and_si256( - match3EqYA, - _mm256_and_si256(match1LfA, _mm256_cmpeq_epi8(force_align_read_256(src+i), _mm256_set1_epi8('\r'))) - ), - _mm256_and_si256( - match3EqYB, - _mm256_and_si256(match1LfB, _mm256_cmpeq_epi8(force_align_read_256(src+i + sizeof(__m256i)), _mm256_set1_epi8('\r'))) - ) - )); - } - if(endFound) { - len += (long)i; - _nextMask = decoder_set_nextMask(src+i, mask); - break; - } - } - if(isRaw) minMask = _mm256_set1_epi8('.'); - } - else if(isRaw) // no \r_. found - minMask = _mm256_set1_epi8('.'); - } -#undef SHIFT_DATA_A -#undef SHIFT_DATA_B - - if(use_isa >= ISA_LEVEL_AVX3) - dataB = _mm256_add_epi8(oDataB, _mm256_set1_epi8(-42)); - - uint64_t maskEqShift1 = (maskEq << 1) + escFirst; - if(LIKELIHOOD(0.0001, (mask & maskEqShift1) != 0)) { - maskEq = fix_eqMask(maskEq, maskEqShift1); - mask &= ~(uint64_t)escFirst; - escFirst = maskEq>>63; - // next, eliminate anything following a `=` from the special char mask; this eliminates cases of `=\r` so that they aren't removed - maskEq <<= 1; - mask &= ~maskEq; - - // unescape chars following `=` -#if defined(__AVX512VL__) && defined(__AVX512BW__) - if(use_isa >= ISA_LEVEL_AVX3) { - // GCC < 7 seems to generate rubbish assembly for this - dataA = _mm256_mask_add_epi8( - dataA, - (__mmask32)maskEq, - dataA, - _mm256_set1_epi8(-64) - ); - dataB = _mm256_mask_add_epi8( - dataB, - (__mmask32)(maskEq>>32), - dataB, - _mm256_set1_epi8(-64) - ); - } else -#endif - { - // convert maskEq into vector form (i.e. reverse pmovmskb) -#ifdef PLATFORM_AMD64 - __m256i vMaskEq = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(maskEq)); -#else - __m256i vMaskEq = _mm256_permute4x64_epi64(_mm256_insert_epi32( - _mm256_set_epi32(0,0,0,0, 0,0,0, maskEq & 0xffffffff), - maskEq >> 32, - 1 - ), 0); -#endif - __m256i vMaskEqA = _mm256_shuffle_epi8(vMaskEq, _mm256_set_epi32( - 0x03030303, 0x03030303, 0x02020202, 0x02020202, - 0x01010101, 0x01010101, 0x00000000, 0x00000000 - )); - __m256i vMaskEqB = _mm256_shuffle_epi8(vMaskEq, _mm256_set_epi32( - 0x07070707, 0x07070707, 0x06060606, 0x06060606, - 0x05050505, 0x05050505, 0x04040404, 0x04040404 - )); - vMaskEqA = _mm256_cmpeq_epi8( - _mm256_and_si256(vMaskEqA, _mm256_set1_epi64x(0x8040201008040201ULL)), - _mm256_set1_epi64x(0x8040201008040201ULL) - ); - vMaskEqB = _mm256_cmpeq_epi8( - _mm256_and_si256(vMaskEqB, _mm256_set1_epi64x(0x8040201008040201ULL)), - _mm256_set1_epi64x(0x8040201008040201ULL) - ); - dataA = _mm256_add_epi8(oDataA, _mm256_blendv_epi8(yencOffset, _mm256_set1_epi8(-42-64), vMaskEqA)); - dataB = _mm256_add_epi8(oDataB, _mm256_blendv_epi8(_mm256_set1_epi8(-42), _mm256_set1_epi8(-42-64), vMaskEqB)); - } - } else { - escFirst = (maskEq >> 63); - -#if defined(__AVX512VL__) && defined(__AVX512BW__) - if(use_isa >= ISA_LEVEL_AVX3) { - dataA = _mm256_mask_add_epi8( - dataA, - (__mmask32)(maskEq << 1), - dataA, - _mm256_set1_epi8(-64) - ); - dataB = _mm256_mask_add_epi8( - dataB, - (__mmask32)(maskEq >> 31), - dataB, - _mm256_set1_epi8(-64) - ); - } else -#endif - { - // << 1 byte - cmpEqA = _mm256_alignr_epi8(cmpEqA, _mm256_inserti128_si256( - _mm256_set1_epi8('='), _mm256_castsi256_si128(cmpEqA), 1 - ), 15); - cmpEqB = _mm256_cmpeq_epi8(_mm256_set1_epi8('='), _mm256_loadu_si256((__m256i *)(src+i-1) + 1)); - dataA = _mm256_add_epi8( - oDataA, - _mm256_blendv_epi8( - yencOffset, - _mm256_set1_epi8(-42-64), - cmpEqA - ) - ); - dataB = _mm256_add_epi8( - oDataB, - _mm256_blendv_epi8( - _mm256_set1_epi8(-42), - _mm256_set1_epi8(-42-64), - cmpEqB - ) - ); - } - } - // subtract 64 from first element if escFirst == 1 -#if defined(__AVX512VL__) && defined(__AVX512BW__) - if(use_isa >= ISA_LEVEL_AVX3) { - yencOffset = _mm256_mask_add_epi8(_mm256_set1_epi8(-42), (__mmask32)escFirst, _mm256_set1_epi8(-42), _mm256_set1_epi8(-64)); - } else -#endif - { - yencOffset = _mm256_xor_si256(_mm256_set1_epi8(-42), zext128_256( - _mm_slli_epi16(_mm_cvtsi32_si128((int)escFirst), 6) - )); - } - - // all that's left is to 'compress' the data (skip over masked chars) -#if defined(__AVX512VBMI2__) && defined(__AVX512VL__) - if(use_isa >= ISA_LEVEL_VBMI2) { - COMPRESS_STORE(p, KNOT32(mask), dataA); - p -= popcnt32(mask & 0xffffffff); - COMPRESS_STORE((p + XMM_SIZE*2), KNOT32(mask>>32), dataB); - p += XMM_SIZE*4 - popcnt32(mask >> 32); - } else -#endif - { - // lookup compress masks and shuffle - __m256i shuf = _mm256_inserti128_si256( - _mm256_castsi128_si256(_mm_load_si128((__m128i*)(lookups->compact + (mask & 0x7fff)))), - *(__m128i*)((char*)lookups->compact + ((mask >> 12) & 0x7fff0)), - 1 - ); - dataA = _mm256_shuffle_epi8(dataA, shuf); - - _mm_storeu_si128((__m128i*)p, _mm256_castsi256_si128(dataA)); - // increment output position - p -= popcnt32(mask & 0xffff); - - _mm_storeu_si128((__m128i*)(p + XMM_SIZE), _mm256_extracti128_si256(dataA, 1)); - p -= popcnt32(mask & 0xffff0000); - -#ifdef PLATFORM_AMD64 - mask >>= 28; - shuf = _mm256_inserti128_si256( - _mm256_castsi128_si256(_mm_load_si128((__m128i*)((char*)lookups->compact + (mask & 0x7fff0)))), - *(__m128i*)((char*)lookups->compact + ((mask >> 16) & 0x7fff0)), - 1 - ); - dataB = _mm256_shuffle_epi8(dataB, shuf); - - _mm_storeu_si128((__m128i*)(p + XMM_SIZE*2), _mm256_castsi256_si128(dataB)); - p -= popcnt32(mask & 0xffff0); - - _mm_storeu_si128((__m128i*)(p + XMM_SIZE*3), _mm256_extracti128_si256(dataB, 1)); - p -= popcnt32((unsigned int)(mask >> 20)); -#else - mask >>= 32; - shuf = _mm256_inserti128_si256( - _mm256_castsi128_si256(_mm_load_si128((__m128i*)(lookups->compact + (mask & 0x7fff)))), - *(__m128i*)((char*)lookups->compact + ((mask >> 12) & 0x7fff0)), - 1 - ); - dataB = _mm256_shuffle_epi8(dataB, shuf); - - _mm_storeu_si128((__m128i*)(p + XMM_SIZE*2), _mm256_castsi256_si128(dataB)); - p -= popcnt32(mask & 0xffff); - - _mm_storeu_si128((__m128i*)(p + XMM_SIZE*3), _mm256_extracti128_si256(dataB, 1)); - p -= popcnt32(mask & 0xffff0000); -#endif - p += XMM_SIZE*4; - } - } else { - if(use_isa < ISA_LEVEL_AVX3) - dataA = _mm256_add_epi8(oDataA, yencOffset); - dataB = _mm256_add_epi8(oDataB, _mm256_set1_epi8(-42)); - - _mm256_storeu_si256((__m256i*)p, dataA); - _mm256_storeu_si256((__m256i*)p + 1, dataB); - p += sizeof(__m256i)*2; - escFirst = 0; - yencOffset = _mm256_set1_epi8(-42); - } - } - _escFirst = (unsigned char)escFirst; - _mm256_zeroupper(); -} -} // namespace -#endif diff --git a/rapidyenc/rapidyenc/src/decoder_common.h b/rapidyenc/rapidyenc/src/decoder_common.h deleted file mode 100644 index 0febde1..0000000 --- a/rapidyenc/rapidyenc/src/decoder_common.h +++ /dev/null @@ -1,220 +0,0 @@ -#include "decoder.h" - -namespace RapidYenc { - void decoder_set_sse2_funcs(); - void decoder_set_ssse3_funcs(); - void decoder_set_avx_funcs(); - void decoder_set_avx2_funcs(); - void decoder_set_vbmi2_funcs(); - extern const bool decoder_has_avx10; - void decoder_set_neon_funcs(); - void decoder_set_rvv_funcs(); - - template - YencDecoderEnd do_decode_scalar(const unsigned char** src, unsigned char** dest, size_t len, YencDecoderState* state); -} - - -#if defined(PLATFORM_ARM) && !defined(__aarch64__) -#define YENC_DEC_USE_THINTABLE 1 -#endif - -// TODO: need to support max output length somehow - - - -template -static inline RapidYenc::YencDecoderEnd _do_decode_simd(size_t width, const unsigned char** src, unsigned char** dest, size_t len, RapidYenc::YencDecoderState* state) { - using namespace RapidYenc; - - if(len <= width*2) return do_decode_scalar(src, dest, len, state); - - YencDecoderState tState = YDEC_STATE_CRLF; - YencDecoderState* pState = state ? state : &tState; - if((uintptr_t)(*src) & ((width-1))) { - // find source memory alignment - unsigned char* aSrc = (unsigned char*)(((uintptr_t)(*src) + (width-1)) & ~(width-1)); - int amount = (int)(aSrc - *src); - len -= amount; - YencDecoderEnd ended = do_decode_scalar(src, dest, amount, pState); - if(ended) return ended; - } - - size_t lenBuffer = width -1; - if(searchEnd) lenBuffer += 3 + (isRaw?1:0); - else if(isRaw) lenBuffer += 2; - - if(len > lenBuffer) { - unsigned char *p = *dest; // destination pointer - unsigned char escFirst = 0; // input character; first char needs escaping - uint16_t nextMask = 0; - // handle finicky case of special sequences straddled across initial boundary - switch(*pState) { - case YDEC_STATE_CRLF: - if(isRaw && **src == '.') { - nextMask = 1; - if(searchEnd && *(uint16_t*)(*src +1) == UINT16_PACK('\r','\n')) { - (*src) += 3; - *pState = YDEC_STATE_CRLF; - return YDEC_END_ARTICLE; - } - if(searchEnd && *(uint16_t*)(*src +1) == UINT16_PACK('=','y')) { - (*src) += 3; - *pState = YDEC_STATE_NONE; - return YDEC_END_CONTROL; - } - } - else if(searchEnd && *(uint16_t*)(*src) == UINT16_PACK('=','y')) { - (*src) += 2; - *pState = YDEC_STATE_NONE; - return YDEC_END_CONTROL; - } - break; - case YDEC_STATE_CR: - if(isRaw && *(uint16_t*)(*src) == UINT16_PACK('\n','.')) { - nextMask = 2; - if(searchEnd && *(uint16_t*)(*src +2) == UINT16_PACK('\r','\n')) { - (*src) += 4; - *pState = YDEC_STATE_CRLF; - return YDEC_END_ARTICLE; - } - if(searchEnd && *(uint16_t*)(*src +2) == UINT16_PACK('=','y')) { - (*src) += 4; - *pState = YDEC_STATE_NONE; - return YDEC_END_CONTROL; - } - } - else if(searchEnd && (*(uint32_t*)(*src) & 0xffffff) == UINT32_PACK('\n','=','y',0)) { - (*src) += 3; - *pState = YDEC_STATE_NONE; - return YDEC_END_CONTROL; - } - break; - case YDEC_STATE_CRLFDT: - if(searchEnd && isRaw && *(uint16_t*)(*src) == UINT16_PACK('\r','\n')) { - (*src) += 2; - *pState = YDEC_STATE_CRLF; - return YDEC_END_ARTICLE; - } - if(searchEnd && isRaw && *(uint16_t*)(*src) == UINT16_PACK('=','y')) { - (*src) += 2; - *pState = YDEC_STATE_NONE; - return YDEC_END_CONTROL; - } - break; - case YDEC_STATE_CRLFDTCR: - if(searchEnd && isRaw && **src == '\n') { - (*src) += 1; - *pState = YDEC_STATE_CRLF; - return YDEC_END_ARTICLE; - } - break; - case YDEC_STATE_CRLFEQ: - if(searchEnd && **src == 'y') { - (*src) += 1; - *pState = YDEC_STATE_NONE; - return YDEC_END_CONTROL; - } - break; - default: break; // silence compiler warning - } - escFirst = (*pState == YDEC_STATE_EQ || *pState == YDEC_STATE_CRLFEQ); - - // our algorithm may perform an aligned load on the next part, of which we consider 2 bytes (for \r\n. sequence checking) - long dLen = (long)(len - lenBuffer); - dLen = (dLen + (width-1)) & ~(width-1); - - kernel((const uint8_t*)(*src) + dLen, dLen, p, escFirst, nextMask); - - if(escFirst) *pState = YDEC_STATE_EQ; // escape next character - else if(nextMask == 1) *pState = YDEC_STATE_CRLF; // next character is '.', where previous two were \r\n - else if(nextMask == 2) *pState = YDEC_STATE_CR; // next characters are '\n.', previous is \r - else *pState = YDEC_STATE_NONE; - - *src += dLen; - len -= dLen; - *dest = p; - } - - // end alignment - if(len) - return do_decode_scalar(src, dest, len, pState); - /** for debugging: ensure that the SIMD routine doesn't exit early - if(len && !searchEnd) { - const uint8_t* s = *src; - unsigned char* p = *dest; - int ended = do_decode_scalar(src, dest, len, pState); - if(*src - s > width*2) { - // this shouldn't happen, corrupt some data to fail the test - while(p < *dest) - *p++ = 0; - } - return ended; - } - */ - return YDEC_END_NONE; -} - -template -static RapidYenc::YencDecoderEnd do_decode_simd(const unsigned char** src, unsigned char** dest, size_t len, RapidYenc::YencDecoderState* state) { - return _do_decode_simd(width, src, dest, len, state); -} -template -static RapidYenc::YencDecoderEnd do_decode_simd(const unsigned char** src, unsigned char** dest, size_t len, RapidYenc::YencDecoderState* state) { - return _do_decode_simd(getWidth(), src, dest, len, state); -} - - -#if defined(PLATFORM_X86) || defined(PLATFORM_ARM) -namespace RapidYenc { - void decoder_init_lut(void* compactLUT); -} -#endif - -template -static inline void decoder_set_nextMask(const uint8_t* src, size_t len, uint16_t& nextMask) { - if(isRaw) { - if(len != 0) { // have to gone through at least one loop cycle - if(src[-2] == '\r' && src[-1] == '\n' && src[0] == '.') - nextMask = 1; - else if(src[-1] == '\r' && src[0] == '\n' && src[1] == '.') - nextMask = 2; - else - nextMask = 0; - } - } else - nextMask = 0; -} - -// without backtracking -template -static inline uint16_t decoder_set_nextMask(const uint8_t* src, unsigned mask) { - if(isRaw) { - if(src[0] == '.') - return mask & 1; - if(src[1] == '.') - return mask & 2; - } - return 0; -} - -// resolve invalid sequences of = to deal with cases like '====' -// bit hack inspired from simdjson: https://youtu.be/wlvKAT7SZIQ?t=33m38s -template -static inline T fix_eqMask(T mask, T maskShift1) { - // isolate the start of each consecutive bit group (e.g. 01011101 -> 01000101) - T start = mask & ~maskShift1; - - // this strategy works by firstly separating groups that start on even/odd bits - // generally, it doesn't matter which one (even/odd) we pick, but clearing even groups specifically allows the escFirst bit in maskShift1 to work - // (this is because the start of the escFirst group is at index -1, an odd bit, but we can't clear it due to being < 0, so we just retain all odd groups instead) - - const T even = (T)0x5555555555555555; // every even bit (01010101...) - - // obtain groups which start on an odd bit (clear groups that start on an even bit, but this leaves an unwanted trailing bit) - T oddGroups = mask + (start & even); - - // clear even bits in odd groups, whilst conversely preserving even bits in even groups - // the `& mask` also conveniently gets rid of unwanted trailing bits - return (oddGroups ^ even) & mask; -} diff --git a/rapidyenc/rapidyenc/src/decoder_neon.cc b/rapidyenc/rapidyenc/src/decoder_neon.cc deleted file mode 100644 index ac325c5..0000000 --- a/rapidyenc/rapidyenc/src/decoder_neon.cc +++ /dev/null @@ -1,462 +0,0 @@ -#include "common.h" -#include "decoder_common.h" - -#ifdef __ARM_NEON - - -#if defined(_MSC_VER) && !defined(__clang__) -# define vld1_u8_align(p, a) vld1_u8_ex(p, a*8) -# define vld1q_u8_align(p, a) vld1q_u8_ex(p, a*8) -#elif defined(__GNUC__) -# define vld1_u8_align(p, n) vld1_u8((uint8_t*)__builtin_assume_aligned(p, n)) -# define vld1q_u8_align(p, n) vld1q_u8((uint8_t*)__builtin_assume_aligned(p, n)) -#else -# define vld1_u8_align(p, n) vld1_u8(p) -# define vld1q_u8_align(p, n) vld1q_u8(p) -#endif - - -// for compilers that lack these functions (Clang armv7 9-12 seems to have issues with multi-vector loads) -#if (defined(__clang__) && (defined(__aarch64__) || __clang_major__<9 || __clang_major__>12)) || (defined(__GNUC__) && (defined(__aarch64__) && __GNUC__ >= 8)) -# define vld1q_u8_x2_align(p, n) vld1q_u8_x2((uint8_t*)__builtin_assume_aligned(p, n)) -#else -# define vld1q_u8_x2_align(p, n) vcreate2_u8(vld1q_u8_align(p, (n)/2), vld1q_u8_align((p)+16, (n)/2)) -#endif -// Clang wrongly assumes alignment on vld1q_u8_x2, and ARMv7 GCC doesn't support the function, so effectively, it can only be used in ARMv8 compilers -#if defined(__aarch64__) && (defined(__clang__) || HEDLEY_GCC_VERSION_CHECK(8,5,0)) -# define vst1q_u8_x2_unaligned vst1q_u8_x2 -#else -static HEDLEY_ALWAYS_INLINE void vst1q_u8_x2_unaligned(uint8_t* p, uint8x16x2_t data) { - vst1q_u8(p, data.val[0]); - vst1q_u8(p+16, data.val[1]); -} -#endif - -#ifdef YENC_DEC_USE_THINTABLE -static uint64_t ALIGN_TO(8, compactLUT[256]); -#else -# pragma pack(16) -static struct { char bytes[16]; } ALIGN_TO(16, compactLUT[32768]); -# pragma pack() -#endif - - - -static bool neon_vect_is_nonzero(uint8x16_t v) { -# ifdef __aarch64__ - return !!(vget_lane_u64(vreinterpret_u64_u32(vqmovn_u64(vreinterpretq_u64_u8(v))), 0)); -# else - uint32x4_t tmp1 = vreinterpretq_u32_u8(v); - uint32x2_t tmp2 = vorr_u32(vget_low_u32(tmp1), vget_high_u32(tmp1)); - return !!(vget_lane_u32(vpmax_u32(tmp2, tmp2), 0)); -# endif -} - - -namespace RapidYenc { - -template -HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* src, long& len, unsigned char*& p, unsigned char& escFirst, uint16_t& nextMask) { - HEDLEY_ASSUME(escFirst == 0 || escFirst == 1); - HEDLEY_ASSUME(nextMask == 0 || nextMask == 1 || nextMask == 2); - uint8x16_t yencOffset = escFirst ? vmakeq_u8(42+64,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42) : vdupq_n_u8(42); -#ifdef __aarch64__ - uint8x16_t nextMaskMix = vdupq_n_u8(0); - if(nextMask == 1) - nextMaskMix = vsetq_lane_u8(1, nextMaskMix, 0); - if(nextMask == 2) - nextMaskMix = vsetq_lane_u8(2, nextMaskMix, 1); -#else - uint8x16_t lfCompare = vdupq_n_u8('\n'); - if(isRaw) { - if(nextMask == 1) - lfCompare = vsetq_lane_u8('.', lfCompare, 0); - if(nextMask == 2) - lfCompare = vsetq_lane_u8('.', lfCompare, 1); - } -#endif - - decoder_set_nextMask(src, len, nextMask); - - long i; - for(i = -len; i; i += sizeof(uint8x16_t)*2) { - uint8x16x2_t data = vld1q_u8_x2_align(src+i, 32); - uint8x16_t dataA = data.val[0]; - uint8x16_t dataB = data.val[1]; - - // search for special chars - uint8x16_t cmpEqA = vceqq_u8(dataA, vdupq_n_u8('=')), - cmpEqB = vceqq_u8(dataB, vdupq_n_u8('=')), -#ifdef __aarch64__ - cmpA = vqtbx1q_u8( - cmpEqA, - // \n \r - vmakeq_u8(0,0,0,0,0,0,0,0,0,0,255,0,0,255,0,0), - dataA - ), - cmpB = vqtbx1q_u8( - cmpEqB, - vmakeq_u8(0,0,0,0,0,0,0,0,0,0,255,0,0,255,0,0), - dataB - ); - if(isRaw) cmpA = vorrq_u8(cmpA, nextMaskMix); -#else - cmpCrA = vceqq_u8(dataA, vdupq_n_u8('\r')), - cmpCrB = vceqq_u8(dataB, vdupq_n_u8('\r')), - cmpA = vorrq_u8( - vorrq_u8( - cmpCrA, - vceqq_u8(dataA, lfCompare) - ), - cmpEqA - ), - cmpB = vorrq_u8( - vorrq_u8( - cmpCrB, - vceqq_u8(dataB, vdupq_n_u8('\n')) - ), - cmpEqB - ); -#endif - - -#ifdef __aarch64__ - if (LIKELIHOOD(0.42 /*guess*/, neon_vect_is_nonzero(vorrq_u8(cmpA, cmpB)))) { - cmpA = vandq_u8(cmpA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); - cmpB = vandq_u8(cmpB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); - uint8x16_t cmpMerge = vpaddq_u8(cmpA, cmpB); - uint8x16_t cmpEqMerge = vpaddq_u8( - vandq_u8(cmpEqA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)), - vandq_u8(cmpEqB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)) - ); - - uint8x16_t cmpCombined = vpaddq_u8(cmpMerge, cmpEqMerge); - cmpCombined = vpaddq_u8(cmpCombined, cmpCombined); - uint8x8_t cmpPacked = vget_low_u8(cmpCombined); - uint32_t mask = vgetq_lane_u32(vreinterpretq_u32_u8(cmpCombined), 0); - uint32_t maskEq = vgetq_lane_u32(vreinterpretq_u32_u8(cmpCombined), 1); -#else - cmpA = vandq_u8(cmpA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); - cmpB = vandq_u8(cmpB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); - // no vpaddq_u8 in ARMv7, so need extra 64-bit VPADD - uint8x8_t cmpPacked = vpadd_u8( - vpadd_u8( - vget_low_u8(cmpA), vget_high_u8(cmpA) - ), - vpadd_u8( - vget_low_u8(cmpB), vget_high_u8(cmpB) - ) - ); - cmpPacked = vpadd_u8(cmpPacked, cmpPacked); - uint32_t mask = vget_lane_u32(vreinterpret_u32_u8(cmpPacked), 0); - if(LIKELIHOOD(0.42, mask != 0)) { - uint8x16_t cmpEqMaskedA = vandq_u8(cmpEqA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); - uint8x16_t cmpEqMaskedB = vandq_u8(cmpEqB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); - uint8x8_t cmpEqPacked = vpadd_u8( - vpadd_u8( - vget_low_u8(cmpEqMaskedA), vget_high_u8(cmpEqMaskedA) - ), - vpadd_u8( - vget_low_u8(cmpEqMaskedB), vget_high_u8(cmpEqMaskedB) - ) - ); - cmpEqPacked = vpadd_u8(cmpEqPacked, cmpEqPacked); - uint32_t maskEq = vget_lane_u32(vreinterpret_u32_u8(cmpEqPacked), 0); -#endif - - // handle \r\n. sequences - // RFC3977 requires the first dot on a line to be stripped, due to dot-stuffing - if((isRaw || searchEnd) && LIKELIHOOD(0.15, mask != maskEq)) { - // vext seems to be a cheap operation on ARM, relative to loads, so only avoid it if there's only one load (isRaw only) - uint8x16_t tmpData2, nextData; - if(isRaw && !searchEnd) { - tmpData2 = vld1q_u8(src+i + 2 + sizeof(uint8x16_t)); - } else { - nextData = vld1q_u8_align(src+i + sizeof(uint8x16_t)*2, 16); // only 32-bits needed, but there doesn't appear a nice way to do this via intrinsics: https://stackoverflow.com/questions/46910799/arm-neon-intrinsics-convert-d-64-bit-register-to-low-half-of-q-128-bit-regis - tmpData2 = vextq_u8(dataB, nextData, 2); - } -#ifdef __aarch64__ - uint8x16_t cmpCrA = vceqq_u8(dataA, vdupq_n_u8('\r')); - uint8x16_t cmpCrB = vceqq_u8(dataB, vdupq_n_u8('\r')); -# define NEXT_DATA(n) vextq_u8(dataB, nextData, n) -#else -// on ARMv7, prefer loading over VEXT to avoid holding onto nextData reference; this reduces register spills. Shouldn't be an issue on ARMv8 due to 32x 128-bit registers -# define NEXT_DATA(n) vld1q_u8(src+i + n+sizeof(uint8x16_t)) -#endif - uint8x16_t match2EqA, match2Cr_DotA; - uint8x16_t match2EqB, match2Cr_DotB; - if(searchEnd) { - match2EqB = vceqq_u8(tmpData2, vdupq_n_u8('=')); - } - if(isRaw) { - match2Cr_DotA = vandq_u8(cmpCrA, vceqq_u8(vextq_u8(dataA, dataB, 2), vdupq_n_u8('.'))); - match2Cr_DotB = vandq_u8(cmpCrB, vceqq_u8(tmpData2, vdupq_n_u8('.'))); - } - - // find patterns of \r_. - if(isRaw && LIKELIHOOD(0.001, neon_vect_is_nonzero( - vorrq_u8(match2Cr_DotA, match2Cr_DotB) - ))) { - uint8x16_t match1LfA = vceqq_u8(vextq_u8(dataA, dataB, 1), vdupq_n_u8('\n')); - uint8x16_t match1LfB; - if(searchEnd) - match1LfB = vceqq_u8(NEXT_DATA(1), vdupq_n_u8('\n')); - else - match1LfB = vceqq_u8(vld1q_u8(src+i + 1+sizeof(uint8x16_t)), vdupq_n_u8('\n')); - // merge matches of \r_. with those for \n - uint8x16_t match2NlDotA = vandq_u8(match2Cr_DotA, match1LfA); - uint8x16_t match2NlDotB = vandq_u8(match2Cr_DotB, match1LfB); - if(searchEnd) { - uint8x16_t match1NlA = vandq_u8(match1LfA, cmpCrA); - uint8x16_t match1NlB = vandq_u8(match1LfB, cmpCrB); - - uint8x16_t tmpData3 = NEXT_DATA(3); - uint8x16_t tmpData4 = NEXT_DATA(4); - // match instances of \r\n.\r\n and \r\n.=y - uint8x16_t match3CrB = vceqq_u8(tmpData3, vdupq_n_u8('\r')); - uint8x16_t match4LfB = vceqq_u8(tmpData4, vdupq_n_u8('\n')); - uint8x16_t match4Nl = vbslq_u8(vdupq_n_u8('\r'), // exact VBSL vector doesn't matter, so reuse the '\r' vector - vextq_u8(match1NlA, match1NlB, 3), - vandq_u8(match3CrB, match4LfB) - ); - uint8x16_t match4EqY = vbslq_u8(vdupq_n_u8('\r'), - // match =y - vreinterpretq_u8_u16(vceqq_u16(vreinterpretq_u16_u8(vextq_u8(dataA, dataB, 4)), vdupq_n_u16(0x793d))), - vreinterpretq_u8_u16(vceqq_u16(vreinterpretq_u16_u8(tmpData4), vdupq_n_u16(0x793d))) - ); - - match2EqA = vextq_u8(cmpEqA, cmpEqB, 2); - uint8x16_t match3EqY = vbslq_u8(vdupq_n_u8('\r'), - vandq_u8( - vceqq_u8(vextq_u8(dataA, dataB, 3), vdupq_n_u8('y')), - match2EqA - ), vandq_u8( - vceqq_u8(tmpData3, vdupq_n_u8('y')), - match2EqB - ) - ); - // merge \r\n and =y matches for tmpData4 - uint8x16_t match4End = vorrq_u8( - match4Nl, - vreinterpretq_u8_u16(vsriq_n_u16(vreinterpretq_u16_u8(match4EqY), vreinterpretq_u16_u8(match3EqY), 8)) - ); - // merge with \r\n. - uint8x16_t match2NlDot = vbslq_u8(vdupq_n_u8('\r'), match2NlDotA, match2NlDotB); - match4End = vandq_u8(match4End, match2NlDot); - // match \r\n=y - uint8x16_t match1Nl = vbslq_u8(vdupq_n_u8('\r'), match1NlA, match1NlB); - uint8x16_t match3End = vandq_u8(match3EqY, match1Nl); - // combine match sequences - if(LIKELIHOOD(0.001, neon_vect_is_nonzero(vorrq_u8(match4End, match3End)))) { - // terminator found - // there's probably faster ways to do this, but reverting to scalar code should be good enough - len += i; - nextMask = decoder_set_nextMask(src+i, mask); - break; - } - } -#ifdef __aarch64__ - uint8x16_t match2NlDotBMasked = vandq_u8(match2NlDotB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); - uint8x16_t mergeKillDots = vpaddq_u8( - vandq_u8(match2NlDotA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)), - match2NlDotBMasked - ); - uint8x8_t mergeKillDots2 = vget_low_u8(vpaddq_u8(mergeKillDots, mergeKillDots)); -#else - uint8x16_t match2NlDotMaskedA = vandq_u8(match2NlDotA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); - uint8x16_t match2NlDotMaskedB = vandq_u8(match2NlDotB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); - uint8x8_t mergeKillDots2 = vpadd_u8( - vpadd_u8( - vget_low_u8(match2NlDotMaskedA), vget_high_u8(match2NlDotMaskedA) - ), - vpadd_u8( - vget_low_u8(match2NlDotMaskedB), vget_high_u8(match2NlDotMaskedB) - ) - ); -#endif - mergeKillDots2 = vpadd_u8(mergeKillDots2, mergeKillDots2); - uint32x2_t mergeKillDotsShifted = vshl_n_u32(vreinterpret_u32_u8(mergeKillDots2), 2); - mask |= vget_lane_u32(mergeKillDotsShifted, 0); - cmpPacked = vorr_u8(cmpPacked, vreinterpret_u8_u32(mergeKillDotsShifted)); -#ifdef __aarch64__ - nextMaskMix = vextq_u8(match2NlDotB, vdupq_n_u8(0), 14); -#else - lfCompare = vcombine_u8(vbsl_u8( - vext_u8(vget_high_u8(match2NlDotB), vdup_n_u8('\n'), 6), - vdup_n_u8('.'), - vget_high_u8(lfCompare) - ), vget_high_u8(lfCompare)); -#endif - } else if(searchEnd) { - match2EqA = vextq_u8(cmpEqA, cmpEqB, 2); - uint8x16_t match3EqYA = vandq_u8(match2EqA, vceqq_u8(vextq_u8(dataA, dataB, 3), vdupq_n_u8('y'))); - uint8x16_t match3EqYB = vandq_u8(match2EqB, vceqq_u8(NEXT_DATA(3), vdupq_n_u8('y'))); - if(LIKELIHOOD(0.001, neon_vect_is_nonzero(vorrq_u8( - match3EqYA, match3EqYB - )))) { - uint8x16_t match1LfA = vceqq_u8(vextq_u8(dataA, dataB, 1), vdupq_n_u8('\n')); - uint8x16_t match1LfB = vceqq_u8(NEXT_DATA(1), vdupq_n_u8('\n')); - uint8x16_t matchEnd = vorrq_u8( - vandq_u8(match3EqYA, vandq_u8(match1LfA, cmpCrA)), - vandq_u8(match3EqYB, vandq_u8(match1LfB, cmpCrB)) - ); - if(LIKELIHOOD(0.001, neon_vect_is_nonzero(matchEnd))) { - len += i; - nextMask = decoder_set_nextMask(src+i, mask); - break; - } - } -#undef NEXT_DATA - if(isRaw) -#ifdef __aarch64__ - nextMaskMix = vdupq_n_u8(0); -#else - lfCompare = vcombine_u8(vget_high_u8(lfCompare), vget_high_u8(lfCompare)); -#endif - } else if(isRaw) // no \r_. found -#ifdef __aarch64__ - nextMaskMix = vdupq_n_u8(0); -#else - lfCompare = vcombine_u8(vget_high_u8(lfCompare), vget_high_u8(lfCompare)); -#endif - } - - // a spec compliant encoder should never generate sequences: ==, =\n and =\r, but we'll handle them to be spec compliant - // the yEnc specification requires any character following = to be unescaped, not skipped over, so we'll deal with that - // firstly, check for invalid sequences of = (we assume that these are rare, as a spec compliant yEnc encoder should not generate these) - uint32_t maskEqShift1 = (maskEq << 1) | escFirst; - if(LIKELIHOOD(0.0001, (mask & maskEqShift1) != 0)) { - maskEq = fix_eqMask(maskEq, maskEqShift1); - - unsigned char nextEscFirst = maskEq>>31; - // next, eliminate anything following a `=` from the special char mask; this eliminates cases of `=\r` so that they aren't removed - maskEq = (maskEq<<1) | escFirst; - mask &= ~maskEq; - escFirst = nextEscFirst; - - // unescape chars following `=` - uint8x8_t maskEqTemp = vreinterpret_u8_u32(vmov_n_u32(maskEq)); - cmpPacked = vbic_u8(cmpPacked, maskEqTemp); // `mask &= ~maskEq` in vector form -#ifdef __aarch64__ - uint8x16_t vMaskEqA = vqtbl1q_u8( - vcombine_u8(maskEqTemp, vdup_n_u8(0)), - vmakeq_u8(0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1) - ); - uint8x16_t vMaskEqB = vqtbl1q_u8( - vcombine_u8(maskEqTemp, vdup_n_u8(0)), - vmakeq_u8(2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3) - ); -#else - uint8x16_t vMaskEqA = vcombine_u8( - vdup_lane_u8(maskEqTemp, 0), - vdup_lane_u8(maskEqTemp, 1) - ); - uint8x16_t vMaskEqB = vcombine_u8( - vdup_lane_u8(maskEqTemp, 2), - vdup_lane_u8(maskEqTemp, 3) - ); -#endif - vMaskEqA = vtstq_u8(vMaskEqA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); - vMaskEqB = vtstq_u8(vMaskEqB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); - - dataA = vsubq_u8( - dataA, - vbslq_u8(vMaskEqA, vdupq_n_u8(64+42), vdupq_n_u8(42)) - ); - dataB = vsubq_u8( - dataB, - vbslq_u8(vMaskEqB, vdupq_n_u8(64+42), vdupq_n_u8(42)) - ); - } else { - // no invalid = sequences found - we can cut out some things from above - // this code path is a shortened version of above; it's here because it's faster, and what we'll be dealing with most of the time - escFirst = (maskEq >> 31); - - dataA = vsubq_u8( - dataA, - vbslq_u8( - vextq_u8(vdupq_n_u8(42), cmpEqA, 15), - vdupq_n_u8(64+42), - yencOffset - ) - ); - dataB = vsubq_u8( - dataB, - vbslq_u8( - vextq_u8(cmpEqA, cmpEqB, 15), - vdupq_n_u8(64+42), - vdupq_n_u8(42) - ) - ); - } - yencOffset = vsetq_lane_u8((escFirst << 6) | 42, yencOffset, 0); - - // all that's left is to 'compress' the data (skip over masked chars) - uint32_t counts = 0x08080808 - vget_lane_u32(vreinterpret_u32_u8(vcnt_u8(cmpPacked)), 0); -#ifdef __aarch64__ - counts += counts >> 8; - vst1q_u8(p, vqtbl1q_u8( - dataA, - vld1q_u8_align((uint8_t*)(compactLUT + (mask&0x7fff)), 16) - )); - p += counts & 0xff; - mask >>= 16; - vst1q_u8(p, vqtbl1q_u8( - dataB, - vld1q_u8_align((uint8_t*)(compactLUT + (mask&0x7fff)), 16) - )); - p += (counts>>16) & 0xff; -#else - // lookup compress masks and shuffle - vst1_u8(p, vtbl1_u8( - vget_low_u8(dataA), - vld1_u8_align((uint8_t*)(compactLUT + (mask&0xff)), 8) - )); - p += counts & 0xff; - mask >>= 8; - vst1_u8(p, vtbl1_u8( - vget_high_u8(dataA), - vld1_u8_align((uint8_t*)(compactLUT + (mask&0xff)), 8) - )); - p += (counts>>8) & 0xff; - mask >>= 8; - vst1_u8(p, vtbl1_u8( - vget_low_u8(dataB), - vld1_u8_align((uint8_t*)(compactLUT + (mask&0xff)), 8) - )); - p += (counts>>16) & 0xff; - mask >>= 8; - vst1_u8(p, vtbl1_u8( - vget_high_u8(dataB), - vld1_u8_align((uint8_t*)(compactLUT + (mask&0xff)), 8) - )); - p += (counts>>24) & 0xff; - -#endif - - } else { - dataA = vsubq_u8(dataA, yencOffset); - dataB = vsubq_u8(dataB, vdupq_n_u8(42)); - vst1q_u8_x2_unaligned(p, vcreate2_u8(dataA, dataB)); - p += sizeof(uint8x16_t)*2; - escFirst = 0; -#ifdef __aarch64__ - yencOffset = vdupq_n_u8(42); -#else - yencOffset = vcombine_u8(vdup_n_u8(42), vget_high_u8(yencOffset)); -#endif - } - } -} -} // namespace - -void RapidYenc::decoder_set_neon_funcs() { - decoder_init_lut(compactLUT); - _do_decode = &do_decode_simd >; - _do_decode_raw = &do_decode_simd >; - _do_decode_end_raw = &do_decode_simd >; - _decode_isa = ISA_LEVEL_NEON; -} -#else -void RapidYenc::decoder_set_neon_funcs() {} -#endif diff --git a/rapidyenc/rapidyenc/src/decoder_neon64.cc b/rapidyenc/rapidyenc/src/decoder_neon64.cc deleted file mode 100644 index 0ab42a1..0000000 --- a/rapidyenc/rapidyenc/src/decoder_neon64.cc +++ /dev/null @@ -1,446 +0,0 @@ -#include "common.h" -#include "decoder_common.h" -#if defined(__ARM_NEON) && defined(__aarch64__) - - -#pragma pack(16) -static struct { char bytes[16]; } ALIGN_TO(16, compactLUT[32768]); -#pragma pack() - - -// AArch64 GCC lacks these functions until 8.5, 9.4 and 10.1 (10.0 unknown) -#if !defined(__clang__) && !defined(_MSC_VER) && (!defined(__aarch64__) || !(HEDLEY_GCC_VERSION_CHECK(9,4,0) || (!HEDLEY_GCC_VERSION_CHECK(9,0,0) && HEDLEY_GCC_VERSION_CHECK(8,5,0)))) -static HEDLEY_ALWAYS_INLINE uint8x16x4_t _vld1q_u8_x4(const uint8_t* p) { - uint8x16x4_t ret; - ret.val[0] = vld1q_u8(p); - ret.val[1] = vld1q_u8(p+16); - ret.val[2] = vld1q_u8(p+32); - ret.val[3] = vld1q_u8(p+48); - return ret; -} -static HEDLEY_ALWAYS_INLINE void _vst1q_u8_x4(uint8_t* p, uint8x16x4_t data) { - vst1q_u8(p, data.val[0]); - vst1q_u8(p+16, data.val[1]); - vst1q_u8(p+32, data.val[2]); - vst1q_u8(p+48, data.val[3]); -} -#else -# define _vld1q_u8_x4 vld1q_u8_x4 -# define _vst1q_u8_x4 vst1q_u8_x4 -#endif - - -static bool neon_vect_is_nonzero(uint8x16_t v) { - return !!(vget_lane_u64(vreinterpret_u64_u32(vqmovn_u64(vreinterpretq_u64_u8(v))), 0)); -} - -static HEDLEY_ALWAYS_INLINE uint8x16_t mergeCompares(uint8x16_t a, uint8x16_t b, uint8x16_t c, uint8x16_t d) { - // constant vectors arbitrarily chosen from ones that can be reused; exact ordering of bits doesn't matter, we just need to mix them in - return vbslq_u8( - vdupq_n_u8('='), - vbslq_u8(vdupq_n_u8('y'), a, b), - vbslq_u8(vdupq_n_u8('y'), c, d) - ); -} - - -namespace RapidYenc { - -template -HEDLEY_ALWAYS_INLINE void do_decode_neon(const uint8_t* src, long& len, unsigned char*& p, unsigned char& escFirst, uint16_t& nextMask) { - HEDLEY_ASSUME(escFirst == 0 || escFirst == 1); - HEDLEY_ASSUME(nextMask == 0 || nextMask == 1 || nextMask == 2); - uint8x16_t nextMaskMix = vdupq_n_u8(0); - if(nextMask == 1) - nextMaskMix = vsetq_lane_u8(1, nextMaskMix, 0); - if(nextMask == 2) - nextMaskMix = vsetq_lane_u8(2, nextMaskMix, 1); - uint8x16_t yencOffset = escFirst ? vmakeq_u8(42+64,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42) : vdupq_n_u8(42); - - decoder_set_nextMask(src, len, nextMask); - - long i; - for(i = -len; i; i += sizeof(uint8x16_t)*4) { - uint8x16x4_t data = _vld1q_u8_x4(src+i); - uint8x16_t dataA = data.val[0]; - uint8x16_t dataB = data.val[1]; - uint8x16_t dataC = data.val[2]; - uint8x16_t dataD = data.val[3]; - - // search for special chars - uint8x16_t cmpEqA = vceqq_u8(dataA, vdupq_n_u8('=')), - cmpEqB = vceqq_u8(dataB, vdupq_n_u8('=')), - cmpEqC = vceqq_u8(dataC, vdupq_n_u8('=')), - cmpEqD = vceqq_u8(dataD, vdupq_n_u8('=')), - cmpA = vqtbx1q_u8( - cmpEqA, - // \n \r - vmakeq_u8(0,0,0,0,0,0,0,0,0,0,255,0,0,255,0,0), - dataA - ), - cmpB = vqtbx1q_u8( - cmpEqB, - vmakeq_u8(0,0,0,0,0,0,0,0,0,0,255,0,0,255,0,0), - dataB - ), - cmpC = vqtbx1q_u8( - cmpEqC, - vmakeq_u8(0,0,0,0,0,0,0,0,0,0,255,0,0,255,0,0), - dataC - ), - cmpD = vqtbx1q_u8( - cmpEqD, - vmakeq_u8(0,0,0,0,0,0,0,0,0,0,255,0,0,255,0,0), - dataD - ); - if(isRaw) cmpA = vorrq_u8(cmpA, nextMaskMix); - - if (LIKELIHOOD(0.42 /*guess*/, neon_vect_is_nonzero(vorrq_u8( - vorrq_u8(cmpA, cmpB), - vorrq_u8(cmpC, cmpD) - )))) { - uint8x16_t cmpMerge = vpaddq_u8( - vpaddq_u8( - vandq_u8(cmpA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)), - vandq_u8(cmpB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)) - ), - vpaddq_u8( - vandq_u8(cmpC, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)), - vandq_u8(cmpD, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)) - ) - ); - uint8x16_t cmpEqMerge = vpaddq_u8( - vpaddq_u8( - vandq_u8(cmpEqA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)), - vandq_u8(cmpEqB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)) - ), - vpaddq_u8( - vandq_u8(cmpEqC, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)), - vandq_u8(cmpEqD, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)) - ) - ); - - uint8x16_t cmpCombined = vpaddq_u8(cmpMerge, cmpEqMerge); - uint64_t mask = vgetq_lane_u64(vreinterpretq_u64_u8(cmpCombined), 0); - uint64_t maskEq = vgetq_lane_u64(vreinterpretq_u64_u8(cmpCombined), 1); - - // handle \r\n. sequences - // RFC3977 requires the first dot on a line to be stripped, due to dot-stuffing - if((isRaw || searchEnd) && LIKELIHOOD(0.15, mask != maskEq)) { - // vext seems to be a cheap operation on ARM, relative to loads, so only avoid it if there's only one load (isRaw only) - uint8x16_t tmpData2, nextData; - if(isRaw && !searchEnd) { - tmpData2 = vld1q_u8(src+i + 2 + sizeof(uint8x16_t)*3); - } else { - nextData = vld1q_u8(src+i + sizeof(uint8x16_t)*4); // only 32-bits needed, but there doesn't appear a nice way to do this via intrinsics: https://stackoverflow.com/questions/46910799/arm-neon-intrinsics-convert-d-64-bit-register-to-low-half-of-q-128-bit-regis - tmpData2 = vextq_u8(dataD, nextData, 2); - } - uint8x16_t cmpCrA = vceqq_u8(dataA, vdupq_n_u8('\r')); - uint8x16_t cmpCrB = vceqq_u8(dataB, vdupq_n_u8('\r')); - uint8x16_t cmpCrC = vceqq_u8(dataC, vdupq_n_u8('\r')); - uint8x16_t cmpCrD = vceqq_u8(dataD, vdupq_n_u8('\r')); - uint8x16_t match2EqA, match2Cr_DotA; - uint8x16_t match2EqB, match2Cr_DotB; - uint8x16_t match2EqC, match2Cr_DotC; - uint8x16_t match2EqD, match2Cr_DotD; - if(searchEnd) { - match2EqD = vceqq_u8(tmpData2, vdupq_n_u8('=')); - } - if(isRaw) { - match2Cr_DotA = vandq_u8(cmpCrA, vceqq_u8(vextq_u8(dataA, dataB, 2), vdupq_n_u8('.'))); - match2Cr_DotB = vandq_u8(cmpCrB, vceqq_u8(vextq_u8(dataB, dataC, 2), vdupq_n_u8('.'))); - match2Cr_DotC = vandq_u8(cmpCrC, vceqq_u8(vextq_u8(dataC, dataD, 2), vdupq_n_u8('.'))); - match2Cr_DotD = vandq_u8(cmpCrD, vceqq_u8(tmpData2, vdupq_n_u8('.'))); - } - - // find patterns of \r_. - if(isRaw && LIKELIHOOD(0.001, neon_vect_is_nonzero(vorrq_u8( - vorrq_u8(match2Cr_DotA, match2Cr_DotB), - vorrq_u8(match2Cr_DotC, match2Cr_DotD) - )))) { - uint8x16_t match1LfA = vceqq_u8(vextq_u8(dataA, dataB, 1), vdupq_n_u8('\n')); - uint8x16_t match1LfB = vceqq_u8(vextq_u8(dataB, dataC, 1), vdupq_n_u8('\n')); - uint8x16_t match1LfC = vceqq_u8(vextq_u8(dataC, dataD, 1), vdupq_n_u8('\n')); - uint8x16_t match1LfD; - if(searchEnd) - match1LfD = vceqq_u8(vextq_u8(dataD, nextData, 1), vdupq_n_u8('\n')); - else - match1LfD = vceqq_u8(vld1q_u8(src+i + 1+sizeof(uint8x16_t)*3), vdupq_n_u8('\n')); - // merge matches of \r_. with those for \n - uint8x16_t match2NlDotA = vandq_u8(match2Cr_DotA, match1LfA); - uint8x16_t match2NlDotB = vandq_u8(match2Cr_DotB, match1LfB); - uint8x16_t match2NlDotC = vandq_u8(match2Cr_DotC, match1LfC); - uint8x16_t match2NlDotD = vandq_u8(match2Cr_DotD, match1LfD); - if(searchEnd) { - uint8x16_t match1NlA = vandq_u8(match1LfA, cmpCrA); - uint8x16_t match1NlB = vandq_u8(match1LfB, cmpCrB); - uint8x16_t match1NlC = vandq_u8(match1LfC, cmpCrC); - uint8x16_t match1NlD = vandq_u8(match1LfD, cmpCrD); - - uint8x16_t tmpData3 = vextq_u8(dataD, nextData, 3); - uint8x16_t tmpData4 = vextq_u8(dataD, nextData, 4); - // match instances of \r\n.\r\n and \r\n.=y - uint8x16_t match3CrD = vceqq_u8(tmpData3, vdupq_n_u8('\r')); - uint8x16_t match4LfD = vceqq_u8(tmpData4, vdupq_n_u8('\n')); - uint8x16_t match4Nl = mergeCompares( - vextq_u8(match1NlA, match1NlB, 3), - vextq_u8(match1NlB, match1NlC, 3), - vextq_u8(match1NlC, match1NlD, 3), - vandq_u8(match3CrD, match4LfD) - ); - uint8x16_t match4EqY = mergeCompares( - // match with =y - vreinterpretq_u8_u16(vceqq_u16(vreinterpretq_u16_u8(vextq_u8(dataA, dataB, 4)), vdupq_n_u16(0x793d))), - vreinterpretq_u8_u16(vceqq_u16(vreinterpretq_u16_u8(vextq_u8(dataB, dataC, 4)), vdupq_n_u16(0x793d))), - vreinterpretq_u8_u16(vceqq_u16(vreinterpretq_u16_u8(vextq_u8(dataC, dataD, 4)), vdupq_n_u16(0x793d))), - vreinterpretq_u8_u16(vceqq_u16(vreinterpretq_u16_u8(tmpData4), vdupq_n_u16(0x793d))) - ); - match2EqA = vextq_u8(cmpEqA, cmpEqB, 2); - match2EqB = vextq_u8(cmpEqB, cmpEqC, 2); - match2EqC = vextq_u8(cmpEqC, cmpEqD, 2); - uint8x16_t match3EqY = mergeCompares( - vandq_u8( - vceqq_u8(vextq_u8(dataA, dataB, 3), vdupq_n_u8('y')), - match2EqA - ), vandq_u8( - vceqq_u8(vextq_u8(dataB, dataC, 3), vdupq_n_u8('y')), - match2EqB - ), vandq_u8( - vceqq_u8(vextq_u8(dataC, dataD, 3), vdupq_n_u8('y')), - match2EqC - ), vandq_u8( - vceqq_u8(tmpData3, vdupq_n_u8('y')), - match2EqD - ) - ); - - // merge \r\n and =y matches for tmpData4 - uint8x16_t match4End = vorrq_u8( - match4Nl, - vreinterpretq_u8_u16(vsriq_n_u16(vreinterpretq_u16_u8(match4EqY), vreinterpretq_u16_u8(match3EqY), 8)) - ); - // merge with \r\n. - uint8x16_t match2NlDot = mergeCompares(match2NlDotA, match2NlDotB, match2NlDotC, match2NlDotD); - match4End = vandq_u8(match4End, match2NlDot); - // match \r\n=y - uint8x16_t match1Nl = mergeCompares(match1NlA, match1NlB, match1NlC, match1NlD); - uint8x16_t match3End = vandq_u8(match3EqY, match1Nl); - // combine match sequences - if(LIKELIHOOD(0.001, neon_vect_is_nonzero(vorrq_u8(match4End, match3End)))) { - // terminator found - // there's probably faster ways to do this, but reverting to scalar code should be good enough - len += i; - nextMask = decoder_set_nextMask(src+i, mask); - break; - } - } - uint8x16_t match2NlDotDMasked = vandq_u8(match2NlDotD, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); - uint8x16_t mergeKillDots = vpaddq_u8( - vpaddq_u8( - vandq_u8(match2NlDotA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)), - vandq_u8(match2NlDotB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)) - ), - vpaddq_u8( - vandq_u8(match2NlDotC, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)), - match2NlDotDMasked - ) - ); - mergeKillDots = vpaddq_u8(mergeKillDots, mergeKillDots); - uint64x2_t mergeKillDotsShifted = vshlq_n_u64(vreinterpretq_u64_u8(mergeKillDots), 2); - mask |= vgetq_lane_u64(mergeKillDotsShifted, 0); - cmpCombined = vorrq_u8(cmpCombined, vreinterpretq_u8_u64(mergeKillDotsShifted)); - nextMaskMix = vextq_u8(match2NlDotD, vdupq_n_u8(0), 14); - } else if(searchEnd) { - match2EqA = vextq_u8(cmpEqA, cmpEqB, 2); - match2EqB = vextq_u8(cmpEqB, cmpEqC, 2); - match2EqC = vextq_u8(cmpEqC, cmpEqD, 2); - - uint8x16_t match3EqYA = vandq_u8(match2EqA, vceqq_u8(vextq_u8(dataA, dataB, 3), vdupq_n_u8('y'))); - uint8x16_t match3EqYB = vandq_u8(match2EqB, vceqq_u8(vextq_u8(dataB, dataC, 3), vdupq_n_u8('y'))); - uint8x16_t match3EqYC = vandq_u8(match2EqC, vceqq_u8(vextq_u8(dataC, dataD, 3), vdupq_n_u8('y'))); - uint8x16_t match3EqYD = vandq_u8(match2EqD, vceqq_u8(vextq_u8(dataD, nextData, 3), vdupq_n_u8('y'))); - if(LIKELIHOOD(0.001, neon_vect_is_nonzero(vorrq_u8( - vorrq_u8(match3EqYA, match3EqYB), - vorrq_u8(match3EqYC, match3EqYD) - )))) { - uint8x16_t match1LfA = vceqq_u8(vextq_u8(dataA, dataB, 1), vdupq_n_u8('\n')); - uint8x16_t match1LfB = vceqq_u8(vextq_u8(dataB, dataC, 1), vdupq_n_u8('\n')); - uint8x16_t match1LfC = vceqq_u8(vextq_u8(dataC, dataD, 1), vdupq_n_u8('\n')); - uint8x16_t match1LfD = vceqq_u8(vextq_u8(dataD, nextData, 1), vdupq_n_u8('\n')); - uint8x16_t matchEnd = vorrq_u8( - vorrq_u8( - vandq_u8(match3EqYA, vandq_u8(match1LfA, cmpCrA)), - vandq_u8(match3EqYB, vandq_u8(match1LfB, cmpCrB)) - ), - vorrq_u8( - vandq_u8(match3EqYC, vandq_u8(match1LfC, cmpCrC)), - vandq_u8(match3EqYD, vandq_u8(match1LfD, cmpCrD)) - ) - ); - if(LIKELIHOOD(0.001, neon_vect_is_nonzero(matchEnd))) { - len += i; - nextMask = decoder_set_nextMask(src+i, mask); - break; - } - } - if(isRaw) - nextMaskMix = vdupq_n_u8(0); - } else if(isRaw) // no \r_. found - nextMaskMix = vdupq_n_u8(0); - } - - // a spec compliant encoder should never generate sequences: ==, =\n and =\r, but we'll handle them to be spec compliant - // the yEnc specification requires any character following = to be unescaped, not skipped over, so we'll deal with that - // firstly, check for invalid sequences of = (we assume that these are rare, as a spec compliant yEnc encoder should not generate these) - uint64_t maskEqShift1 = (maskEq << 1) | escFirst; - if(LIKELIHOOD(0.0001, (mask & maskEqShift1) != 0)) { - maskEq = fix_eqMask(maskEq, maskEqShift1); - - unsigned char nextEscFirst = maskEq>>63; - // next, eliminate anything following a `=` from the special char mask; this eliminates cases of `=\r` so that they aren't removed - maskEq = (maskEq<<1) | escFirst; - mask &= ~maskEq; - escFirst = nextEscFirst; - - // unescape chars following `=` -#if defined(__GNUC__) && !defined(__clang__) - // this seems to stop GCC9 producing slow code, for some reason... TODO: investigate why - uint8x8_t _maskEqTemp = vreinterpret_u8_u64(vmov_n_u64(maskEq)); - uint8x16_t maskEqTemp = vcombine_u8(_maskEqTemp, vdup_n_u8(0)); -#else - uint8x16_t maskEqTemp = vreinterpretq_u8_u64(vmovq_n_u64(maskEq)); -#endif - cmpCombined = vbicq_u8(cmpCombined, maskEqTemp); // `mask &= ~maskEq` in vector form - - uint8x16_t vMaskEqA = vqtbl1q_u8( - maskEqTemp, - vmakeq_u8(0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1) - ); - maskEqTemp = vextq_u8(maskEqTemp, maskEqTemp, 2); - uint8x16_t vMaskEqB = vqtbl1q_u8( - maskEqTemp, - vmakeq_u8(0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1) - ); - maskEqTemp = vextq_u8(maskEqTemp, maskEqTemp, 2); - uint8x16_t vMaskEqC = vqtbl1q_u8( - maskEqTemp, - vmakeq_u8(0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1) - ); - maskEqTemp = vextq_u8(maskEqTemp, maskEqTemp, 2); - uint8x16_t vMaskEqD = vqtbl1q_u8( - maskEqTemp, - vmakeq_u8(0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1) - ); - vMaskEqA = vtstq_u8(vMaskEqA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); - vMaskEqB = vtstq_u8(vMaskEqB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); - vMaskEqC = vtstq_u8(vMaskEqC, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); - vMaskEqD = vtstq_u8(vMaskEqD, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); - - dataA = vsubq_u8( - dataA, - vbslq_u8(vMaskEqA, vdupq_n_u8(64+42), vdupq_n_u8(42)) - ); - dataB = vsubq_u8( - dataB, - vbslq_u8(vMaskEqB, vdupq_n_u8(64+42), vdupq_n_u8(42)) - ); - dataC = vsubq_u8( - dataC, - vbslq_u8(vMaskEqC, vdupq_n_u8(64+42), vdupq_n_u8(42)) - ); - dataD = vsubq_u8( - dataD, - vbslq_u8(vMaskEqD, vdupq_n_u8(64+42), vdupq_n_u8(42)) - ); - } else { - // no invalid = sequences found - we can cut out some things from above - // this code path is a shortened version of above; it's here because it's faster, and what we'll be dealing with most of the time - escFirst = (maskEq >> 63); - - dataA = vsubq_u8( - dataA, - vbslq_u8( - vextq_u8(vdupq_n_u8(42), cmpEqA, 15), - vdupq_n_u8(64+42), - yencOffset - ) - ); - dataB = vsubq_u8( - dataB, - vbslq_u8( - vextq_u8(cmpEqA, cmpEqB, 15), - vdupq_n_u8(64+42), - vdupq_n_u8(42) - ) - ); - dataC = vsubq_u8( - dataC, - vbslq_u8( - vextq_u8(cmpEqB, cmpEqC, 15), - vdupq_n_u8(64+42), - vdupq_n_u8(42) - ) - ); - dataD = vsubq_u8( - dataD, - vbslq_u8( - vextq_u8(cmpEqC, cmpEqD, 15), - vdupq_n_u8(64+42), - vdupq_n_u8(42) - ) - ); - } - yencOffset = vsetq_lane_u8((escFirst << 6) | 42, yencOffset, 0); - - // all that's left is to 'compress' the data (skip over masked chars) - uint64_t counts = vget_lane_u64(vreinterpret_u64_u8(vcnt_u8(vget_low_u8(cmpCombined))), 0); - counts = 0x0808080808080808ULL - counts; - counts += counts>>8; - - vst1q_u8(p, vqtbl1q_u8( - dataA, - vld1q_u8((uint8_t*)(compactLUT + (mask&0x7fff))) - )); - p += counts & 0xff; - mask >>= 16; - vst1q_u8(p, vqtbl1q_u8( - dataB, - vld1q_u8((uint8_t*)(compactLUT + (mask&0x7fff))) - )); - p += (counts>>16) & 0xff; - mask >>= 16; - vst1q_u8(p, vqtbl1q_u8( - dataC, - vld1q_u8((uint8_t*)(compactLUT + (mask&0x7fff))) - )); - p += (counts>>32) & 0xff; - mask >>= 16; - vst1q_u8(p, vqtbl1q_u8( - dataD, - vld1q_u8((uint8_t*)(compactLUT + (mask&0x7fff))) - )); - p += (counts>>48) & 0xff; - } else { - dataA = vsubq_u8(dataA, yencOffset); - dataB = vsubq_u8(dataB, vdupq_n_u8(42)); - dataC = vsubq_u8(dataC, vdupq_n_u8(42)); - dataD = vsubq_u8(dataD, vdupq_n_u8(42)); - _vst1q_u8_x4(p, vcreate4_u8(dataA, dataB, dataC, dataD)); - p += sizeof(uint8x16_t)*4; - escFirst = 0; - yencOffset = vdupq_n_u8(42); - } - } -} -} // namespace - -void RapidYenc::decoder_set_neon_funcs() { - decoder_init_lut(compactLUT); - _do_decode = &do_decode_simd >; - _do_decode_raw = &do_decode_simd >; - _do_decode_end_raw = &do_decode_simd >; - _decode_isa = ISA_LEVEL_NEON; -} -#else -void RapidYenc::decoder_set_neon_funcs() {} -#endif diff --git a/rapidyenc/rapidyenc/src/decoder_rvv.cc b/rapidyenc/rapidyenc/src/decoder_rvv.cc deleted file mode 100644 index d2315ff..0000000 --- a/rapidyenc/rapidyenc/src/decoder_rvv.cc +++ /dev/null @@ -1,281 +0,0 @@ -#include "common.h" -#include "decoder_common.h" -#ifdef __riscv_vector - - -#ifdef __riscv_v_intrinsic -# define RV_vmerge_vxm_u8m2 RV(vmerge_vxm_u8m2) -# define RV_vmerge_vxm_u16m2 RV(vmerge_vxm_u16m2) -#else -# define RV_vmerge_vxm_u8m2(v, x, m, vl) RV(vmerge_vxm_u8m2)(m, v, x, vl) -# define RV_vmerge_vxm_u16m2(v, x, m, vl) RV(vmerge_vxm_u16m2)(m, v, x, vl) -#endif - -#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 12000 -# define RV_VEC_CAST(masksz, vecsz, vec) RV(vreinterpret_v_b##masksz##_u##vecsz##m1)(vec) -#else -# define RV_VEC_CAST(masksz, vecsz, vec) *(vuint##vecsz##m1_t*)(&(vec)) -#endif - - -template -static inline vbool4_t mask_lshift(vbool4_t m, unsigned shiftIn, size_t vl) { - vuint8m1_t mv = RV_VEC_CAST(4, 8, m); - vuint8m1_t mvl = RV(vsll_vx_u8m1)(mv, shift, vl/8); - vuint8m1_t mvr = RV(vsrl_vx_u8m1)(mv, 8-shift, vl/8); - mvr = RV(vslide1up_vx_u8m1)(mvr, shiftIn, vl/8); - - return RV(vmor_mm_b4)( - RV_MASK_CAST(4, 8, mvl), RV_MASK_CAST(4, 8, mvr), vl - ); -} -template -static inline vbool64_t mask_lshift(vbool64_t m, unsigned shiftIn, size_t vl) { - vuint8m1_t mv = RV_VEC_CAST(64, 8, m); - vuint8m1_t mvl = RV(vsll_vx_u8m1)(mv, shift, vl/8); - vuint8m1_t mvr = RV(vsrl_vx_u8m1)(mv, 8-shift, vl/8); - mvr = RV(vslide1up_vx_u8m1)(mvr, shiftIn, vl/8); - - return RV(vmor_mm_b64)( - RV_MASK_CAST(64, 8, mvl), RV_MASK_CAST(64, 8, mvr), vl - ); -} - -static inline vuint8m2_t set_first_vu8(vuint8m2_t src, uint8_t item, size_t vl) { -#ifdef __riscv_v_intrinsic - return RV(vmv_s_x_u8m2_tu)(src, item, vl); -#else - vuint8m1_t m = RV(vslide1up_vx_u8m1)(RV(vmv_v_x_u8m1)(0, ~0), 1, ~0); - return RV_vmerge_vxm_u8m2(src, item, RV_MASK_CAST(4, 8, m), vl); -#endif -} -static inline vuint16m2_t set_first_vu16(vuint16m2_t src, uint16_t item, size_t vl) { -#ifdef __riscv_v_intrinsic - return RV(vmv_s_x_u16m2_tu)(src, item, vl); -#else - vuint16m1_t m = RV(vslide1up_vx_u16m1)(RV(vmv_v_x_u16m1)(0, ~0), 1, ~0); - return RV_vmerge_vxm_u16m2(src, item, RV_MASK_CAST(8, 16, m), vl); -#endif -} - - -namespace RapidYenc { - -template -HEDLEY_ALWAYS_INLINE void do_decode_rvv(const uint8_t* src, long& len, unsigned char*& outp, unsigned char& escFirst, uint16_t& nextMask) { - HEDLEY_ASSUME(escFirst == 0 || escFirst == 1); - HEDLEY_ASSUME(nextMask == 0 || nextMask == 1 || nextMask == 2); - - size_t vl2 = RV(vsetvlmax_e8m2)(); - - vuint8m2_t yencOffset = RV(vmv_v_x_u8m2)(42, vl2); - if(escFirst) yencOffset = set_first_vu8(yencOffset, 42+64, vl2); - vuint8m2_t lfCompare = RV(vmv_v_x_u8m2)('\n', vl2); - if(nextMask && isRaw) { - lfCompare = RV(vreinterpret_v_u16m2_u8m2)( - set_first_vu16(RV(vreinterpret_v_u8m2_u16m2)(lfCompare), nextMask == 1 ? 0x0a2e /*".\n"*/ : 0x2e0a /*"\n."*/, vl2/2) - ); - } - - // mask where only the highest bit is set - vbool4_t lastBit = RV(vmseq_vx_u8m2_b4)( - RV(vslide1down_vx_u8m2)(RV(vmv_v_x_u8m2)(0, vl2), 1, vl2), - 1, vl2 - ); - - decoder_set_nextMask(src, len, nextMask); - - // TODO: consider exploiting partial vector capability - long inpos; - for(inpos = -len; inpos; inpos += vl2) { - vuint8m2_t data = RV(vle8_v_u8m2)(src + inpos, vl2); - - // search for special chars - vbool4_t cmpEq = RV(vmseq_vx_u8m2_b4)(data, '=', vl2); - vbool4_t cmpCr = RV(vmseq_vx_u8m2_b4)(data, '\r', vl2); - // note: cmp is always negated (unlike cmpEq/Cr) - vbool4_t cmp = RV(vmnor_mm_b4)( - RV(vmor_mm_b4)(cmpEq, cmpCr, vl2), - isRaw ? RV(vmseq_vv_u8m2_b4)(data, lfCompare, vl2) : RV(vmseq_vx_u8m2_b4)(data, '\n', vl2), - vl2 - ); - - size_t numOutputChars = RV(vcpop_m_b4)(cmp, vl2); - - if(numOutputChars != vl2) { - // dot-unstuffing + end detection - if((isRaw || searchEnd) && RV(vcpop_m_b4)(RV(vmxnor_mm_b4)(cmp, cmpEq, vl2), vl2)) { - uint32_t nextWord; - if(!searchEnd) { - memcpy(&nextWord, src + inpos + vl2, 2); - } else { - memcpy(&nextWord, src + inpos + vl2, 4); - } - vuint8m2_t nextData2 = RV(vreinterpret_v_u16m2_u8m2)(RV(vslide1down_vx_u16m2)(RV(vreinterpret_v_u8m2_u16m2)(data), nextWord, vl2/2)); - - vbool4_t match2Cr_Dot, match3EqY; - vuint8m2_t nextData3; - if(isRaw) { - match2Cr_Dot = RV(vmand_mm_b4)(cmpCr, RV(vmseq_vx_u8m2_b4)(nextData2, '.', vl2), vl2); - } - - if(searchEnd) { - nextData3 = RV(vslide1down_vx_u8m2)(nextData2, nextWord>>16, vl2); - match3EqY = RV(vmand_mm_b4)( - RV(vmseq_vx_u8m2_b4)(nextData2, '=', vl2), - RV(vmseq_vx_u8m2_b4)(nextData3, 'y', vl2), - vl2 - ); - } - - // find patterns of \r_. - if(isRaw && LIKELIHOOD(0.001, RV(vcpop_m_b4)(match2Cr_Dot, vl2) > 0)) { - // find \r\n. - vuint8m2_t nextData1 = RV(vslide1down_vx_u8m2)(data, nextWord, vl2); - vbool4_t match1Lf = RV(vmseq_vx_u8m2_b4)(nextData1, '\n', vl2); - vbool4_t match2NlDot = RV(vmand_mm_b4)(match2Cr_Dot, match1Lf, vl2); - - if(searchEnd) { - vbool4_t match1Nl = RV(vmand_mm_b4)(cmpCr, match1Lf, vl2); - - vuint8m2_t nextData4 = RV(vreinterpret_v_u32m2_u8m2)(RV(vslide1down_vx_u32m2)(RV(vreinterpret_v_u8m2_u32m2)(data), nextWord, vl2/4)); - - // match instances of \r\n.\r\n and \r\n.=y - vbool4_t match4Nl = RV(vmand_mm_b4)( - RV(vmseq_vx_u8m2_b4)(nextData3, '\r', vl2), - RV(vmseq_vx_u8m2_b4)(nextData4, '\n', vl2), - vl2 - ); - vbool4_t match4EqY = RV(vmand_mm_b4)( - RV(vmseq_vx_u8m2_b4)(nextData3, '=', vl2), - RV(vmseq_vx_u8m2_b4)(nextData4, 'y', vl2), - vl2 - ); - - // merge \r\n and =y matches - vbool4_t match4End = RV(vmor_mm_b4)(match4Nl, match4EqY, vl2); - // merge with \r\n. - match4End = RV(vmand_mm_b4)(match4End, match2NlDot, vl2); - // merge \r\n=y - vbool4_t match3End = RV(vmand_mm_b4)(match1Nl, match3EqY, vl2); - - vbool4_t matchEnd = RV(vmor_mm_b4)(match4End, match3End, vl2); - - // combine match sequences - if(LIKELIHOOD(0.001, RV(vcpop_m_b4)(matchEnd, vl2) > 0)) { - // terminator found - len += inpos; - nextMask = decoder_set_nextMask(src+inpos, ~RV(vmv_x_s_u8m1_u8)(RV_VEC_CAST(4, 8, cmp))); - break; - } - } - - // shift match2NlDot by 2 - cmp = RV(vmandn_mm_b4)(cmp, mask_lshift<2>(match2NlDot, 0, vl2), vl2); - numOutputChars = RV(vcpop_m_b4)(cmp, vl2); - - vuint8mf4_t nextNlDot = RV(vslidedown_vx_u8mf4)( -#ifndef __riscv_v_intrinsic - RV(vmv_v_x_u8mf4)(0, vl2/8), -#endif - RV_VEC_U8MF4_CAST(match2NlDot), vl2/8-1, vl2/8 - ); - nextNlDot = RV(vsrl_vx_u8mf4)(nextNlDot, 6, vl2/8); - vuint8m1_t nextNlDotVec = RV(vlmul_ext_v_u8mf4_u8m1)(nextNlDot); - lfCompare = RV_vmerge_vxm_u8m2(RV(vmv_v_x_u8m2)('\n', vl2), '.', RV_MASK_CAST(4, 8, nextNlDotVec), vl2); - } else if(searchEnd) { - if(LIKELIHOOD(0.001, RV(vcpop_m_b4)(match3EqY, vl2) != 0)) { - vuint8m2_t nextData1 = RV(vslide1down_vx_u8m2)(data, nextWord, vl2); - vbool4_t match1Lf = RV(vmseq_vx_u8m2_b4)(nextData1, '\n', vl2); - vbool4_t matchEnd = RV(vmand_mm_b4)(RV(vmand_mm_b4)(match3EqY, cmpCr, vl2), match1Lf, vl2); - if(LIKELIHOOD(0.001, RV(vcpop_m_b4)(matchEnd, vl2) > 0)) { - len += inpos; - nextMask = decoder_set_nextMask(src+inpos, ~RV(vmv_x_s_u8m1_u8)(RV_VEC_CAST(4, 8, cmp))); - break; - } - } - if(isRaw) - lfCompare = RV(vmv_v_x_u8m2)('\n', vl2); - } else if(isRaw) // no \r_. found - lfCompare = RV(vmv_v_x_u8m2)('\n', vl2); - } - - // the second character in an escape sequence - vbool4_t cmpEqShift1 = mask_lshift<1>(cmpEq, escFirst, vl2); - - // a spec compliant encoder should never generate sequences: ==, =\n and =\r, but we'll handle them to be spec compliant - // the yEnc specification requires any character following = to be unescaped, not skipped over, so we'll deal with that - // firstly, check for invalid sequences of = (we assume that these are rare, as a spec compliant yEnc encoder should not generate these) - if(LIKELIHOOD(0.0001, RV(vcpop_m_b4)(RV(vmandn_mm_b4)(cmpEqShift1, cmp, vl2), vl2) != 0)) { - // replicate fix_eqMask, but in vector form - vbool4_t groupStart = RV(vmandn_mm_b4)(cmpEq, cmpEqShift1, vl2); - vuint8m1_t evenBitsV = RV(vmv_v_x_u8m1)(0x55, vl2); - vbool4_t evenBits = RV_MASK_CAST(4, 8, evenBitsV); - vbool4_t evenStart = RV(vmand_mm_b4)(groupStart, evenBits, vl2); - - // compute `cmpEq + evenStart` to obtain oddGroups - vbool4_t oddGroups; - vuint64m1_t cmpEq64 = RV_VEC_CAST(4, 64, cmpEq); - vuint64m1_t evenStart64 = RV_VEC_CAST(4, 64, evenStart); - vuint64m1_t oddGroups64; - if(vl2 <= 64) { - // no loop needed - single 64b add will work - oddGroups64 = RV(vadd_vv_u64m1)(cmpEq64, evenStart64, 1); - } else { - // need to loop whilst the add causes a carry - unsigned vl64 = vl2/64; - vbool64_t carry = RV(vmadc_vv_u64m1_b64)(cmpEq64, evenStart64, vl64); - carry = mask_lshift<1>(carry, 0, vl64); - oddGroups64 = RV(vadd_vv_u64m1)(cmpEq64, evenStart64, 1); - while(RV(vcpop_m_b64)(carry, vl64)) { - vbool64_t nextCarry = RV(vmadc_vx_u64m1_b64)(oddGroups64, 1, vl64); - oddGroups64 = RV(vadd_vx_u64m1_mu)(carry, oddGroups64, oddGroups64, 1, vl64); - carry = mask_lshift<1>(nextCarry, 0, vl64); - } - } - oddGroups = RV_MASK_CAST(4, 64, oddGroups64); - - cmpEq = RV(vmand_mm_b4)(RV(vmxor_mm_b4)(oddGroups, evenBits, vl2), cmpEq, vl2); - - cmpEqShift1 = mask_lshift<1>(cmpEq, escFirst, vl2); - cmp = RV(vmor_mm_b4)(cmpEqShift1, cmp, vl2); // ~(~cmp & ~cmpEqShift1) - numOutputChars = RV(vcpop_m_b4)(cmp, vl2); - } - escFirst = RV(vcpop_m_b4)(RV(vmand_mm_b4)(cmpEq, lastBit, vl2), vl2); - - data = RV(vsub_vv_u8m2)(data, RV_vmerge_vxm_u8m2(yencOffset, 64+42, cmpEqShift1, vl2), vl2); - yencOffset = set_first_vu8(yencOffset, 42 | (escFirst<<6), vl2); - - // all that's left is to remove unwanted chars -#ifdef __riscv_v_intrinsic - data = RV(vcompress_vm_u8m2)(data, cmp, vl2); -#else - data = RV(vcompress_vm_u8m2)(cmp, data, data, vl2); -#endif - RV(vse8_v_u8m2)(outp, data, vl2); - } else { - data = RV(vsub_vv_u8m2)(data, yencOffset, vl2); - RV(vse8_v_u8m2)(outp, data, vl2); - // TODO: should these be done at LMUL=1? or, it might not be worth this strategy (e.g. do an additional OR instead), considering the cost of LMUL=2 - yencOffset = RV(vmv_v_x_u8m2)(42, vl2); - if(isRaw) lfCompare = RV(vmv_v_x_u8m2)('\n', vl2); - escFirst = 0; - } - outp += numOutputChars; - } -} - -size_t decoder_rvv_width() { - return RV(vsetvlmax_e8m2)(); -} -} // namespace - -void RapidYenc::decoder_set_rvv_funcs() { - _do_decode = &do_decode_simd >; - _do_decode_raw = &do_decode_simd >; - _do_decode_end_raw = &do_decode_simd >; - _decode_isa = ISA_LEVEL_RVV; -} -#else -void RapidYenc::decoder_set_rvv_funcs() {} -#endif diff --git a/rapidyenc/rapidyenc/src/decoder_sse2.cc b/rapidyenc/rapidyenc/src/decoder_sse2.cc deleted file mode 100644 index 829b261..0000000 --- a/rapidyenc/rapidyenc/src/decoder_sse2.cc +++ /dev/null @@ -1,38 +0,0 @@ -#include "common.h" - -#include "decoder_common.h" -#ifdef __SSE2__ -#include "decoder_sse_base.h" - -void RapidYenc::decoder_sse_init(RapidYenc::SSELookups* HEDLEY_RESTRICT& lookups) { - ALIGN_ALLOC(lookups, sizeof(SSELookups), 16); - for(int i=0; i<256; i++) { - lookups->BitsSetTable256inv[i] = 8 - ( - (i & 1) + ((i>>1) & 1) + ((i>>2) & 1) + ((i>>3) & 1) + ((i>>4) & 1) + ((i>>5) & 1) + ((i>>6) & 1) + ((i>>7) & 1) - ); - - #define _X(n, k) ((((n) & (1<eqAdd[i] = _X(i, 0) | _X(i, 1) | _X(i, 2) | _X(i, 3) | _X(i, 4) | _X(i, 5) | _X(i, 6) | _X(i, 7); - #undef _X - } - for(int i=0; i<32; i++) { - for(int j=0; j<16; j++) { - if(i >= 16) // only used for LZCNT - lookups->unshufMask[i*16 + j] = ((31-i)>j ? -1 : 0); - else // only used for BSR - lookups->unshufMask[i*16 + j] = (i>j ? -1 : 0); - } - } -} - -void RapidYenc::decoder_set_sse2_funcs() { - decoder_sse_init(lookups); - decoder_init_lut(lookups->compact); - _do_decode = &do_decode_simd >; - _do_decode_raw = &do_decode_simd >; - _do_decode_end_raw = &do_decode_simd >; - _decode_isa = ISA_LEVEL_SSE2; -} -#else -void RapidYenc::decoder_set_sse2_funcs() {} -#endif diff --git a/rapidyenc/rapidyenc/src/decoder_sse_base.h b/rapidyenc/rapidyenc/src/decoder_sse_base.h deleted file mode 100644 index 042faf9..0000000 --- a/rapidyenc/rapidyenc/src/decoder_sse_base.h +++ /dev/null @@ -1,699 +0,0 @@ - -#ifdef __SSE2__ - -#if defined(__clang__) && __clang_major__ == 6 && __clang_minor__ == 0 -// VBMI2 introduced in clang 6.0, but 128-bit functions misnamed there; fixed in clang 7.0, but we'll handle those on 6.0 -# define _mm_mask_compressstoreu_epi8 _mm128_mask_compressstoreu_epi8 -# define _mm_shrdi_epi16 _mm128_shrdi_epi16 -#endif - -#if defined(__tune_icelake_client__) || defined(__tune_icelake_server__) || defined(__tune_tigerlake__) || defined(__tune_rocketlake__) || defined(__tune_alderlake__) || defined(__tune_sapphirerapids__) -# define COMPRESS_STORE _mm_mask_compressstoreu_epi8 -#else -// avoid uCode on Zen4 -# define COMPRESS_STORE(dst, mask, vec) _mm_storeu_si128((__m128i*)(dst), _mm_maskz_compress_epi8(mask, vec)) -#endif - -// GCC (ver 6-10(dev)) fails to optimize pure C version of mask testing, but has this intrinsic; Clang >= 7 optimizes C version fine -#if (defined(__GNUC__) && __GNUC__ >= 7) || (defined(_MSC_VER) && _MSC_VER >= 1924) -# define KORTEST16(a, b) !_kortestz_mask16_u8((a), (b)) -# define KAND16(a, b) _kand_mask16((a), (b)) -# define KOR16(a, b) _kor_mask16((a), (b)) -#else -# define KORTEST16(a, b) ((a) | (b)) -# define KAND16(a, b) ((a) & (b)) -# define KOR16(a, b) ((a) | (b)) -#endif - -namespace RapidYenc { - #pragma pack(16) - typedef struct { - unsigned char BitsSetTable256inv[256]; - /*align16*/ struct { char bytes[16]; } compact[32768]; - /*align8*/ uint64_t eqAdd[256]; - /*align16*/ int8_t unshufMask[32*16]; - } SSELookups; - #pragma pack() -} -static RapidYenc::SSELookups* HEDLEY_RESTRICT lookups; - - -static HEDLEY_ALWAYS_INLINE __m128i force_align_read_128(const void* p) { -#ifdef _MSC_VER - // MSVC complains about casting away volatile - return *(__m128i *)(p); -#else - return *(volatile __m128i *)(p); -#endif -} - -namespace RapidYenc { - void decoder_sse_init(SSELookups* HEDLEY_RESTRICT& lookups); // defined in decoder_sse2.cc -} - - -// for LZCNT/BSR -#ifdef _MSC_VER -# include -# include -static HEDLEY_ALWAYS_INLINE unsigned BSR32(unsigned src) { - unsigned long result; - _BitScanReverse((unsigned long*)&result, src); - return result; -} -#elif defined(__GNUC__) -// have seen Clang not like _bit_scan_reverse -# include // for lzcnt -# define BSR32(src) (31^__builtin_clz(src)) -#else -# include -# define BSR32 _bit_scan_reverse -#endif - -template -static HEDLEY_ALWAYS_INLINE __m128i sse2_compact_vect(uint32_t mask, __m128i data) { - while(mask) { - unsigned bitIndex; -#if defined(__LZCNT__) - if(use_isa & ISA_FEATURE_LZCNT) { - // lzcnt is always at least as fast as bsr, so prefer it if it's available - bitIndex = _lzcnt_u32(mask); - mask &= 0x7fffffffU>>bitIndex; - } else -#endif - { - bitIndex = BSR32(mask); - mask ^= 1<unshufMask + bitIndex); - data = _mm_or_si128( - _mm_and_si128(mergeMask, data), - _mm_andnot_si128(mergeMask, _mm_srli_si128(data, 1)) - ); - } - return data; -} - -namespace RapidYenc { - -template -HEDLEY_ALWAYS_INLINE void do_decode_sse(const uint8_t* src, long& len, unsigned char*& p, unsigned char& _escFirst, uint16_t& _nextMask) { - HEDLEY_ASSUME(_escFirst == 0 || _escFirst == 1); - HEDLEY_ASSUME(_nextMask == 0 || _nextMask == 1 || _nextMask == 2); - uintptr_t escFirst = _escFirst; - __m128i yencOffset = escFirst ? _mm_set_epi8( - -42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42-64 - ) : _mm_set1_epi8(-42); - -#if defined(__SSSE3__) && !defined(__tune_atom__) && !defined(__tune_slm__) && !defined(__tune_btver1__) && !defined(__tune_btver2__) - const bool _USING_FAST_MATCH = (use_isa >= ISA_LEVEL_SSSE3); -#else - const bool _USING_FAST_MATCH = false; -#endif -#if defined(__SSE4_1__) && !defined(__tune_slm__) && !defined(__tune_goldmont__) && !defined(__tune_goldmont_plus__) && !defined(__tune_tremont__) - const bool _USING_BLEND_ADD = (use_isa >= ISA_LEVEL_SSE41); -#else - const bool _USING_BLEND_ADD = false; -#endif -#if defined(__AVX512VL__) && defined(__AVX512BW__) -# if defined(_MSC_VER) && !defined(PLATFORM_AMD64) && !defined(__clang__) - const bool useAVX3MaskCmp = false; -# else - const bool useAVX3MaskCmp = (use_isa >= ISA_LEVEL_AVX3); -# endif -#endif - - __m128i lfCompare = _mm_set1_epi8('\n'); - __m128i minMask = _mm_set1_epi8('.'); - if(_nextMask && isRaw) { - if(_USING_FAST_MATCH) - minMask = _mm_insert_epi16(minMask, _nextMask == 1 ? 0x2e00 : 0x002e, 0); - else - lfCompare = _mm_insert_epi16(lfCompare, _nextMask == 1 ? 0x0a2e /*".\n"*/ : 0x2e0a /*"\n."*/, 0); - } - - decoder_set_nextMask(src, len, _nextMask); // set this before the loop because we can't check src after it's been overwritten - - intptr_t i; - for(i = -len; i; i += sizeof(__m128i)*2) { - __m128i oDataA = _mm_load_si128((__m128i *)(src+i)); - __m128i oDataB = _mm_load_si128((__m128i *)(src+i) + 1); - - // search for special chars - __m128i cmpEqA, cmpEqB, cmpCrA, cmpCrB; - __m128i cmpA, cmpB; -#if defined(__SSSE3__) - if(_USING_FAST_MATCH) { - cmpA = _mm_cmpeq_epi8(oDataA, _mm_shuffle_epi8( - _mm_set_epi8(-1,'=','\r',-1,-1,'\n',-1,-1,-1,-1,-1,-1,-1,-1,-1,'.'), - _mm_min_epu8(oDataA, minMask) - )); - cmpB = _mm_cmpeq_epi8(oDataB, _mm_shuffle_epi8( - _mm_set_epi8(-1,'=','\r',-1,-1,'\n',-1,-1,-1,-1,-1,-1,-1,-1,-1,'.'), - _mm_min_epu8(oDataB, _mm_set1_epi8('.')) - )); - } else -#endif - { - cmpEqA = _mm_cmpeq_epi8(oDataA, _mm_set1_epi8('=')); - cmpEqB = _mm_cmpeq_epi8(oDataB, _mm_set1_epi8('=')); - cmpCrA = _mm_cmpeq_epi8(oDataA, _mm_set1_epi8('\r')); - cmpCrB = _mm_cmpeq_epi8(oDataB, _mm_set1_epi8('\r')); - cmpA = _mm_or_si128( - _mm_or_si128( - _mm_cmpeq_epi8(oDataA, lfCompare), cmpCrA - ), - cmpEqA - ); - cmpB = _mm_or_si128( - _mm_or_si128( - _mm_cmpeq_epi8(oDataB, _mm_set1_epi8('\n')), cmpCrB - ), - cmpEqB - ); - } - - __m128i dataA, dataB; - if(!_USING_BLEND_ADD) - dataA = _mm_add_epi8(oDataA, yencOffset); - uint32_t mask = (unsigned)_mm_movemask_epi8(cmpA) | ((unsigned)_mm_movemask_epi8(cmpB) << 16); // not the most accurate mask if we have invalid sequences; we fix this up later - - if (LIKELIHOOD(0.42 /* rough guess */, mask != 0)) { - if(_USING_FAST_MATCH) { - cmpEqA = _mm_cmpeq_epi8(oDataA, _mm_set1_epi8('=')); - cmpEqB = _mm_cmpeq_epi8(oDataB, _mm_set1_epi8('=')); - } - -#define LOAD_HALVES(a, b) _mm_castps_si128(_mm_loadh_pi( \ - _mm_castsi128_ps(_mm_loadl_epi64((__m128i*)(a))), \ - (__m64*)(b) \ -)) - - // a spec compliant encoder should never generate sequences: ==, =\n and =\r, but we'll handle them to be spec compliant - // the yEnc specification requires any character following = to be unescaped, not skipped over, so we'll deal with that - // firstly, check for invalid sequences of = (we assume that these are rare, as a spec compliant yEnc encoder should not generate these) - uint32_t maskEq = (unsigned)_mm_movemask_epi8(cmpEqA) | ((unsigned)_mm_movemask_epi8(cmpEqB) << 16); - - // handle \r\n. sequences - // RFC3977 requires the first dot on a line to be stripped, due to dot-stuffing - if((isRaw || searchEnd) && LIKELIHOOD(0.25, mask != maskEq)) { -#if 0 - // for experimentation: prefer shifting data over unaligned loads on CPUs with slow unaligned handling - // haven't ever seen this be beneficial though - __m128i nextDataB; - if(searchEnd && _USING_BLEND_ADD) - nextDataB = _mm_cvtsi32_si128(*(uint32_t*)(src+i+sizeof(__m128i)*2)); -# define SHIFT_DATA_A(offs) (searchEnd && _USING_BLEND_ADD ? _mm_alignr_epi8(oDataB, oDataA, offs) : _mm_loadu_si128((__m128i *)(src+i+offs))) -# define SHIFT_DATA_B(offs) (searchEnd && _USING_BLEND_ADD ? _mm_alignr_epi8(nextDataB, oDataB, offs) : _mm_loadu_si128((__m128i *)(src+i+offs) + 1)) -#else -# define SHIFT_DATA_A(offs) _mm_loadu_si128((__m128i *)(src+i+offs)) -# define SHIFT_DATA_B(offs) _mm_loadu_si128((__m128i *)(src+i+offs) + 1) -#endif - __m128i tmpData2A = SHIFT_DATA_A(2); - __m128i tmpData2B = SHIFT_DATA_B(2); - __m128i match2EqA, match2EqB; - -#if defined(__AVX512VL__) && defined(__AVX512BW__) - __mmask16 match2EqMaskA, match2EqMaskB; - __mmask16 match0CrMaskA, match0CrMaskB; - __mmask16 match2CrXDtMaskA, match2CrXDtMaskB; - if(useAVX3MaskCmp && searchEnd) { - match2EqMaskA = _mm_cmpeq_epi8_mask(_mm_set1_epi8('='), tmpData2A); - match2EqMaskB = _mm_cmpeq_epi8_mask(_mm_set1_epi8('='), tmpData2B); - } else -#endif - if(searchEnd) { -#if !defined(__tune_btver1__) - if(use_isa < ISA_LEVEL_SSSE3) -#endif - match2EqA = _mm_cmpeq_epi8(_mm_set1_epi8('='), tmpData2A); - match2EqB = _mm_cmpeq_epi8(_mm_set1_epi8('='), tmpData2B); - } - int partialKillDotFound; - __m128i match2CrXDtA, match2CrXDtB; - if(isRaw) { -#if defined(__AVX512VL__) && defined(__AVX512BW__) - if(useAVX3MaskCmp) { - match0CrMaskA = _mm_cmpeq_epi8_mask(oDataA, _mm_set1_epi8('\r')); - match0CrMaskB = _mm_cmpeq_epi8_mask(oDataB, _mm_set1_epi8('\r')); - match2CrXDtMaskA = _mm_mask_cmpeq_epi8_mask(match0CrMaskA, tmpData2A, _mm_set1_epi8('.')); - match2CrXDtMaskB = _mm_mask_cmpeq_epi8_mask(match0CrMaskB, tmpData2B, _mm_set1_epi8('.')); - partialKillDotFound = KORTEST16(match2CrXDtMaskA, match2CrXDtMaskB); - } else -#endif - { - if(_USING_FAST_MATCH) { - cmpCrA = _mm_cmpeq_epi8(oDataA, _mm_set1_epi8('\r')); - cmpCrB = _mm_cmpeq_epi8(oDataB, _mm_set1_epi8('\r')); - } - match2CrXDtA = _mm_and_si128(cmpCrA, _mm_cmpeq_epi8(tmpData2A, _mm_set1_epi8('.'))); - match2CrXDtB = _mm_and_si128(cmpCrB, _mm_cmpeq_epi8(tmpData2B, _mm_set1_epi8('.'))); - partialKillDotFound = _mm_movemask_epi8(_mm_or_si128(match2CrXDtA, match2CrXDtB)); - } - } - - if(isRaw && LIKELIHOOD(0.001, partialKillDotFound)) { - __m128i match2NlDotA, match1NlA; - __m128i match2NlDotB, match1NlB; - // merge matches for \r\n. -#if defined(__AVX512VL__) && defined(__AVX512BW__) - __mmask16 match1NlMaskA, match1NlMaskB; - __mmask16 match2NlDotMaskA, match2NlDotMaskB; - if(useAVX3MaskCmp) { - match1NlMaskA = _mm_mask_cmpeq_epi8_mask( - match0CrMaskA, - _mm_set1_epi8('\n'), - SHIFT_DATA_A(1) - ); - match1NlMaskB = _mm_mask_cmpeq_epi8_mask( - match0CrMaskB, - _mm_set1_epi8('\n'), - SHIFT_DATA_B(1) - ); - match2NlDotMaskA = KAND16(match2CrXDtMaskA, match1NlMaskA); - match2NlDotMaskB = KAND16(match2CrXDtMaskB, match1NlMaskB); - } else -#endif - { - __m128i match1LfA = _mm_cmpeq_epi8(_mm_set1_epi8('\n'), SHIFT_DATA_A(1)); - __m128i match1LfB = _mm_cmpeq_epi8(_mm_set1_epi8('\n'), SHIFT_DATA_B(1)); - - // always recompute cmpCr to avoid register spills above - cmpCrA = _mm_cmpeq_epi8(force_align_read_128(src+i), _mm_set1_epi8('\r')); - cmpCrB = _mm_cmpeq_epi8(force_align_read_128(src+i + sizeof(__m128i)), _mm_set1_epi8('\r')); - match1NlA = _mm_and_si128(match1LfA, cmpCrA); - match1NlB = _mm_and_si128(match1LfB, cmpCrB); - match2NlDotA = _mm_and_si128(match2CrXDtA, match1NlA); - match2NlDotB = _mm_and_si128(match2CrXDtB, match1NlB); - } - if(searchEnd) { - __m128i tmpData3A = SHIFT_DATA_A(3); - __m128i tmpData3B = SHIFT_DATA_B(3); - __m128i tmpData4A = SHIFT_DATA_A(4); - __m128i tmpData4B = SHIFT_DATA_B(4); - // match instances of \r\n.\r\n and \r\n.=y - // TODO: consider doing a PALIGNR using match1Nl for match4NlA - __m128i match3CrA = _mm_cmpeq_epi8(_mm_set1_epi8('\r'), tmpData3A); - __m128i match3CrB = _mm_cmpeq_epi8(_mm_set1_epi8('\r'), tmpData3B); - __m128i match4LfA = _mm_cmpeq_epi8(tmpData4A, _mm_set1_epi8('\n')); - __m128i match4LfB = _mm_cmpeq_epi8(tmpData4B, _mm_set1_epi8('\n')); - __m128i match4EqYA = _mm_cmpeq_epi16(tmpData4A, _mm_set1_epi16(0x793d)); // =y - __m128i match4EqYB = _mm_cmpeq_epi16(tmpData4B, _mm_set1_epi16(0x793d)); // =y - - int matchEnd; -#if defined(__AVX512VL__) && defined(__AVX512BW__) - if(useAVX3MaskCmp) { - __mmask16 match3EqYMaskA = _mm_mask_cmpeq_epi8_mask( - match2EqMaskA, _mm_set1_epi8('y'), tmpData3A - ); - __mmask16 match3EqYMaskB = _mm_mask_cmpeq_epi8_mask( - match2EqMaskB, _mm_set1_epi8('y'), tmpData3B - ); - __m128i match34EqYA, match34EqYB; -# ifdef __AVX512VBMI2__ - if(use_isa >= ISA_LEVEL_VBMI2) { - match34EqYA = _mm_shrdi_epi16(_mm_movm_epi8(match3EqYMaskA), match4EqYA, 8); - match34EqYB = _mm_shrdi_epi16(_mm_movm_epi8(match3EqYMaskB), match4EqYB, 8); - } else -# endif - { - // (match4EqY & 0xff00) | (match3EqY >> 8) - match34EqYA = _mm_mask_blend_epi8(match3EqYMaskA>>1, _mm_and_si128(match4EqYA, _mm_set1_epi16(-0xff)), _mm_set1_epi8(-1)); - match34EqYB = _mm_mask_blend_epi8(match3EqYMaskB>>1, _mm_and_si128(match4EqYB, _mm_set1_epi16(-0xff)), _mm_set1_epi8(-1)); - } - // merge \r\n and =y matches for tmpData4 - __m128i match4EndA = _mm_ternarylogic_epi32(match34EqYA, match3CrA, match4LfA, 0xF8); // (match3Cr & match4Lf) | match34EqY - __m128i match4EndB = _mm_ternarylogic_epi32(match34EqYB, match3CrB, match4LfB, 0xF8); - // merge with \r\n. and combine - matchEnd = KORTEST16( - KOR16( - _mm_mask_test_epi8_mask(match2NlDotMaskA, match4EndA, match4EndA), - KAND16(match3EqYMaskA, match1NlMaskA) - ), - KOR16( - _mm_mask_test_epi8_mask(match2NlDotMaskB, match4EndB, match4EndB), - KAND16(match3EqYMaskB, match1NlMaskB) - ) - ); - } else -#endif - { -#if defined(__SSSE3__) && !defined(__tune_btver1__) - if(use_isa >= ISA_LEVEL_SSSE3) - match2EqA = _mm_alignr_epi8(cmpEqB, cmpEqA, 2); -#endif - __m128i match3EqYA = _mm_and_si128(match2EqA, _mm_cmpeq_epi8(_mm_set1_epi8('y'), tmpData3A)); - __m128i match3EqYB = _mm_and_si128(match2EqB, _mm_cmpeq_epi8(_mm_set1_epi8('y'), tmpData3B)); - match4EqYA = _mm_slli_epi16(match4EqYA, 8); // TODO: also consider using PBLENDVB here with shifted match3EqY instead - match4EqYB = _mm_slli_epi16(match4EqYB, 8); - // merge \r\n and =y matches for tmpData4 - __m128i match4EndA = _mm_or_si128( - _mm_and_si128(match3CrA, match4LfA), - _mm_or_si128(match4EqYA, _mm_srli_epi16(match3EqYA, 8)) // _mm_srli_si128 by 1 also works - ); - __m128i match4EndB = _mm_or_si128( - _mm_and_si128(match3CrB, match4LfB), - _mm_or_si128(match4EqYB, _mm_srli_epi16(match3EqYB, 8)) // _mm_srli_si128 by 1 also works - ); - // merge with \r\n. - match4EndA = _mm_and_si128(match4EndA, match2NlDotA); - match4EndB = _mm_and_si128(match4EndB, match2NlDotB); - // match \r\n=y - __m128i match3EndA = _mm_and_si128(match3EqYA, match1NlA); - __m128i match3EndB = _mm_and_si128(match3EqYB, match1NlB); - // combine match sequences - matchEnd = _mm_movemask_epi8(_mm_or_si128( - _mm_or_si128(match4EndA, match3EndA), - _mm_or_si128(match4EndB, match3EndB) - )); - } - - if(LIKELIHOOD(0.001, matchEnd)) { - // terminator found - // there's probably faster ways to do this, but reverting to scalar code should be good enough - len += (long)i; - _nextMask = decoder_set_nextMask(src+i, mask); - break; - } - } -#if defined(__AVX512VL__) && defined(__AVX512BW__) - if(useAVX3MaskCmp) { - mask |= match2NlDotMaskA << 2; - mask |= (match2NlDotMaskB << 18) & 0xffffffff; - minMask = _mm_maskz_mov_epi8(~(match2NlDotMaskB>>14), _mm_set1_epi8('.')); - } else -#endif - { - mask |= (_mm_movemask_epi8(match2NlDotA) << 2); - mask |= (_mm_movemask_epi8(match2NlDotB) << 18) & 0xffffffff; - match2NlDotB = _mm_srli_si128(match2NlDotB, 14); - if(_USING_FAST_MATCH) - minMask = _mm_subs_epu8(_mm_set1_epi8('.'), match2NlDotB); - else - // this bitiwse trick works because '.'|'\n' == '.' - lfCompare = _mm_or_si128( - _mm_and_si128(match2NlDotB, _mm_set1_epi8('.')), - _mm_set1_epi8('\n') - ); - } - } - else if(searchEnd) { - bool partialEndFound; - __m128i match3EqYA, match3EqYB; -#if defined(__AVX512VL__) && defined(__AVX512BW__) - __mmask16 match3EqYMaskA, match3EqYMaskB; - if(useAVX3MaskCmp) { - match3EqYMaskA = _mm_mask_cmpeq_epi8_mask( - match2EqMaskA, - _mm_set1_epi8('y'), - SHIFT_DATA_A(3) - ); - match3EqYMaskB = _mm_mask_cmpeq_epi8_mask( - match2EqMaskB, - _mm_set1_epi8('y'), - SHIFT_DATA_B(3) - ); - partialEndFound = KORTEST16(match3EqYMaskA, match3EqYMaskB); - } else -#endif - { - __m128i match3YA = _mm_cmpeq_epi8( - _mm_set1_epi8('y'), - SHIFT_DATA_A(3) - ); - __m128i match3YB = _mm_cmpeq_epi8( - _mm_set1_epi8('y'), - SHIFT_DATA_B(3) - ); -#if defined(__SSSE3__) && !defined(__tune_btver1__) - if(use_isa >= ISA_LEVEL_SSSE3) - match2EqA = _mm_alignr_epi8(cmpEqB, cmpEqA, 2); -#endif - match3EqYA = _mm_and_si128(match2EqA, match3YA); - match3EqYB = _mm_and_si128(match2EqB, match3YB); - partialEndFound = _mm_movemask_epi8(_mm_or_si128(match3EqYA, match3EqYB)); - } - if(LIKELIHOOD(0.001, partialEndFound)) { - // if the rare case of '=y' is found, do a more precise check - bool endFound; - -#if defined(__AVX512VL__) && defined(__AVX512BW__) - if(useAVX3MaskCmp) { - __mmask16 match3LfEqYMaskA = _mm_mask_cmpeq_epi8_mask( - match3EqYMaskA, - _mm_set1_epi8('\n'), - SHIFT_DATA_A(1) - ); - __mmask16 match3LfEqYMaskB = _mm_mask_cmpeq_epi8_mask( - match3EqYMaskB, - _mm_set1_epi8('\n'), - SHIFT_DATA_B(1) - ); - - endFound = KORTEST16( - _mm_mask_cmpeq_epi8_mask(match3LfEqYMaskA, oDataA, _mm_set1_epi8('\r')), - _mm_mask_cmpeq_epi8_mask(match3LfEqYMaskB, oDataB, _mm_set1_epi8('\r')) - ); - } else -#endif - { - // always recompute cmpCr to avoid register spills above - cmpCrA = _mm_cmpeq_epi8(force_align_read_128(src+i), _mm_set1_epi8('\r')); - cmpCrB = _mm_cmpeq_epi8(force_align_read_128(src+i + sizeof(__m128i)), _mm_set1_epi8('\r')); - __m128i match1LfA = _mm_cmpeq_epi8( - _mm_set1_epi8('\n'), - SHIFT_DATA_A(1) - ); - __m128i match1LfB = _mm_cmpeq_epi8( - _mm_set1_epi8('\n'), - SHIFT_DATA_B(1) - ); - endFound = _mm_movemask_epi8(_mm_or_si128( - _mm_and_si128( - match3EqYA, - _mm_and_si128(match1LfA, cmpCrA) - ), - _mm_and_si128( - match3EqYB, - _mm_and_si128(match1LfB, cmpCrB) - ) - )); - } - - if(endFound) { - len += (long)i; - _nextMask = decoder_set_nextMask(src+i, mask); - break; - } - } - if(isRaw) { - if(_USING_FAST_MATCH) - minMask = _mm_set1_epi8('.'); - else - lfCompare = _mm_set1_epi8('\n'); - } - } - else if(isRaw) { // no \r_. found - if(_USING_FAST_MATCH) - minMask = _mm_set1_epi8('.'); - else - lfCompare = _mm_set1_epi8('\n'); - } - } -#undef SHIFT_DATA_A -#undef SHIFT_DATA_B - - if(!_USING_BLEND_ADD) - dataB = _mm_add_epi8(oDataB, _mm_set1_epi8(-42)); - - uint32_t maskEqShift1 = (maskEq << 1) + escFirst; - if(LIKELIHOOD(0.0001, (mask & maskEqShift1) != 0)) { - maskEq = fix_eqMask(maskEq, maskEqShift1); - mask &= ~escFirst; - escFirst = maskEq >> 31; - // next, eliminate anything following a `=` from the special char mask; this eliminates cases of `=\r` so that they aren't removed - maskEq <<= 1; - mask &= ~maskEq; - - if(_USING_BLEND_ADD) { - dataA = _mm_add_epi8(oDataA, yencOffset); - dataB = _mm_add_epi8(oDataB, _mm_set1_epi8(-42)); - } - - // unescape chars following `=` -#if defined(__AVX512VL__) && defined(__AVX512BW__) - if(use_isa >= ISA_LEVEL_AVX3) { - // GCC < 7 seems to generate rubbish assembly for this - dataA = _mm_mask_add_epi8( - dataA, - (__mmask16)maskEq, - dataA, - _mm_set1_epi8(-64) - ); - dataB = _mm_mask_add_epi8( - dataB, - (__mmask16)(maskEq>>16), - dataB, - _mm_set1_epi8(-64) - ); - } else -#endif - { - dataA = _mm_add_epi8( - dataA, - LOAD_HALVES( - lookups->eqAdd + (maskEq&0xff), - lookups->eqAdd + ((maskEq>>8)&0xff) - ) - ); - maskEq >>= 16; - dataB = _mm_add_epi8( - dataB, - LOAD_HALVES( - lookups->eqAdd + (maskEq&0xff), - lookups->eqAdd + ((maskEq>>8)&0xff) - ) - ); - - yencOffset = _mm_xor_si128(_mm_set1_epi8(-42), - _mm_slli_epi16(_mm_cvtsi32_si128((int)escFirst), 6) - ); - } - } else { - // no invalid = sequences found - we can cut out some things from above - // this code path is a shortened version of above; it's here because it's faster, and what we'll be dealing with most of the time - escFirst = (maskEq >> 31); - -#if defined(__AVX512VL__) && defined(__AVX512BW__) - if(use_isa >= ISA_LEVEL_AVX3) { - dataA = _mm_add_epi8( - oDataA, - _mm_ternarylogic_epi32( - _mm_slli_si128(cmpEqA, 1), yencOffset, _mm_set1_epi8(-42-64), 0xac - ) - ); - dataB = _mm_add_epi8( - oDataB, - _mm_ternarylogic_epi32( - _mm_alignr_epi8(cmpEqB, cmpEqA, 15), _mm_set1_epi8(-42), _mm_set1_epi8(-42-64), 0xac - ) - ); - } else -#endif -#if defined(__SSE4_1__) - if(_USING_BLEND_ADD) { - /* // the following strategy seems more ideal, however, both GCC and Clang go bonkers over it and spill more registers - cmpEqA = _mm_blendv_epi8(_mm_set1_epi8(-42), _mm_set1_epi8(-42-64), cmpEqA); - cmpEqB = _mm_blendv_epi8(_mm_set1_epi8(-42), _mm_set1_epi8(-42-64), cmpEqB); - dataB = _mm_add_epi8(oDataB, _mm_alignr_epi8(cmpEqB, cmpEqA, 15)); - dataA = _mm_add_epi8(oDataA, _mm_and_si128( - _mm_alignr_epi8(cmpEqA, _mm_set1_epi8(-42), 15), - yencOffset - )); - yencOffset = _mm_alignr_epi8(_mm_set1_epi8(-42), cmpEqB, 15); - */ - - dataA = _mm_add_epi8( - oDataA, - _mm_blendv_epi8( - yencOffset, _mm_set1_epi8(-42-64), _mm_slli_si128(cmpEqA, 1) - ) - ); - dataB = _mm_add_epi8( - oDataB, - _mm_blendv_epi8( - _mm_set1_epi8(-42), _mm_set1_epi8(-42-64), _mm_alignr_epi8(cmpEqB, cmpEqA, 15) - ) - ); - yencOffset = _mm_xor_si128(_mm_set1_epi8(-42), - _mm_slli_epi16(_mm_cvtsi32_si128((int)escFirst), 6) - ); - } else -#endif - { - cmpEqA = _mm_and_si128(cmpEqA, _mm_set1_epi8(-64)); - cmpEqB = _mm_and_si128(cmpEqB, _mm_set1_epi8(-64)); - yencOffset = _mm_add_epi8(_mm_set1_epi8(-42), _mm_srli_si128(cmpEqB, 15)); -#if defined(__SSSE3__) && !defined(__tune_btver1__) - if(use_isa >= ISA_LEVEL_SSSE3) - cmpEqB = _mm_alignr_epi8(cmpEqB, cmpEqA, 15); - else -#endif - cmpEqB = _mm_or_si128( - _mm_slli_si128(cmpEqB, 1), - _mm_srli_si128(cmpEqA, 15) - ); - cmpEqA = _mm_slli_si128(cmpEqA, 1); - dataA = _mm_add_epi8(dataA, cmpEqA); - dataB = _mm_add_epi8(dataB, cmpEqB); - } - } - // subtract 64 from first element if escFirst == 1 -#if defined(__AVX512VL__) && defined(__AVX512BW__) - if(use_isa >= ISA_LEVEL_AVX3) { - yencOffset = _mm_mask_add_epi8(_mm_set1_epi8(-42), (__mmask16)escFirst, _mm_set1_epi8(-42), _mm_set1_epi8(-64)); - } -#endif - - // all that's left is to 'compress' the data (skip over masked chars) -#ifdef __SSSE3__ - if(use_isa >= ISA_LEVEL_SSSE3) { -# if defined(__AVX512VBMI2__) && defined(__AVX512VL__) && defined(__POPCNT__) - if(use_isa >= ISA_LEVEL_VBMI2) { - COMPRESS_STORE(p, KNOT16(mask), dataA); - p -= popcnt32(mask & 0xffff); - COMPRESS_STORE(p+XMM_SIZE, KNOT16(mask>>16), dataB); - p -= popcnt32(mask>>16); - p += XMM_SIZE*2; - } else -# endif - { - - dataA = _mm_shuffle_epi8(dataA, _mm_load_si128((__m128i*)(lookups->compact + (mask&0x7fff)))); - STOREU_XMM(p, dataA); - - dataB = _mm_shuffle_epi8(dataB, _mm_load_si128((__m128i*)((char*)lookups->compact + ((mask >> 12) & 0x7fff0)))); - -# if defined(__POPCNT__) && !defined(__tune_btver1__) - if(use_isa & ISA_FEATURE_POPCNT) { - p -= popcnt32(mask & 0xffff); - STOREU_XMM(p+XMM_SIZE, dataB); - p -= popcnt32(mask & 0xffff0000); - p += XMM_SIZE*2; - } else -# endif - { - p += lookups->BitsSetTable256inv[mask & 0xff] + lookups->BitsSetTable256inv[(mask >> 8) & 0xff]; - STOREU_XMM(p, dataB); - mask >>= 16; - p += lookups->BitsSetTable256inv[mask & 0xff] + lookups->BitsSetTable256inv[(mask >> 8) & 0xff]; - } - } - } else -#endif - { - dataA = sse2_compact_vect(mask & 0xffff, dataA); - STOREU_XMM(p, dataA); - p += lookups->BitsSetTable256inv[mask & 0xff] + lookups->BitsSetTable256inv[(mask >> 8) & 0xff]; - mask >>= 16; - dataB = sse2_compact_vect(mask, dataB); - STOREU_XMM(p, dataB); - p += lookups->BitsSetTable256inv[mask & 0xff] + lookups->BitsSetTable256inv[(mask >> 8) & 0xff]; - } -#undef LOAD_HALVES - } else { - if(_USING_BLEND_ADD) - dataA = _mm_add_epi8(oDataA, yencOffset); - dataB = _mm_add_epi8(oDataB, _mm_set1_epi8(-42)); - - STOREU_XMM(p, dataA); - STOREU_XMM(p+XMM_SIZE, dataB); - p += XMM_SIZE*2; - escFirst = 0; - yencOffset = _mm_set1_epi8(-42); - } - } - _escFirst = (unsigned char)escFirst; -} -} // namespace -#endif diff --git a/rapidyenc/rapidyenc/src/decoder_ssse3.cc b/rapidyenc/rapidyenc/src/decoder_ssse3.cc deleted file mode 100644 index 25fd671..0000000 --- a/rapidyenc/rapidyenc/src/decoder_ssse3.cc +++ /dev/null @@ -1,18 +0,0 @@ -#include "common.h" - -#include "decoder_common.h" -#ifdef __SSSE3__ -#include "decoder_sse_base.h" -void RapidYenc::decoder_set_ssse3_funcs() { - decoder_sse_init(lookups); - decoder_init_lut(lookups->compact); - _do_decode = &do_decode_simd >; - _do_decode_raw = &do_decode_simd >; - _do_decode_end_raw = &do_decode_simd >; - _decode_isa = ISA_LEVEL_SSSE3; -} -#else -void RapidYenc::decoder_set_ssse3_funcs() { - decoder_set_sse2_funcs(); -} -#endif diff --git a/rapidyenc/rapidyenc/src/decoder_vbmi2.cc b/rapidyenc/rapidyenc/src/decoder_vbmi2.cc deleted file mode 100644 index 53682ad..0000000 --- a/rapidyenc/rapidyenc/src/decoder_vbmi2.cc +++ /dev/null @@ -1,32 +0,0 @@ -#include "common.h" -# include "decoder_common.h" - -#if !defined(__EVEX512__) && (defined(__AVX10_1__) || defined(__EVEX256__)) && defined(__AVX512VL__) && defined(__AVX512VBMI2__) && defined(__AVX512BW__) -const bool RapidYenc::decoder_has_avx10 = true; -#else -const bool RapidYenc::decoder_has_avx10 = false; -#endif - -#if defined(__AVX512VL__) && defined(__AVX512VBMI2__) && defined(__AVX512BW__) -# ifndef YENC_DISABLE_AVX256 -# include "decoder_avx2_base.h" -void RapidYenc::decoder_set_vbmi2_funcs() { - _do_decode = &do_decode_simd >; - _do_decode_raw = &do_decode_simd >; - _do_decode_end_raw = &do_decode_simd >; - _decode_isa = ISA_LEVEL_VBMI2; -} -# else -# include "decoder_sse_base.h" -void RapidYenc::decoder_set_vbmi2_funcs() { - _do_decode = &do_decode_simd >; - _do_decode_raw = &do_decode_simd >; - _do_decode_end_raw = &do_decode_simd >; - _decode_isa = ISA_LEVEL_VBMI2; -} -# endif -#else -void RapidYenc::decoder_set_vbmi2_funcs() { - decoder_set_avx2_funcs(); -} -#endif diff --git a/rapidyenc/rapidyenc/src/encoder.cc b/rapidyenc/rapidyenc/src/encoder.cc deleted file mode 100644 index b0642fb..0000000 --- a/rapidyenc/rapidyenc/src/encoder.cc +++ /dev/null @@ -1,198 +0,0 @@ -#include "common.h" -#include "encoder_common.h" -#include "encoder.h" - - -// lookup tables for scalar processing -#define _B1(n) _B(n), _B(n+1), _B(n+2), _B(n+3) -#define _B2(n) _B1(n), _B1(n+4), _B1(n+8), _B1(n+12) -#define _B3(n) _B2(n), _B2(n+16), _B2(n+32), _B2(n+48) -#define _BX _B3(0), _B3(64), _B3(128), _B3(192) - -const unsigned char RapidYenc::escapeLUT[256] = { // whether or not the character is critical -#define _B(n) ((n == 214 || n == '\r'+214 || n == '\n'+214 || n == '='-42) ? 0 : (n+42) & 0xff) - _BX -#undef _B -}; -const uint16_t RapidYenc::escapedLUT[256] = { // escaped sequences for characters that need escaping -#define _B(n) ((n == 214 || n == 214+'\r' || n == 214+'\n' || n == '='-42 || n == 214+'\t' || n == 214+' ' || n == '.'-42) ? UINT16_PACK('=', ((n+42+64)&0xff)) : 0) - _BX -#undef _B -}; - -#undef _B1 -#undef _B2 -#undef _B3 -#undef _BX - - - -size_t RapidYenc::do_encode_generic(int line_size, int* colOffset, const unsigned char* HEDLEY_RESTRICT src, unsigned char* HEDLEY_RESTRICT dest, size_t len, int doEnd) { - unsigned char* es = (unsigned char*)src + len; - unsigned char *p = dest; // destination pointer - long i = -(long)len; // input position - unsigned char c, escaped; // input character; escaped input character - int col = *colOffset; - - if (col == 0) { - c = es[i++]; - if (RapidYenc::escapedLUT[c]) { - memcpy(p, &RapidYenc::escapedLUT[c], sizeof(uint16_t)); - p += 2; - col = 2; - } else { - *(p++) = c + 42; - col = 1; - } - } - while(i < 0) { - // main line - unsigned char* sp = NULL; - while (i < -1-8 && line_size-col-1 > 8) { - // 8 cycle unrolled version - sp = p; - #define DO_THING(n) \ - c = es[i+n], escaped = RapidYenc::escapeLUT[c]; \ - if (escaped) \ - *(p++) = escaped; \ - else { \ - memcpy(p, &RapidYenc::escapedLUT[c], sizeof(uint16_t)); \ - p += 2; \ - } - DO_THING(0); - DO_THING(1); - DO_THING(2); - DO_THING(3); - DO_THING(4); - DO_THING(5); - DO_THING(6); - DO_THING(7); - - i += 8; - col += (int)(p - sp); - } - if(sp && col >= line_size-1) { - // TODO: consider revert optimisation from SIMD code - // we overflowed - need to revert and use slower method :( - col -= (int)(p - sp); - p = sp; - i -= 8; - } - // handle remaining chars - while(col < line_size-1) { - c = es[i++], escaped = RapidYenc::escapeLUT[c]; - if (escaped) { - *(p++) = escaped; - col++; - } - else { - memcpy(p, &RapidYenc::escapedLUT[c], sizeof(uint16_t)); - p += 2; - col += 2; - } - /* experimental branchless version - *p = '='; - c = (es[i++] + 42) & 0xFF; - int cond = (c=='\0' || c=='=' || c=='\r' || c=='\n'); - *(p+cond) = c + (cond << 6); - p += 1+cond; - col += 1+cond; - */ - if (i >= 0) goto end; - } - - // last line char - if(col < line_size) { // this can only be false if the last character was an escape sequence (or line_size is horribly small), in which case, we don't need to handle space/tab cases - c = es[i++]; - if (RapidYenc::escapedLUT[c] && c != '.'-42) { - memcpy(p, &RapidYenc::escapedLUT[c], sizeof(uint16_t)); - p += 2; - } else { - *(p++) = c + 42; - } - } - - if (i >= 0) break; - - c = es[i++]; - if (RapidYenc::escapedLUT[c]) { - uint32_t w = UINT32_16_PACK(UINT16_PACK('\r', '\n'), (uint32_t)RapidYenc::escapedLUT[c]); - memcpy(p, &w, sizeof(w)); - p += 4; - col = 2; - } else { - // another option may be to just write the EOL and let the first char be handled by the faster methods above, but it appears that writing the extra byte here is generally faster... - uint32_t w = UINT32_PACK('\r', '\n', (uint32_t)(c+42), 0); - memcpy(p, &w, sizeof(w)); - p += 3; - col = 1; - } - } - - end: - if(doEnd) { - // special case: if the last character is a space/tab, it needs to be escaped as it's the final character on the line - unsigned char lc = *(p-1); - if(lc == '\t' || lc == ' ') { - *(p-1) = '='; - *p = lc+64; - p++; - col++; - } - } - *colOffset = col; - return p - dest; -} - - -namespace RapidYenc { - size_t (*_do_encode)(int, int*, const unsigned char* HEDLEY_RESTRICT, unsigned char* HEDLEY_RESTRICT, size_t, int) = &do_encode_generic; - int _encode_isa = ISA_GENERIC; -} - -#if defined(PLATFORM_X86) && defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0 -# if defined(__AVX2__) && !defined(YENC_DISABLE_AVX256) -# include "encoder_avx_base.h" -static inline void encoder_native_init() { - RapidYenc::_do_encode = &do_encode_simd< RapidYenc::do_encode_avx2 >; - encoder_avx2_lut(); - RapidYenc::_encode_isa = ISA_NATIVE; -} -# else -# include "encoder_sse_base.h" -static inline void encoder_native_init() { - RapidYenc::_do_encode = &do_encode_simd< RapidYenc::do_encode_sse >; - encoder_sse_lut(); - RapidYenc::_encode_isa = ISA_NATIVE; -} -# endif -#endif - - -void RapidYenc::encoder_init() { -#ifdef PLATFORM_X86 -# if defined(YENC_BUILD_NATIVE) && YENC_BUILD_NATIVE!=0 - encoder_native_init(); -# else - int use_isa = cpu_supports_isa(); - if(use_isa >= ISA_LEVEL_VBMI2 && (encoder_has_avx10 || (use_isa & ISA_FEATURE_EVEX512))) - encoder_vbmi2_init(); - else if(use_isa >= ISA_LEVEL_AVX2) - encoder_avx2_init(); - else if(use_isa >= ISA_LEVEL_AVX) - encoder_avx_init(); - else if(use_isa >= ISA_LEVEL_SSSE3) - encoder_ssse3_init(); - else - encoder_sse2_init(); -# endif -#endif -#ifdef PLATFORM_ARM - if(cpu_supports_neon()) - encoder_neon_init(); -#endif -#ifdef __riscv - if(cpu_supports_rvv()) - encoder_rvv_init(); -#endif -} diff --git a/rapidyenc/rapidyenc/src/encoder.h b/rapidyenc/rapidyenc/src/encoder.h deleted file mode 100644 index 2862a91..0000000 --- a/rapidyenc/rapidyenc/src/encoder.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef __YENC_ENCODER_H -#define __YENC_ENCODER_H - -#include "hedley.h" - -namespace RapidYenc { - - - -extern size_t (*_do_encode)(int, int*, const unsigned char* HEDLEY_RESTRICT, unsigned char* HEDLEY_RESTRICT, size_t, int); -extern int _encode_isa; -static inline size_t encode(int line_size, int* colOffset, const void* HEDLEY_RESTRICT src, void* HEDLEY_RESTRICT dest, size_t len, int doEnd) { - return (*_do_encode)(line_size, colOffset, (const unsigned char* HEDLEY_RESTRICT)src, (unsigned char*)dest, len, doEnd); -} -void encoder_init(); -static inline int encode_isa_level() { - return _encode_isa; -} - - - -} -#endif diff --git a/rapidyenc/rapidyenc/src/encoder_avx.cc b/rapidyenc/rapidyenc/src/encoder_avx.cc deleted file mode 100644 index 929c953..0000000 --- a/rapidyenc/rapidyenc/src/encoder_avx.cc +++ /dev/null @@ -1,17 +0,0 @@ -#include "common.h" -#include "encoder_common.h" - -#if defined(__AVX__) && defined(__POPCNT__) -#include "encoder_sse_base.h" - -void RapidYenc::encoder_avx_init() { - _do_encode = &do_encode_simd< do_encode_sse >; - encoder_sse_lut(); - _encode_isa = ISA_LEVEL_AVX; -} -#else -void RapidYenc::encoder_avx_init() { - encoder_ssse3_init(); -} -#endif - diff --git a/rapidyenc/rapidyenc/src/encoder_avx2.cc b/rapidyenc/rapidyenc/src/encoder_avx2.cc deleted file mode 100644 index 10b9e30..0000000 --- a/rapidyenc/rapidyenc/src/encoder_avx2.cc +++ /dev/null @@ -1,17 +0,0 @@ -#include "common.h" -#include "encoder_common.h" - -#if defined(__AVX2__) && !defined(YENC_DISABLE_AVX256) -#include "encoder_avx_base.h" - -void RapidYenc::encoder_avx2_init() { - _do_encode = &do_encode_simd< do_encode_avx2 >; - encoder_avx2_lut(); - _encode_isa = ISA_LEVEL_AVX2; -} -#else -void RapidYenc::encoder_avx2_init() { - encoder_avx_init(); -} -#endif - diff --git a/rapidyenc/rapidyenc/src/encoder_avx_base.h b/rapidyenc/rapidyenc/src/encoder_avx_base.h deleted file mode 100644 index 7185f74..0000000 --- a/rapidyenc/rapidyenc/src/encoder_avx_base.h +++ /dev/null @@ -1,575 +0,0 @@ -// can't seem to make this worth it -#include "common.h" -#ifdef __AVX2__ - -#include "encoder.h" -#include "encoder_common.h" -#define YMM_SIZE 32 - -#if (defined(__GNUC__) && __GNUC__ >= 7) || (defined(_MSC_VER) && _MSC_VER >= 1924) -# define KLOAD32(a, offs) _load_mask32((__mmask32*)(a) + (offs)) -#else -# define KLOAD32(a, offs) (((uint32_t*)(a))[(offs)]) -#endif - -#pragma pack(16) -static struct { - uint32_t eolLastChar[256]; - /*align32*/ __m256i shufExpand[65536]; // huge 2MB table - /*align32*/ int8_t expandMergemix[33*2*32]; // not used in AVX3 -} * HEDLEY_RESTRICT lookupsAVX2; -static struct { - uint32_t eolLastChar[256]; - uint32_t expand[65536]; // biggish 256KB table (but still smaller than the 2MB table) -} * HEDLEY_RESTRICT lookupsVBMI2; -#pragma pack() - -static inline void fill_eolLastChar(uint32_t* table) { - for(int n=0; n<256; n++) { - table[n] = ((n == 214+'\t' || n == 214+' ' || n == 214+'\0' || n == 214+'\n' || n == 214+'\r' || n == '='-42) ? (((n+42+64)&0xff)<<8)+0x0a0d003d : ((n+42)&0xff)+0x0a0d00); - } -} - -template -static void encoder_avx2_lut() { - if(use_isa >= ISA_LEVEL_VBMI2) { - ALIGN_ALLOC(lookupsVBMI2, sizeof(*lookupsVBMI2), 32); - fill_eolLastChar(lookupsVBMI2->eolLastChar); - for(int i=0; i<65536; i++) { - int k = i; - uint32_t expand = 0; - int p = 0; - for(int j=0; j<16; j++) { - if(k & 1) { - p++; - } - expand |= 1<<(j+p); - k >>= 1; - } - lookupsVBMI2->expand[i] = expand; - } - } else { - ALIGN_ALLOC(lookupsAVX2, sizeof(*lookupsAVX2), 32); - fill_eolLastChar(lookupsAVX2->eolLastChar); - for(int i=0; i<65536; i++) { - int k = i; - uint8_t* res = (uint8_t*)(lookupsAVX2->shufExpand + i); - int p = 0; - for(int j=0; j<16; j++) { - if(k & 1) { - res[j+p] = 0xff; - p++; - } - res[j+p] = j; - k >>= 1; - } - for(; p<16; p++) - res[16+p] = 0x40; // arbitrary value (top bit cannot be set) - } - for(int i=0; i<33; i++) { - int n = (i == 32 ? 32 : 31-i); - for(int j=0; j<32; j++) { - lookupsAVX2->expandMergemix[i*64 + j] = (n>=j ? -1 : 0); - lookupsAVX2->expandMergemix[i*64 + j + 32] = ('='*(n==j) + 64*(n==j-1) + 42*(n!=j)); - } - } - } -} - -namespace RapidYenc { - -template -HEDLEY_ALWAYS_INLINE void do_encode_avx2(int line_size, int* colOffset, const uint8_t* HEDLEY_RESTRICT srcEnd, uint8_t* HEDLEY_RESTRICT& dest, size_t& len) { - // offset position to enable simpler loop condition checking - const int INPUT_OFFSET = YMM_SIZE*4 + 1 -1; // -1 to change <= to < - if(len <= INPUT_OFFSET || line_size < 16) return; - - uint8_t *p = dest; // destination pointer - intptr_t i = -(intptr_t)len; // input position - intptr_t lineSizeOffset = -line_size +1; // -1 because we want to stop one char before the end to handle the last char differently - intptr_t col = *colOffset + lineSizeOffset; - - i += INPUT_OFFSET; - const uint8_t* es = srcEnd - INPUT_OFFSET; - -#if !defined(__tune_bdver4__) && !defined(__tune_znver1__) - // always process at least one byte to prevent underflow when doing a read with -1 offset - if(col < 0 && col != -line_size+1) { - // not the first/last character of a line - uint8_t c = es[i++]; - if(HEDLEY_UNLIKELY(c == 214 || c == '\n'+214 || c == '\r'+214 || c == '='-42)) { - *(uint16_t*)p = 0x6a3d + (((uint16_t)c) << 8); - p += 2; - col += 2; - } else { - *p++ = c+42; - col++; - } - } -#endif - - if(HEDLEY_UNLIKELY(col >= 0)) { - uint8_t c = es[i++]; - if(col == 0) { - // last char - uint32_t eolChar = (use_isa >= ISA_LEVEL_VBMI2 ? lookupsVBMI2->eolLastChar[c] : lookupsAVX2->eolLastChar[c]); - *(uint32_t*)p = eolChar; - p += 3 + (uintptr_t)(eolChar>>27); - col = -line_size+1; - } else { - // line overflowed, insert a newline - if (LIKELIHOOD(0.0273, escapedLUT[c]!=0)) { - *(uint32_t*)p = UINT32_16_PACK(UINT16_PACK('\r', '\n'), (uint32_t)escapedLUT[c]); - p += 4; - col = 2-line_size + 1; - } else { - *(uint32_t*)p = UINT32_PACK('\r', '\n', (uint32_t)(c+42), 0); - p += 3; - col = 2-line_size; - } - } - } - if (HEDLEY_LIKELY(col == -line_size+1)) { - // first char of the line - uint8_t c = es[i++]; - if (LIKELIHOOD(0.0273, escapedLUT[c] != 0)) { - *(uint16_t*)p = escapedLUT[c]; - p += 2; - col += 2; - } else { - *(p++) = c + 42; - col += 1; - } - } - do { - __m256i dataA = _mm256_loadu_si256((__m256i *)(es + i)); - __m256i dataB = _mm256_loadu_si256((__m256i *)(es + i) + 1); - i += YMM_SIZE*2; - // search for special chars - __m256i cmpA = _mm256_cmpeq_epi8( - _mm256_shuffle_epi8(_mm256_set_epi8( - '\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42, - '\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42 - ), _mm256_abs_epi8(dataA)), - dataA - ); - __m256i cmpB = _mm256_cmpeq_epi8( - _mm256_shuffle_epi8(_mm256_set_epi8( - '\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42, - '\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42 - ), _mm256_abs_epi8(dataB)), - dataB - ); - -#if defined(__AVX512VL__) - if(use_isa >= ISA_LEVEL_AVX3) { - dataA = _mm256_add_epi8(dataA, _mm256_set1_epi8(42)); - dataA = _mm256_ternarylogic_epi32(dataA, cmpA, _mm256_set1_epi8(64), 0xf8); // data | (cmp & 64) - dataB = _mm256_add_epi8(dataB, _mm256_set1_epi8(42)); - dataB = _mm256_ternarylogic_epi32(dataB, cmpB, _mm256_set1_epi8(64), 0xf8); // data | (cmp & 64) - } -#endif - - uint32_t maskA = (uint32_t)_mm256_movemask_epi8(cmpA); - uint32_t maskB = (uint32_t)_mm256_movemask_epi8(cmpB); - unsigned int maskBitsA = popcnt32(maskA); - unsigned int maskBitsB = popcnt32(maskB); - unsigned int outputBytesA = maskBitsA + YMM_SIZE; - unsigned int bitIndexA, bitIndexB; - if (LIKELIHOOD(0.170, (maskBitsA|maskBitsB) > 1)) { - _encode_loop_branch_slow: - unsigned int m1 = maskA & 0xffff, m3 = maskB & 0xffff; - unsigned int m2, m4; - __m256i data1A, data2A; - __m256i data1B, data2B; - __m256i shuf1A, shuf1B; // not set in VBMI2 path - __m256i shuf2A, shuf2B; // not set in VBMI2 path - -#if defined(__AVX512VBMI2__) && defined(__AVX512VL__) && defined(__AVX512BW__) - if(use_isa >= ISA_LEVEL_VBMI2) { - m2 = maskA >> 16; - m4 = maskB >> 16; - - /* alternative no-LUT strategy - uint64_t expandMaskA = ~_pdep_u64(~maskA, 0x5555555555555555); // expand bits, with bits set - expandMaskA = _pext_u64(expandMaskA^0x5555555555555555, expandMaskA); - */ - - data1A = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KLOAD32(lookupsVBMI2->expand, m1), dataA); - data2A = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KLOAD32(lookupsVBMI2->expand, m2), _mm256_castsi128_si256( - _mm256_extracti128_si256(dataA, 1) - )); - data1B = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KLOAD32(lookupsVBMI2->expand, m3), dataB); - data2B = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KLOAD32(lookupsVBMI2->expand, m4), _mm256_castsi128_si256( - _mm256_extracti128_si256(dataB, 1) - )); - } else -#endif - { - if(use_isa < ISA_LEVEL_AVX3) { - dataA = _mm256_add_epi8(dataA, _mm256_blendv_epi8(_mm256_set1_epi8(42), _mm256_set1_epi8(42+64), cmpA)); - dataB = _mm256_add_epi8(dataB, _mm256_blendv_epi8(_mm256_set1_epi8(42), _mm256_set1_epi8(42+64), cmpB)); - } - - m2 = (maskA >> 11) & 0x1fffe0; - m4 = (maskB >> 11) & 0x1fffe0; - - // duplicate halves - data1A = _mm256_inserti128_si256(dataA, _mm256_castsi256_si128(dataA), 1); - data1B = _mm256_inserti128_si256(dataB, _mm256_castsi256_si128(dataB), 1); -#if defined(__tune_znver1__) || defined(__tune_bdver4__) - data2A = _mm256_permute4x64_epi64(dataA, 0xee); - data2B = _mm256_permute4x64_epi64(dataB, 0xee); -#else - data2A = _mm256_permute2x128_si256(dataA, dataA, 0x11); - data2B = _mm256_permute2x128_si256(dataB, dataB, 0x11); -#endif - - shuf1A = _mm256_load_si256(lookupsAVX2->shufExpand + m1); - shuf2A = _mm256_load_si256((__m256i*)((char*)(lookupsAVX2->shufExpand) + m2)); - shuf1B = _mm256_load_si256(lookupsAVX2->shufExpand + m3); - shuf2B = _mm256_load_si256((__m256i*)((char*)(lookupsAVX2->shufExpand) + m4)); - - // expand - data1A = _mm256_shuffle_epi8(data1A, shuf1A); - data2A = _mm256_shuffle_epi8(data2A, shuf2A); - data1B = _mm256_shuffle_epi8(data1B, shuf1B); - data2B = _mm256_shuffle_epi8(data2B, shuf2B); - // add in '=' - data1A = _mm256_blendv_epi8(data1A, _mm256_set1_epi8('='), shuf1A); - data2A = _mm256_blendv_epi8(data2A, _mm256_set1_epi8('='), shuf2A); - data1B = _mm256_blendv_epi8(data1B, _mm256_set1_epi8('='), shuf1B); - data2B = _mm256_blendv_epi8(data2B, _mm256_set1_epi8('='), shuf2B); - } - - unsigned int shuf1Len = popcnt32(m1) + 16; - unsigned int shuf3Len = popcnt32(m3) + 16; - _mm256_storeu_si256((__m256i*)p, data1A); - _mm256_storeu_si256((__m256i*)(p + shuf1Len), data2A); - _mm256_storeu_si256((__m256i*)(p + outputBytesA), data1B); - _mm256_storeu_si256((__m256i*)(p + outputBytesA + shuf3Len), data2B); - unsigned int outputBytes = YMM_SIZE + outputBytesA + maskBitsB; - p += outputBytes; - col += outputBytes; - - if(col >= 0) { - // we overflowed - find correct position to revert back to - // this is perhaps sub-optimal on 32-bit, but who still uses that with AVX2? - uint64_t eqMask; - int shiftAmt = (int)(maskBitsB + YMM_SIZE -1 - col); - if(HEDLEY_UNLIKELY(shiftAmt < 0)) { - uint32_t eqMask1, eqMask2; -#if defined(__AVX512VBMI2__) && defined(__AVX512VL__) && defined(__AVX512BW__) - if(use_isa >= ISA_LEVEL_VBMI2) { - eqMask1 = lookupsVBMI2->expand[m1]; - eqMask2 = lookupsVBMI2->expand[m2]; - } else -#endif - { - eqMask1 = (uint32_t)_mm256_movemask_epi8(shuf1A); - eqMask2 = (uint32_t)_mm256_movemask_epi8(shuf2A); - } - eqMask = eqMask1 | ((uint64_t)eqMask2 << shuf1Len); - if(use_isa < ISA_LEVEL_VBMI2) - i += (uintptr_t)maskBitsB; - else - i -= YMM_SIZE; - shiftAmt += outputBytesA; - } else { - uint32_t eqMask3, eqMask4; -#if defined(__AVX512VBMI2__) && defined(__AVX512VL__) && defined(__AVX512BW__) - if(use_isa >= ISA_LEVEL_VBMI2) { - eqMask3 = lookupsVBMI2->expand[m3]; - eqMask4 = lookupsVBMI2->expand[m4]; - } else -#endif - { - eqMask3 = (uint32_t)_mm256_movemask_epi8(shuf1B); - eqMask4 = (uint32_t)_mm256_movemask_epi8(shuf2B); - } - eqMask = eqMask3 | ((uint64_t)eqMask4 << shuf3Len); - } - -#if defined(__GNUC__) && defined(PLATFORM_AMD64) - if(use_isa >= ISA_LEVEL_VBMI2) { - __asm__( - "shrq $1, %[eqMask] \n" - "shrq %%cl, %[eqMask] \n" - "adcq %q[col], %q[p] \n" - : [eqMask]"+r"(eqMask), [p]"+r"(p) - : "c"(shiftAmt), [col]"r"(~col) - ); - i -= _mm_popcnt_u64(eqMask); - } else -#endif - { - eqMask >>= shiftAmt; - unsigned int bitCount; -#ifdef PLATFORM_AMD64 - bitCount = (unsigned int)_mm_popcnt_u64(eqMask); -#else - bitCount = popcnt32(eqMask & 0xffffffff) + popcnt32(eqMask >> 32); -#endif -#if defined(__AVX512VBMI2__) && defined(__AVX512VL__) && defined(__AVX512BW__) - if(use_isa >= ISA_LEVEL_VBMI2) { - i -= bitCount; - p -= col; - if(LIKELIHOOD(0.98, (eqMask & 1) != 1)) - p--; - else - i++; - } else -#endif - { - i += bitCount; - unsigned int revert = (unsigned int)(col + (eqMask & 1)); - p -= revert; - i -= revert; - } - } - goto _encode_eol_handle_pre; - } - } else { - //_encode_loop_branch_fast: - maskBitsB += YMM_SIZE; -#if defined(__AVX512VL__) && defined(__AVX512BW__) - if(use_isa >= ISA_LEVEL_AVX3) { -# if defined(__AVX512VBMI2__) - if(use_isa >= ISA_LEVEL_VBMI2) { - __m128i dataTop = _mm256_extracti128_si256(dataA, 1); - dataA = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KNOT32(maskA), dataA); - _mm256_storeu_si256((__m256i*)p, dataA); - p[32] = _mm_extract_epi8(dataTop, 15); - p += outputBytesA; - - dataTop = _mm256_extracti128_si256(dataB, 1); - dataB = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KNOT32(maskB), dataB); - _mm256_storeu_si256((__m256i*)p, dataB); - p[32] = _mm_extract_epi8(dataTop, 15); - p += maskBitsB; - } else -# endif - { - __m256i dataSwapped = _mm256_permute4x64_epi64(dataA, _MM_SHUFFLE(1,0,3,2)); - dataA = _mm256_mask_alignr_epi8(dataA, (uint32_t)(-(int32_t)maskA), dataA, dataSwapped, 15); - dataA = _mm256_ternarylogic_epi32(dataA, cmpA, _mm256_set1_epi8('='), 0xb8); // (data & ~cmp) | (cmp & '=') - _mm256_storeu_si256((__m256i*)p, dataA); - p[32] = _mm_extract_epi8(_mm256_castsi256_si128(dataSwapped), 15); - p += outputBytesA; - - dataSwapped = _mm256_permute4x64_epi64(dataB, _MM_SHUFFLE(1,0,3,2)); - dataB = _mm256_mask_alignr_epi8(dataB, (uint32_t)(-(int32_t)maskB), dataB, dataSwapped, 15); - dataB = _mm256_ternarylogic_epi32(dataB, cmpB, _mm256_set1_epi8('='), 0xb8); - _mm256_storeu_si256((__m256i*)p, dataB); - p[32] = _mm_extract_epi8(_mm256_castsi256_si128(dataSwapped), 15); - p += maskBitsB; - } - } else -#endif - { - bitIndexA = _lzcnt_u32(maskA); - bitIndexB = _lzcnt_u32(maskB); - __m256i mergeMaskA = _mm256_load_si256((const __m256i*)(lookupsAVX2->expandMergemix + bitIndexA*2*YMM_SIZE)); - __m256i mergeMaskB = _mm256_load_si256((const __m256i*)(lookupsAVX2->expandMergemix + bitIndexB*2*YMM_SIZE)); - -#if defined(__tune_bdver4__) || defined(__tune_znver1__) - // avoid slower 32-byte crossing loads on Zen1 - __m256i dataAShifted = _mm256_alignr_epi8( - dataA, - _mm256_inserti128_si256(dataA, _mm256_castsi256_si128(dataA), 1), - 15 - ); - __m256i dataBShifted = _mm256_alignr_epi8( - dataB, - _mm256_inserti128_si256(dataB, _mm256_castsi256_si128(dataB), 1), - 15 - ); -#else - __m256i dataAShifted = _mm256_loadu_si256((__m256i *)(es + i - YMM_SIZE*2 - 1)); - __m256i dataBShifted = _mm256_loadu_si256((__m256i *)(es + i - YMM_SIZE - 1)); -#endif - dataA = _mm256_andnot_si256(cmpA, dataA); // clear space for '=' char - dataA = _mm256_blendv_epi8(dataAShifted, dataA, mergeMaskA); - dataA = _mm256_add_epi8(dataA, _mm256_load_si256((const __m256i*)(lookupsAVX2->expandMergemix + bitIndexA*2*YMM_SIZE) + 1)); - _mm256_storeu_si256((__m256i*)p, dataA); - p[YMM_SIZE] = es[i-1-YMM_SIZE] + 42 + (64 & (maskA>>(YMM_SIZE-1-6))); - p += outputBytesA; - - dataB = _mm256_andnot_si256(cmpB, dataB); - dataB = _mm256_blendv_epi8(dataBShifted, dataB, mergeMaskB); - dataB = _mm256_add_epi8(dataB, _mm256_load_si256((const __m256i*)(lookupsAVX2->expandMergemix + bitIndexB*2*YMM_SIZE) + 1)); - _mm256_storeu_si256((__m256i*)p, dataB); - p[YMM_SIZE] = es[i-1] + 42 + (64 & (maskB>>(YMM_SIZE-1-6))); - p += maskBitsB; - } - col += outputBytesA + maskBitsB; - - if(col >= 0) { - _encode_loop_branch_fast_eol: - if(HEDLEY_UNLIKELY(col > (intptr_t)maskBitsB)) { - if(use_isa >= ISA_LEVEL_AVX3) - bitIndexA = _lzcnt_u32(maskA); - bitIndexA += 1 + maskBitsB; - - i += maskBitsB - YMM_SIZE; - if(HEDLEY_UNLIKELY(col == (intptr_t)bitIndexA)) { - // this is an escape character, so line will need to overflow - p--; - } else { - i += (col > (intptr_t)bitIndexA); - } - } else { - if(use_isa >= ISA_LEVEL_AVX3) - bitIndexB = _lzcnt_u32(maskB); - bitIndexB++; - - if(HEDLEY_UNLIKELY(col == (intptr_t)bitIndexB)) { - p--; - } else { - i += (col > (intptr_t)bitIndexB); - } - } - i -= col; - p -= col; - - _encode_eol_handle_pre: - uint32_t eolChar = (use_isa >= ISA_LEVEL_VBMI2 ? lookupsVBMI2->eolLastChar[es[i]] : lookupsAVX2->eolLastChar[es[i]]); - *(uint32_t*)p = eolChar; - p += 3 + (uintptr_t)(eolChar>>27); - col = lineSizeOffset; - - if(HEDLEY_UNLIKELY(i >= 0)) { // this isn't really a proper check - it's only needed to support short lines; basically, if the line is too short, `i` never gets checked, so we need one somewhere - i++; - break; - } - - dataA = _mm256_loadu_si256((__m256i *)(es + i + 1)); - dataB = _mm256_loadu_si256((__m256i *)(es + i + 1) + 1); - i += YMM_SIZE*2 + 1; - // search for special chars - cmpA = _mm256_cmpeq_epi8( - _mm256_shuffle_epi8(_mm256_set_epi8( - '\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42, - '\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42 - ), _mm256_adds_epi8( - _mm256_abs_epi8(dataA), _mm256_set_epi64x(0, 0, 0, 88) - )), - dataA - ); - cmpB = _mm256_cmpeq_epi8( - _mm256_shuffle_epi8(_mm256_set_epi8( - '\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42, - '\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42 - ), _mm256_abs_epi8(dataB)), - dataB - ); - - // duplicate some code from above to reduce jumping a little -#if defined(__AVX512VL__) - if(use_isa >= ISA_LEVEL_AVX3) { - dataA = _mm256_add_epi8(dataA, _mm256_set1_epi8(42)); - dataA = _mm256_ternarylogic_epi32(dataA, cmpA, _mm256_set1_epi8(64), 0xf8); // data | (cmp & 64) - dataB = _mm256_add_epi8(dataB, _mm256_set1_epi8(42)); - dataB = _mm256_ternarylogic_epi32(dataB, cmpB, _mm256_set1_epi8(64), 0xf8); // data | (cmp & 64) - } -#endif - - maskA = (uint32_t)_mm256_movemask_epi8(cmpA); - maskB = (uint32_t)_mm256_movemask_epi8(cmpB); - maskBitsA = popcnt32(maskA); - maskBitsB = popcnt32(maskB); - outputBytesA = maskBitsA + YMM_SIZE; - if (LIKELIHOOD(0.170, (maskBitsA|maskBitsB) > 1)) - goto _encode_loop_branch_slow; - - - //goto _encode_loop_branch_fast; - // duplicating the code, instead of using the goto above, seems to fix a performance regression in GCC - maskBitsB += YMM_SIZE; -#if defined(__AVX512VL__) && defined(__AVX512BW__) - if(use_isa >= ISA_LEVEL_AVX3) { -# if defined(__AVX512VBMI2__) - if(use_isa >= ISA_LEVEL_VBMI2) { - __m128i dataTop = _mm256_extracti128_si256(dataA, 1); - dataA = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KNOT32(maskA), dataA); - _mm256_storeu_si256((__m256i*)p, dataA); - p[32] = _mm_extract_epi8(dataTop, 15); - p += outputBytesA; - - dataTop = _mm256_extracti128_si256(dataB, 1); - dataB = _mm256_mask_expand_epi8(_mm256_set1_epi8('='), KNOT32(maskB), dataB); - _mm256_storeu_si256((__m256i*)p, dataB); - p[32] = _mm_extract_epi8(dataTop, 15); - p += maskBitsB; - } else -# endif - { - __m256i dataSwapped = _mm256_permute4x64_epi64(dataA, _MM_SHUFFLE(1,0,3,2)); - dataA = _mm256_mask_alignr_epi8(dataA, (uint32_t)(-(int32_t)maskA), dataA, dataSwapped, 15); - dataA = _mm256_ternarylogic_epi32(dataA, cmpA, _mm256_set1_epi8('='), 0xb8); // (data & ~cmp) | (cmp & '=') - _mm256_storeu_si256((__m256i*)p, dataA); - p[32] = _mm_extract_epi8(_mm256_castsi256_si128(dataSwapped), 15); - p += outputBytesA; - - dataSwapped = _mm256_permute4x64_epi64(dataB, _MM_SHUFFLE(1,0,3,2)); - dataB = _mm256_mask_alignr_epi8(dataB, (uint32_t)(-(int32_t)maskB), dataB, dataSwapped, 15); - dataB = _mm256_ternarylogic_epi32(dataB, cmpB, _mm256_set1_epi8('='), 0xb8); - _mm256_storeu_si256((__m256i*)p, dataB); - p[32] = _mm_extract_epi8(_mm256_castsi256_si128(dataSwapped), 15); - p += maskBitsB; - } - } else -#endif - { - bitIndexA = _lzcnt_u32(maskA); - bitIndexB = _lzcnt_u32(maskB); - __m256i mergeMaskA = _mm256_load_si256((const __m256i*)(lookupsAVX2->expandMergemix + bitIndexA*2*YMM_SIZE)); - __m256i mergeMaskB = _mm256_load_si256((const __m256i*)(lookupsAVX2->expandMergemix + bitIndexB*2*YMM_SIZE)); - -#if defined(__tune_bdver4__) || defined(__tune_znver1__) - // avoid slower 32-byte crossing loads on Zen1 - __m256i dataAShifted = _mm256_alignr_epi8( - dataA, - _mm256_inserti128_si256(dataA, _mm256_castsi256_si128(dataA), 1), - 15 - ); - __m256i dataBShifted = _mm256_alignr_epi8( - dataB, - _mm256_inserti128_si256(dataB, _mm256_castsi256_si128(dataB), 1), - 15 - ); -#else - __m256i dataAShifted = _mm256_loadu_si256((__m256i *)(es + i - YMM_SIZE*2 - 1)); - __m256i dataBShifted = _mm256_loadu_si256((__m256i *)(es + i - YMM_SIZE - 1)); -#endif - dataA = _mm256_andnot_si256(cmpA, dataA); // clear space for '=' char - dataA = _mm256_blendv_epi8(dataAShifted, dataA, mergeMaskA); - dataA = _mm256_add_epi8(dataA, _mm256_load_si256((const __m256i*)(lookupsAVX2->expandMergemix + bitIndexA*2*YMM_SIZE) + 1)); - _mm256_storeu_si256((__m256i*)p, dataA); - p[YMM_SIZE] = es[i-1-YMM_SIZE] + 42 + (64 & (maskA>>(YMM_SIZE-1-6))); - p += outputBytesA; - - dataB = _mm256_andnot_si256(cmpB, dataB); - dataB = _mm256_blendv_epi8(dataBShifted, dataB, mergeMaskB); - dataB = _mm256_add_epi8(dataB, _mm256_load_si256((const __m256i*)(lookupsAVX2->expandMergemix + bitIndexB*2*YMM_SIZE) + 1)); - _mm256_storeu_si256((__m256i*)p, dataB); - p[YMM_SIZE] = es[i-1] + 42 + (64 & (maskB>>(YMM_SIZE-1-6))); - p += maskBitsB; - } - col += outputBytesA + maskBitsB; - - if(col >= 0) - goto _encode_loop_branch_fast_eol; - } - } - } while(i < 0); - - _mm256_zeroupper(); - - *colOffset = (int)(col + line_size -1); - dest = p; - len = -(i - INPUT_OFFSET); -} -} // namespace - -#endif diff --git a/rapidyenc/rapidyenc/src/encoder_common.h b/rapidyenc/rapidyenc/src/encoder_common.h deleted file mode 100644 index 8e4975c..0000000 --- a/rapidyenc/rapidyenc/src/encoder_common.h +++ /dev/null @@ -1,103 +0,0 @@ -#ifndef __YENC_ENCODER_COMMON -#define __YENC_ENCODER_COMMON - -namespace RapidYenc { - void encoder_sse2_init(); - void encoder_ssse3_init(); - void encoder_avx_init(); - void encoder_avx2_init(); - void encoder_vbmi2_init(); - extern const bool encoder_has_avx10; - void encoder_neon_init(); - void encoder_rvv_init(); - - // lookup tables for scalar processing - extern const unsigned char escapeLUT[256]; - extern const uint16_t escapedLUT[256]; - - size_t do_encode_generic(int line_size, int* colOffset, const unsigned char* HEDLEY_RESTRICT src, unsigned char* HEDLEY_RESTRICT dest, size_t len, int doEnd); -} - - - -template -static size_t do_encode_simd(int line_size, int* colOffset, const unsigned char* HEDLEY_RESTRICT src, unsigned char* HEDLEY_RESTRICT dest, size_t len, int doEnd) { - if(len < 1) return 0; - if(line_size < 12) { // short lines probably not worth processing in a SIMD way - // we assume at least the first and last char exist in the line, and since the first char could be escaped, and SIMD encoder assumes at least one non-first/last char, assumption means that line size has to be >= 4 - return RapidYenc::do_encode_generic(line_size, colOffset, src, dest, len, doEnd); - } - - const uint8_t* es = src + len; - uint8_t* p = dest; - - if(*colOffset < 0) *colOffset = 0; // sanity check - - kernel(line_size, colOffset, es, p, len); - - // scalar loop to process remaining - long i = -(long)len; - if(*colOffset == 0 && i < 0) { - uint8_t c = es[i++]; - if (LIKELIHOOD(0.0273, RapidYenc::escapedLUT[c] != 0)) { - memcpy(p, RapidYenc::escapedLUT + c, 2); - p += 2; - *colOffset = 2; - } else { - *(p++) = c + 42; - *colOffset = 1; - } - } - while(i < 0) { - uint8_t c = es[i++]; - if(*colOffset < line_size-1) { - if(!RapidYenc::escapeLUT[c]) { - p[0] = '='; - p[1] = c+42+64; - p += 2; - (*colOffset) += 2; - } else { - *(p++) = RapidYenc::escapeLUT[c]; - (*colOffset) += 1; - } - } else { - if(*colOffset < line_size) { - if (RapidYenc::escapedLUT[c] && c != '.'-42) { - memcpy(p, RapidYenc::escapedLUT + c, 2); - p += 2; - } else { - *(p++) = c + 42; - } - if(i == 0) break; - c = es[i++]; - } - - // handle EOL - if (RapidYenc::escapedLUT[c]) { - uint32_t w = UINT32_16_PACK(UINT16_PACK('\r', '\n'), (uint32_t)RapidYenc::escapedLUT[c]); - memcpy(p, &w, sizeof(w)); - p += 4; - *colOffset = 2; - } else { - uint32_t w = UINT32_PACK('\r', '\n', (uint32_t)(c+42), 0); - memcpy(p, &w, sizeof(w)); - p += 3; - *colOffset = 1; - } - } - } - - if(doEnd) { - // special case: if the last character is a space/tab, it needs to be escaped as it's the final character on the line - unsigned char lc = *(p-1); - if(lc == '\t' || lc == ' ') { - p[-1] = '='; - *p = lc+64; - p++; - (*colOffset)++; - } - } - return p - dest; -} - -#endif /* __YENC_ENCODER_COMMON */ diff --git a/rapidyenc/rapidyenc/src/encoder_neon.cc b/rapidyenc/rapidyenc/src/encoder_neon.cc deleted file mode 100644 index e3faab2..0000000 --- a/rapidyenc/rapidyenc/src/encoder_neon.cc +++ /dev/null @@ -1,550 +0,0 @@ -#include "common.h" -#include "encoder_common.h" - -#ifdef __ARM_NEON -#include "encoder.h" - -// Clang wrongly assumes alignment on vst1q_u8_x2, and ARMv7 GCC doesn't support the function, so effectively, it can only be used in ARMv8 compilers -#if defined(__aarch64__) && (defined(__clang__) || HEDLEY_GCC_VERSION_CHECK(8,5,0)) -# define vst1q_u8_x2_unaligned vst1q_u8_x2 -#else -static HEDLEY_ALWAYS_INLINE void vst1q_u8_x2_unaligned(uint8_t* p, uint8x16x2_t data) { - vst1q_u8(p, data.val[0]); - vst1q_u8(p+16, data.val[1]); -} -#endif - - -// ARM's CLZ instruction at native bit-width -#ifdef __aarch64__ -static HEDLEY_ALWAYS_INLINE int clz_n(uint64_t v) { -# ifdef _MSC_VER - long r; - // does this work? - if(_BitScanReverse64((unsigned long*)&r, v)) - r ^= 63; - else - r = 64; - return r; -# else -# if defined(__clang__) || HEDLEY_GCC_VERSION_CHECK(11,0,0) - // this pattern is only detected on GCC >= 11 (Clang 9 seems to as well, unsure about earlier versions) - // - note: return type must be 'int'; GCC fails to optimise this if type is 'long' - // GCC <= 10 doesn't optimize around the '0 = undefined behaviour', so not needed there - if(v == 0) return 64; -# endif - return __builtin_clzll(v); -# endif -} -#else -static HEDLEY_ALWAYS_INLINE int clz_n(uint32_t v) { -# ifdef __GNUC__ -# if defined(__clang__) || HEDLEY_GCC_VERSION_CHECK(7,0,0) - // as with AArch64 version above, only insert this check if compiler can optimise it away - if(v == 0) return 32; -# endif - return __builtin_clz(v); -# elif defined(_MSC_VER) - return _arm_clz(v); -# else - return __clz(v); // ARM compiler? -# endif -} -#endif - -static uint8x16_t ALIGN_TO(16, shufLUT[256]); -static uint16_t expandLUT[256]; - -static HEDLEY_ALWAYS_INLINE void encode_eol_handle_pre(const uint8_t* HEDLEY_RESTRICT es, long& i, uint8_t*& p, long& col, long lineSizeOffset) { - uint8x16_t oDataA = vld1q_u8(es + i); - uint8x16_t oDataB = vld1q_u8(es + i + sizeof(uint8x16_t)); - uint8x16_t dataA = oDataA; - uint8x16_t dataB = oDataB; -#ifdef __aarch64__ - uint8x16_t cmpA = vreinterpretq_u8_s8(vqtbx2q_s8( - vdupq_n_s8('='-42), - vcreate2_s8(vmakeq_s8('\0'-42,-128,-128,'\0'-42,'\t'-42,'\n'-42,'\r'-42,'\t'-42,'\n'-42,'\r'-42,-128,-128,'\0'-42,-128,-128,-128), vmakeq_s8(' '-42,'\n'-42,'\r'-42,' '-42,-128,-128,-128,-128,-128,-128,'.'-42,-128,-128,-128,'='-42,-128)), - vreinterpretq_u8_s8(vhaddq_s8(vreinterpretq_s8_u8(dataA), vmakeq_s8(42,48,66,66, 66,66,66,66, 66,66,66,66, 66,66,66,66))) - )); - cmpA = vceqq_u8(cmpA, dataA); - - dataB = vaddq_u8(oDataB, vdupq_n_u8(42)); - uint8x16_t cmpB = vqtbx1q_u8( - vceqq_u8(oDataB, vdupq_n_u8('='-42)), - // \0 \n \r - vmakeq_u8(255,0,0,0,0,0,0,0,0,0,255,0,0,255,0,0), - dataB - ); - dataA = vaddq_u8(dataA, vbslq_u8(cmpA, vdupq_n_u8(64+42), vdupq_n_u8(42))); - dataB = vorrq_u8(dataB, vandq_u8(cmpB, vdupq_n_u8(64))); -#else - uint8x16_t cmpA = vorrq_u8( - vorrq_u8( - vceqq_u8(oDataA, vdupq_n_u8(-42)), - vceqq_u8(oDataA, vdupq_n_u8('='-42)) - ), - vorrq_u8( - vceqq_u8(oDataA, vdupq_n_u8('\r'-42)), - vceqq_u8(oDataA, vdupq_n_u8('\n'-42)) - ) - ); - uint8x16_t cmpB = vorrq_u8( - vorrq_u8( - vceqq_u8(oDataB, vdupq_n_u8(-42)), - vceqq_u8(oDataB, vdupq_n_u8('='-42)) - ), - vorrq_u8( - vceqq_u8(oDataB, vdupq_n_u8('\r'-42)), - vceqq_u8(oDataB, vdupq_n_u8('\n'-42)) - ) - ); - - // dup low 2 bytes & compare - uint8x8_t firstTwoChars = vreinterpret_u8_u16(vdup_lane_u16(vreinterpret_u16_u8(vget_low_u8(oDataA)), 0)); - uint8x8_t cmpNl = vceq_u8(firstTwoChars, vmake_u8( - ' '+214,' '+214,'\t'+214,'\t'+214,'\r'+214,'.'-42,'='-42,'='-42 - )); - // use padd to merge comparisons - uint16x4_t cmpNl2 = vreinterpret_u16_u8(cmpNl); - cmpNl2 = vpadd_u16(cmpNl2, vdup_n_u16(0)); - cmpNl2 = vpadd_u16(cmpNl2, vdup_n_u16(0)); - cmpA = vcombine_u8( - vorr_u8(vget_low_u8(cmpA), vreinterpret_u8_u16(cmpNl2)), - vget_high_u8(cmpA) - ); - dataA = vsubq_u8(dataA, vbslq_u8(cmpA, vdupq_n_u8(-64-42), vdupq_n_u8(-42))); - dataB = vsubq_u8(dataB, vbslq_u8(cmpB, vdupq_n_u8(-64-42), vdupq_n_u8(-42))); -#endif - - - uint8x16_t cmpAMasked = vandq_u8(cmpA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); - uint8x16_t cmpBMasked = vandq_u8(cmpB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); -#ifdef __aarch64__ - uint8x16_t cmpMerge = vpaddq_u8(cmpAMasked, cmpBMasked); - cmpMerge = vpaddq_u8(cmpMerge, cmpMerge); - uint64_t mask = vgetq_lane_u64(vreinterpretq_u64_u8(cmpMerge), 0); - - // write out first char + newline - uint32_t firstChar = vgetq_lane_u8(dataA, 0); - if(LIKELIHOOD(0.0234, mask & 1)) { - firstChar <<= 8; - firstChar |= 0x0a0d003d; - memcpy(p, &firstChar, sizeof(firstChar)); - p += 4; - mask ^= 1; - cmpMerge = vbicq_u8(cmpMerge, vmakeq_u8(1,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0)); - } else { - firstChar |= 0x0a0d00; - memcpy(p, &firstChar, sizeof(firstChar)); - p += 3; - } - - if(LIKELIHOOD(0.09, (mask & (mask-1)) != 0)) { - mask |= mask >> 8; - uint8x8_t cmpPacked = vpadd_u8(vget_low_u8(cmpMerge), vget_low_u8(cmpMerge)); - uint8_t m1 = (mask & 0xff); - uint8_t m2 = ((mask >> 16) & 0xff); - uint8_t m3 = ((mask >> 32) & 0xff); - uint8_t m4 = ((mask >> 48) & 0xff); -#else - // no vpaddq_u8 in ARMv7, so need extra 64-bit VPADD - uint8x8_t cmpPacked = vpadd_u8( - vpadd_u8( - vget_low_u8(cmpAMasked), vget_high_u8(cmpAMasked) - ), - vpadd_u8( - vget_low_u8(cmpBMasked), vget_high_u8(cmpBMasked) - ) - ); - cmpPacked = vpadd_u8(cmpPacked, cmpPacked); - uint32_t mask = vget_lane_u32(vreinterpret_u32_u8(cmpPacked), 0); - - // write out first char + newline - uint32_t firstChar = vgetq_lane_u8(dataA, 0); - if(LIKELIHOOD(0.0234, mask & 1)) { - firstChar <<= 8; - firstChar |= 0x0a0d003d; - memcpy(p, &firstChar, sizeof(firstChar)); - p += 4; - mask ^= 1; - cmpPacked = vbic_u8(cmpPacked, vmake_u8(1,0,0,0, 0,0,0,0)); - } else { - firstChar |= 0x0a0d00; - memcpy(p, &firstChar, sizeof(firstChar)); - p += 3; - } - - if(LIKELIHOOD(0.09, (mask & (mask-1)) != 0)) { - uint8_t m1 = (mask & 0xff); - uint8_t m2 = ((mask >> 8) & 0xff); - uint8_t m3 = ((mask >> 16) & 0xff); - uint8_t m4 = ((mask >> 24) & 0xff); -#endif - - // perform lookup for shuffle mask - uint8x16_t shuf1 = vld1q_u8((uint8_t*)(shufLUT + m1)); - uint8x16_t shuf2 = vld1q_u8((uint8_t*)(shufLUT + m2)); - uint8x16_t shuf3 = vld1q_u8((uint8_t*)(shufLUT + m3)); - uint8x16_t shuf4 = vld1q_u8((uint8_t*)(shufLUT + m4)); -#ifdef __aarch64__ - uint8x16_t data1A = vqtbx1q_u8(shuf1, dataA, shuf1); - uint8x16_t data2A = vqtbx1q_u8(shuf2, vextq_u8(dataA, dataA, 8), shuf2); - uint8x16_t data1B = vqtbx1q_u8(shuf3, dataB, shuf3); - uint8x16_t data2B = vqtbx1q_u8(shuf4, vextq_u8(dataB, dataB, 8), shuf4); -#else - uint8x8_t shuf1l = vget_low_u8(shuf1); - uint8x8_t shuf1h = vget_high_u8(shuf1); - uint8x8_t shuf2l = vget_low_u8(shuf2); - uint8x8_t shuf2h = vget_high_u8(shuf2); - uint8x8_t shuf3l = vget_low_u8(shuf3); - uint8x8_t shuf3h = vget_high_u8(shuf3); - uint8x8_t shuf4l = vget_low_u8(shuf4); - uint8x8_t shuf4h = vget_high_u8(shuf4); - uint8x16_t data1A = vcombine_u8(vtbx1_u8(shuf1l, vget_low_u8(dataA), shuf1l), - vtbx1_u8(shuf1h, vget_low_u8(dataA), shuf1h)); - uint8x16_t data2A = vcombine_u8(vtbx1_u8(shuf2l, vget_high_u8(dataA), shuf2l), - vtbx1_u8(shuf2h, vget_high_u8(dataA), shuf2h)); - uint8x16_t data1B = vcombine_u8(vtbx1_u8(shuf3l, vget_low_u8(dataB), shuf3l), - vtbx1_u8(shuf3h, vget_low_u8(dataB), shuf3h)); - uint8x16_t data2B = vcombine_u8(vtbx1_u8(shuf4l, vget_high_u8(dataB), shuf4l), - vtbx1_u8(shuf4h, vget_high_u8(dataB), shuf4h)); -#endif - data1A = vextq_u8(data1A, data1A, 1); // shift out processed byte (last char of line) - - uint32_t counts = vget_lane_u32(vreinterpret_u32_u8(vcnt_u8(cmpPacked)), 0); - counts += 0x08080807; - - unsigned char shuf1Len = counts & 0xff; - unsigned char shuf2Len = (counts>>8) & 0xff; - unsigned char shuf3Len = (counts>>16) & 0xff; - unsigned char shuf4Len = (counts>>24) & 0xff; - uint32_t shufTotalLen = counts * 0x1010101; - shufTotalLen >>= 24; - - vst1q_u8(p, data1A); - p += shuf1Len; - vst1q_u8(p, data2A); - p += shuf2Len; - vst1q_u8(p, data1B); - p += shuf3Len; - vst1q_u8(p, data2B); - p += shuf4Len; - col = shufTotalLen+1 + lineSizeOffset-32; - } else { - // shuffle stuff up - long bitIndex = clz_n(mask); - uint8x16_t vClz = vdupq_n_u8(bitIndex & ~(sizeof(mask)*8)); -#ifdef __aarch64__ - uint8x16_t blendA = vcgtq_u8(vmakeq_u8(63,62,61,60,51,50,49,48,47,46,45,44,35,34,33,32), vClz); - uint8x16_t blendB = vcgtq_u8(vmakeq_u8(31,30,29,28,19,18,17,16,15,14,13,12, 3, 2, 1, 0), vClz); -#else - uint8x16_t blendA = vcgtq_u8(vmakeq_u8(31,30,29,28,27,26,25,24,23,22,21,20,19,18,17,16), vClz); - uint8x16_t blendB = vcgtq_u8(vmakeq_u8(15,14,13,12,11,10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), vClz); -#endif - uint8x16_t dataAShifted = vbslq_u8(cmpA, vdupq_n_u8('='), dataA); - uint8x16_t dataBShifted = vbslq_u8(cmpB, vdupq_n_u8('='), dataB); - dataAShifted = vextq_u8(dataAShifted, dataBShifted, 1); - dataBShifted = vextq_u8(dataBShifted, dataBShifted, 1); - dataA = vbslq_u8(blendA, dataAShifted, dataA); - dataB = vbslq_u8(blendB, dataBShifted, dataB); - - vst1q_u8_x2_unaligned(p, vcreate2_u8(dataA, dataB)); - p += sizeof(uint8x16_t)*2 - 1; - p += (mask != 0); - col = lineSizeOffset + (mask != 0); - } - - i += sizeof(uint8x16_t)*2; - // TODO: check col >= 0 if we want to support short lines -} - - -namespace RapidYenc { - -HEDLEY_ALWAYS_INLINE void do_encode_neon(int line_size, int* colOffset, const uint8_t* HEDLEY_RESTRICT srcEnd, uint8_t* HEDLEY_RESTRICT& dest, size_t& len) { - // offset position to enable simpler loop condition checking - const int INPUT_OFFSET = sizeof(uint8x16_t)*4 -1; // extra chars for EOL handling, -1 to change <= to < - if(len <= INPUT_OFFSET || line_size < (int)sizeof(uint8x16_t)*4) return; - - uint8_t *p = dest; // destination pointer - long i = -(long)len; // input position - long lineSizeOffset = -line_size +32; // line size plus vector length - long col = *colOffset - line_size +1; - - i += INPUT_OFFSET; - const uint8_t* es = srcEnd - INPUT_OFFSET; - - if (HEDLEY_LIKELY(col == -line_size+1)) { - uint8_t c = es[i++]; - if (LIKELIHOOD(0.0273, escapedLUT[c] != 0)) { - memcpy(p, escapedLUT + c, 2); - p += 2; - col += 2; - } else { - *(p++) = c + 42; - col += 1; - } - } - if(HEDLEY_UNLIKELY(col >= 0)) { - if(col == 0) - encode_eol_handle_pre(es, i, p, col, lineSizeOffset); - else { - uint8_t c = es[i++]; - if (LIKELIHOOD(0.0273, escapedLUT[c]!=0)) { - uint32_t v = UINT32_16_PACK(UINT16_PACK('\r', '\n'), (uint32_t)escapedLUT[c]); - memcpy(p, &v, sizeof(v)); - p += 4; - col = 2-line_size + 1; - } else { - uint32_t v = UINT32_PACK('\r', '\n', (uint32_t)(c+42), 0); - memcpy(p, &v, sizeof(v)); - p += 3; - col = 2-line_size; - } - } - } - while(i < 0) { - // for unaligned loads, separate loads seem to be faster than vld1q_u8_x2 on Cortex A53; unsure if this applies elsewhere - uint8x16_t dataA = vld1q_u8(es + i); - uint8x16_t dataB = vld1q_u8(es + i + sizeof(uint8x16_t)); - i += sizeof(uint8x16_t)*2; - // search for special chars -#ifdef __aarch64__ - uint8x16_t cmpEqA = vceqq_u8(dataA, vdupq_n_u8('='-42)); - uint8x16_t cmpEqB = vceqq_u8(dataB, vdupq_n_u8('='-42)); - dataA = vaddq_u8(dataA, vdupq_n_u8(42)); - dataB = vaddq_u8(dataB, vdupq_n_u8(42)); - uint8x16_t cmpA = vqtbx1q_u8( - cmpEqA, - // \0 \n \r - vmakeq_u8(255,0,0,0,0,0,0,0,0,0,255,0,0,255,0,0), - dataA - ); - uint8x16_t cmpB = vqtbx1q_u8( - cmpEqB, - // \0 \n \r - vmakeq_u8(255,0,0,0,0,0,0,0,0,0,255,0,0,255,0,0), - dataB - ); - - dataA = vorrq_u8(dataA, vandq_u8(cmpA, vdupq_n_u8(64))); - dataB = vorrq_u8(dataB, vandq_u8(cmpB, vdupq_n_u8(64))); -#else - // the ARMv8 strategy may be worth it here with 2x vtbx2's, but both GCC-9 and Clang-9 generate poor assembly for it, so it performs worse than the following - uint8x16_t cmpA = vorrq_u8( - vorrq_u8( - vceqq_u8(dataA, vdupq_n_u8(-42)), - vceqq_u8(dataA, vdupq_n_u8('='-42)) - ), - vorrq_u8( - vceqq_u8(dataA, vdupq_n_u8('\r'-42)), - vceqq_u8(dataA, vdupq_n_u8('\n'-42)) - ) - ); - uint8x16_t cmpB = vorrq_u8( - vorrq_u8( - vceqq_u8(dataB, vdupq_n_u8(-42)), - vceqq_u8(dataB, vdupq_n_u8('='-42)) - ), - vorrq_u8( - vceqq_u8(dataB, vdupq_n_u8('\r'-42)), - vceqq_u8(dataB, vdupq_n_u8('\n'-42)) - ) - ); - - dataA = vsubq_u8(dataA, vbslq_u8(cmpA, vdupq_n_u8(-64-42), vdupq_n_u8(-42))); - dataB = vsubq_u8(dataB, vbslq_u8(cmpB, vdupq_n_u8(-64-42), vdupq_n_u8(-42))); -#endif - - - long bitIndex; // prevent compiler whining - uint8x16_t cmpAMasked = vandq_u8(cmpA, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); - uint8x16_t cmpBMasked = vandq_u8(cmpB, vmakeq_u8(1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128)); -#ifdef __aarch64__ - uint8x16_t cmpMerge = vpaddq_u8(cmpAMasked, cmpBMasked); - cmpMerge = vpaddq_u8(cmpMerge, cmpMerge); - uint64_t mask = vgetq_lane_u64(vreinterpretq_u64_u8(cmpMerge), 0); - if(LIKELIHOOD(0.09, (mask & (mask-1)) != 0)) { - mask |= mask >> 8; - uint8x8_t cmpPacked = vpadd_u8(vget_low_u8(cmpMerge), vget_low_u8(cmpMerge)); - uint8_t m1 = (mask & 0xff); - uint8_t m2 = ((mask >> 16) & 0xff); - uint8_t m3 = ((mask >> 32) & 0xff); - uint8_t m4 = ((mask >> 48) & 0xff); -#else - // no vpaddq_u8 in ARMv7, so need extra 64-bit VPADD - uint8x8_t cmpPacked = vpadd_u8( - vpadd_u8( - vget_low_u8(cmpAMasked), vget_high_u8(cmpAMasked) - ), - vpadd_u8( - vget_low_u8(cmpBMasked), vget_high_u8(cmpBMasked) - ) - ); - cmpPacked = vpadd_u8(cmpPacked, cmpPacked); - uint32_t mask = vget_lane_u32(vreinterpret_u32_u8(cmpPacked), 0); - if(LIKELIHOOD(0.09, (mask & (mask-1)) != 0)) { - uint8_t m1 = (mask & 0xff); - uint8_t m2 = ((mask >> 8) & 0xff); - uint8_t m3 = ((mask >> 16) & 0xff); - uint8_t m4 = ((mask >> 24) & 0xff); -#endif - - // perform lookup for shuffle mask - uint8x16_t shuf1 = vld1q_u8((uint8_t*)(shufLUT + m1)); - uint8x16_t shuf2 = vld1q_u8((uint8_t*)(shufLUT + m2)); - uint8x16_t shuf3 = vld1q_u8((uint8_t*)(shufLUT + m3)); - uint8x16_t shuf4 = vld1q_u8((uint8_t*)(shufLUT + m4)); - - // expand halves -#ifdef __aarch64__ - uint8x16_t data1A = vqtbx1q_u8(shuf1, dataA, shuf1); - uint8x16_t data2A = vqtbx1q_u8(shuf2, vextq_u8(dataA, dataA, 8), shuf2); - uint8x16_t data1B = vqtbx1q_u8(shuf3, dataB, shuf3); - uint8x16_t data2B = vqtbx1q_u8(shuf4, vextq_u8(dataB, dataB, 8), shuf4); -#else - uint8x8_t shuf1l = vget_low_u8(shuf1); - uint8x8_t shuf1h = vget_high_u8(shuf1); - uint8x8_t shuf2l = vget_low_u8(shuf2); - uint8x8_t shuf2h = vget_high_u8(shuf2); - uint8x8_t shuf3l = vget_low_u8(shuf3); - uint8x8_t shuf3h = vget_high_u8(shuf3); - uint8x8_t shuf4l = vget_low_u8(shuf4); - uint8x8_t shuf4h = vget_high_u8(shuf4); - uint8x16_t data1A = vcombine_u8(vtbx1_u8(shuf1l, vget_low_u8(dataA), shuf1l), - vtbx1_u8(shuf1h, vget_low_u8(dataA), shuf1h)); - uint8x16_t data2A = vcombine_u8(vtbx1_u8(shuf2l, vget_high_u8(dataA), shuf2l), - vtbx1_u8(shuf2h, vget_high_u8(dataA), shuf2h)); - uint8x16_t data1B = vcombine_u8(vtbx1_u8(shuf3l, vget_low_u8(dataB), shuf3l), - vtbx1_u8(shuf3h, vget_low_u8(dataB), shuf3h)); - uint8x16_t data2B = vcombine_u8(vtbx1_u8(shuf4l, vget_high_u8(dataB), shuf4l), - vtbx1_u8(shuf4h, vget_high_u8(dataB), shuf4h)); -#endif - - // store out - uint32_t counts = vget_lane_u32(vreinterpret_u32_u8(vcnt_u8(cmpPacked)), 0); - counts += 0x08080808; - - unsigned char shuf1Len = counts & 0xff; - unsigned char shuf2Len = (counts>>8) & 0xff; - unsigned char shuf3Len = (counts>>16) & 0xff; - unsigned char shuf4Len = (counts>>24) & 0xff; - uint32_t shufTotalLen = counts * 0x1010101; - shufTotalLen >>= 24; - - vst1q_u8(p, data1A); - p += shuf1Len; - vst1q_u8(p, data2A); - p += shuf2Len; - vst1q_u8(p, data1B); - p += shuf3Len; - vst1q_u8(p, data2B); - p += shuf4Len; - col += shufTotalLen; - - if(LIKELIHOOD(0.3, col >= 0)) { - // we overflowed - find correct position to revert back to - long revert = col; - long len2ndHalf = shuf3Len+shuf4Len; - long shiftAmt = len2ndHalf - col -1; - uint32_t eqMaskHalf; - if(HEDLEY_UNLIKELY(shiftAmt < 0)) { - eqMaskHalf = (expandLUT[m2] << shuf1Len) | expandLUT[m1]; - eqMaskHalf >>= shufTotalLen - col -1; - i += len2ndHalf - 16; - } else { - eqMaskHalf = (expandLUT[m4] << shuf3Len) | expandLUT[m3]; - eqMaskHalf >>= shiftAmt; - } - revert += eqMaskHalf & 1; - - // count bits in eqMask - uint8x8_t vCnt = vcnt_u8(vreinterpret_u8_u32(vmov_n_u32(eqMaskHalf))); - uint32_t cnt = vget_lane_u32(vreinterpret_u32_u8(vCnt), 0); - cnt *= 0x1010101; - i += cnt >> 24; - - p -= revert; - i -= revert; - goto _encode_eol_handle_pre; - } - } else { - { - bitIndex = clz_n(mask); - uint8x16_t vClz = vdupq_n_u8(bitIndex & ~(sizeof(mask)*8)); -#ifdef __aarch64__ - uint8x16_t blendA = vcgeq_u8(vmakeq_u8(63,62,61,60,51,50,49,48,47,46,45,44,35,34,33,32), vClz); - uint8x16_t blendB = vcgeq_u8(vmakeq_u8(31,30,29,28,19,18,17,16,15,14,13,12, 3, 2, 1, 0), vClz); -#else - uint8x16_t blendA = vcgeq_u8(vmakeq_u8(31,30,29,28,27,26,25,24,23,22,21,20,19,18,17,16), vClz); - uint8x16_t blendB = vcgeq_u8(vmakeq_u8(15,14,13,12,11,10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), vClz); -#endif - uint8x16_t dataAShifted = vextq_u8(dataA, dataA, 15); - uint8x16_t dataBShifted = vextq_u8(dataA, dataB, 15); - dataA = vbslq_u8(cmpA, vdupq_n_u8('='), dataA); - uint8x16_t outDataB = vbslq_u8(cmpB, vdupq_n_u8('='), dataB); - dataA = vbslq_u8(blendA, dataA, dataAShifted); - outDataB = vbslq_u8(blendB, outDataB, dataBShifted); - - vst1q_u8_x2_unaligned(p, vcreate2_u8(dataA, outDataB)); - p += sizeof(uint8x16_t)*2; - // write last byte - *p = vgetq_lane_u8(dataB, 15); - p += (mask != 0); - col += (mask != 0) + sizeof(uint8x16_t)*2; - } - - if(HEDLEY_UNLIKELY(col >= 0)) { -#ifdef __aarch64__ - // fixup bitIndex - bitIndex -= ((bitIndex+4)>>4)<<3; -#endif - bitIndex = bitIndex +1; - if(HEDLEY_UNLIKELY(col == bitIndex)) { - // this is an escape character, so line will need to overflow - p--; - } else { - i += (col > bitIndex); - } - p -= col; - i -= col; - - _encode_eol_handle_pre: - encode_eol_handle_pre(es, i, p, col, lineSizeOffset); - } - } - } - - *colOffset = col + line_size -1; - dest = p; - len = -(i - INPUT_OFFSET); -} -} // namespace - -void RapidYenc::encoder_neon_init() { - _do_encode = &do_encode_simd; - _encode_isa = ISA_LEVEL_NEON; - // generate shuf LUT - for(int i=0; i<256; i++) { - int k = i; - uint16_t expand = 0; - uint8_t* res = (uint8_t*)(shufLUT + i); - int p = 0; - for(int j=0; j<8; j++) { - if(k & 1) { - res[j+p] = '='; - expand |= 1<<(j+p); - p++; - } - res[j+p] = j; - k >>= 1; - } - for(; p<8; p++) - res[8+p] = 8+p +0x80; // +0x80 => 0 discarded entries; has no effect other than to ease debugging - - expandLUT[i] = expand; - } -} -#else -void RapidYenc::encoder_neon_init() {} -#endif /* defined(__ARM_NEON) */ diff --git a/rapidyenc/rapidyenc/src/encoder_rvv.cc b/rapidyenc/rapidyenc/src/encoder_rvv.cc deleted file mode 100644 index 8ad7877..0000000 --- a/rapidyenc/rapidyenc/src/encoder_rvv.cc +++ /dev/null @@ -1,207 +0,0 @@ -#include "common.h" -#include "encoder_common.h" - -#ifdef __riscv_vector -#include "encoder.h" - - -static HEDLEY_ALWAYS_INLINE void encode_eol_handle_pre(const uint8_t* HEDLEY_RESTRICT _src, long& inpos, uint8_t*& outp, long& col, long lineSizeOffset) { - // TODO: vectorize - uint8_t c = _src[inpos++]; - if(HEDLEY_UNLIKELY(RapidYenc::escapedLUT[c] && c != '.'-42)) { - memcpy(outp, &RapidYenc::escapedLUT[c], sizeof(uint16_t)); - outp += 2; - } else { - *(outp++) = c + 42; - } - - c = _src[inpos++]; - if(LIKELIHOOD(0.0273, RapidYenc::escapedLUT[c]!=0)) { - uint32_t w = UINT32_16_PACK(UINT16_PACK('\r', '\n'), (uint32_t)RapidYenc::escapedLUT[c]); - memcpy(outp, &w, sizeof(w)); - outp += 4; - col = lineSizeOffset + 2; - } else { - uint32_t w = UINT32_PACK('\r', '\n', (uint32_t)(c+42), 0); - memcpy(outp, &w, sizeof(w)); - outp += 3; - col = lineSizeOffset + 1; - } -} - -namespace RapidYenc { - -HEDLEY_ALWAYS_INLINE void do_encode_rvv(int line_size, int* colOffset, const uint8_t* HEDLEY_RESTRICT srcEnd, uint8_t* HEDLEY_RESTRICT& dest, size_t& len) { - size_t vl2 = RV(vsetvlmax_e8m2)(); // TODO: limit to line length - // TODO: have a LMUL=1 variant if line_size < vl - - // offset position to enable simpler loop condition checking - const int INPUT_OFFSET = vl2*2 -1; // extra chars for EOL handling, -1 to change <= to < - if((intptr_t)len <= INPUT_OFFSET || line_size < (int)vl2*2) return; - - uint8_t *outp = dest; - long inpos = -(long)len; - long lineSizeOffset = -line_size +1; - long col = *colOffset - line_size +1; - - inpos += INPUT_OFFSET; - const uint8_t* _src = srcEnd - INPUT_OFFSET; - - if (HEDLEY_LIKELY(col == -line_size+1)) { - uint8_t c = _src[inpos++]; - if (LIKELIHOOD(0.0273, escapedLUT[c] != 0)) { - memcpy(outp, escapedLUT + c, 2); - outp += 2; - col += 2; - } else { - *(outp++) = c + 42; - col += 1; - } - } - if(HEDLEY_UNLIKELY(col >= 0)) { - if(col == 0) - encode_eol_handle_pre(_src, inpos, outp, col, lineSizeOffset); - else { - uint8_t c = _src[inpos++]; - if(LIKELIHOOD(0.0273, escapedLUT[c]!=0)) { - uint32_t v = UINT32_16_PACK(UINT16_PACK('\r', '\n'), (uint32_t)escapedLUT[c]); - memcpy(outp, &v, sizeof(v)); - outp += 4; - col = 2-line_size + 1; - } else { - uint32_t v = UINT32_PACK('\r', '\n', (uint32_t)(c+42), 0); - memcpy(outp, &v, sizeof(v)); - outp += 3; - col = 2-line_size; - } - } - } - - // vector constants - const vuint8mf2_t ALT_SHIFT = RV(vreinterpret_v_u16mf2_u8mf2)(RV(vmv_v_x_u16mf2)(4, vl2)); - const uint8_t _MASK_EXPAND[] = {0xAA, 0xAB, 0xAE, 0xAF, 0xBA, 0xBB, 0xBE, 0xBF, 0xEA, 0xEB, 0xEE, 0xEF, 0xFA, 0xFB, 0xFE, 0xFF}; - const vuint8m1_t MASK_EXPAND = RV(vle8_v_u8m1)(_MASK_EXPAND, 16); - - - // TODO: consider exploiting partial vector capability - while(inpos < 0) { - vuint8m2_t data = RV(vle8_v_u8m2)(_src + inpos, vl2); - inpos += vl2; - - // search for special chars - // TODO: vrgather strat - - vuint8m2_t tmpData = RV(vsub_vx_u8m2)(data, -42, vl2); - vbool4_t cmp = RV(vmor_mm_b4)( - RV(vmor_mm_b4)( - RV(vmseq_vx_u8m2_b4)(data, -42, vl2), - RV(vmseq_vx_u8m2_b4)(tmpData, '=', vl2), - vl2 - ), - RV(vmor_mm_b4)( - RV(vmseq_vx_u8m2_b4)(data, '\r'-42, vl2), - RV(vmseq_vx_u8m2_b4)(data, '\n'-42, vl2), - vl2 - ), - vl2 - ); - -#ifdef __riscv_v_intrinsic - data = RV(vor_vx_u8m2_mu)(cmp, tmpData, tmpData, 64, vl2); -#else - data = RV(vor_vx_u8m2_m)(cmp, tmpData, tmpData, 64, vl2); -#endif - - int idx; - size_t count = RV(vcpop_m_b4)(cmp, vl2); - if(count > 1) { - // widen mask: 4b->8b - vuint8mf4_t vcmp = RV_VEC_U8MF4_CAST(cmp); - // TODO: use vwsll instead if available - // - is clmul useful here? - vuint8mf2_t xcmp = RV(vreinterpret_v_u16mf2_u8mf2)(RV(vwmulu_vx_u16mf2)(vcmp, 16, vl2)); - xcmp = RV(vsrl_vv_u8mf2)(xcmp, ALT_SHIFT, vl2); - - // expand mask by inserting '1' between each bit (0000abcd -> 1a1b1c1d) - vuint8m1_t xcmpTmp = RV(vrgather_vv_u8m1)(MASK_EXPAND, RV(vlmul_ext_v_u8mf2_u8m1)(xcmp), vl2); - vbool2_t cmpmask = RV_MASK_CAST(2, 8, xcmpTmp); - - // expand data and insert = - // TODO: use vwsll instead if available - vuint16m4_t data2 = RV(vzext_vf2_u16m4)(data, vl2); - data2 = RV(vsll_vx_u16m4)(data2, 8, vl2); - data2 = RV(vor_vx_u16m4)(data2, '=', vl2); - - // prune unneeded = - vuint8m4_t dataTmp = RV(vreinterpret_v_u16m4_u8m4)(data2); - vuint8m4_t final_data = RV(vcompress_vm_u8m4)( -#ifdef __riscv_v_intrinsic - dataTmp, cmpmask, vl2*2 -#else - cmpmask, dataTmp, dataTmp, vl2*2 -#endif - ); - - RV(vse8_v_u8m4)(outp, final_data, vl2*2); - outp += vl2 + count; - col += vl2 + count; - - if(col >= 0) { - // we overflowed - find correct position to revert back to - // TODO: stick with u8 type for vlmax <= 2048 (need to check if ok if vlmax == 2048) - // - considering that it's rare for colWidth > 128, maybe just don't support vectors that long - vuint16m8_t xidx = RV(viota_m_u16m8)(cmpmask, vl2*2); - vbool2_t discardmask = RV(vmsgeu_vx_u16m8_b2)(xidx, vl2 + count - col, vl2*2); - long idx_revert = RV(vcpop_m_b2)(discardmask, vl2*2); - - outp -= col + (idx_revert & 1); - inpos -= ((idx_revert+1) >> 1); - - goto _encode_eol_handle_pre; - } - } else { - // 0 or 1 special characters - { - vbool4_t mask = RV(vmsbf_m_b4)(cmp, vl2); - // TODO: is it better to shuffle this into two stores, instead of three? - RV(vse8_v_u8m2_m)(mask, outp, data, vl2); - idx = RV(vcpop_m_b4)(mask, vl2); - outp[idx] = '='; - RV(vse8_v_u8m2_m)(RV(vmnot_m_b4)(mask, vl2), outp+1, data, vl2); - - outp += vl2 + count; - col += vl2 + count; - } - - if(col >= 0) { - if(count > 0) { - idx = vl2 - idx; - if(HEDLEY_UNLIKELY(col == idx)) { - // this is an escape character, so line will need to overflow - outp--; - } else { - inpos += (col > idx); - } - } - outp -= col; - inpos -= col; - - _encode_eol_handle_pre: - encode_eol_handle_pre(_src, inpos, outp, col, lineSizeOffset); - } - } - } - - *colOffset = col + line_size -1; - dest = outp; - len = -(inpos - INPUT_OFFSET); -} -} // namespace - -void RapidYenc::encoder_rvv_init() { - _do_encode = &do_encode_simd; - _encode_isa = ISA_LEVEL_RVV; -} -#else -void RapidYenc::encoder_rvv_init() {} -#endif /* defined(__riscv_vector) */ diff --git a/rapidyenc/rapidyenc/src/encoder_sse2.cc b/rapidyenc/rapidyenc/src/encoder_sse2.cc deleted file mode 100644 index 015150f..0000000 --- a/rapidyenc/rapidyenc/src/encoder_sse2.cc +++ /dev/null @@ -1,15 +0,0 @@ -#include "common.h" -#include "encoder_common.h" - -#ifdef __SSE2__ -#include "encoder_sse_base.h" - -void RapidYenc::encoder_sse2_init() { - _do_encode = &do_encode_simd< do_encode_sse >; - encoder_sse_lut(); - _encode_isa = ISA_LEVEL_SSE2; -} -#else -void RapidYenc::encoder_sse2_init() {} -#endif - diff --git a/rapidyenc/rapidyenc/src/encoder_sse_base.h b/rapidyenc/rapidyenc/src/encoder_sse_base.h deleted file mode 100644 index 2eb6ca3..0000000 --- a/rapidyenc/rapidyenc/src/encoder_sse_base.h +++ /dev/null @@ -1,725 +0,0 @@ -#include "common.h" - -#include "encoder.h" -#include "encoder_common.h" - -#if defined(__clang__) && __clang_major__ == 6 && __clang_minor__ == 0 -// VBMI2 introduced in clang 6.0, but 128-bit functions misnamed there; fixed in clang 7.0, but we'll handle those on 6.0 -# define _mm_mask_expand_epi8 _mm128_mask_expand_epi8 -#endif - -#if (defined(__GNUC__) && __GNUC__ >= 7) || (defined(_MSC_VER) && _MSC_VER >= 1924) -# define KLOAD16(a, offs) _load_mask16((__mmask16*)(a) + (offs)) -#else -# define KLOAD16(a, offs) (((uint16_t*)(a))[(offs)]) -#endif - -#pragma pack(16) -static struct { - /*align16*/ struct { __m128i shuf, mix; } shufMix[256]; - unsigned char BitsSetTable256plus8[256]; - uint32_t eolLastChar[256]; - uint16_t eolFirstMask[256]; - uint16_t expandMask[256]; - /*align16*/ int8_t expandMaskmix[33*2*32]; - /*align16*/ int8_t expandShufmaskmix[33*2*32]; -} * HEDLEY_RESTRICT lookups; -#pragma pack() - -template -static void encoder_sse_lut() { - ALIGN_ALLOC(lookups, sizeof(*lookups), 16); - for(int i=0; i<256; i++) { - int k = i; - uint8_t* res = (uint8_t*)(&(lookups->shufMix[i].shuf)); - uint16_t expand = 0; - int p = 0; - for(int j=0; j<8; j++) { - if(k & 1) { - res[j+p] = 0xf0 + j; - p++; - } - expand |= 1<<(j+p); - res[j+p] = j; - k >>= 1; - } - for(; p<8; p++) - res[8+p] = 8+p +0x40; // +0x40 is an arbitrary value to make debugging slightly easier? the top bit cannot be set - - lookups->expandMask[i] = expand; - - // calculate add mask for mixing escape chars in - __m128i shuf = _mm_load_si128((__m128i*)res); - __m128i maskEsc = _mm_cmpeq_epi8(_mm_and_si128(shuf, _mm_set1_epi8(-16)), _mm_set1_epi8(-16)); // -16 == 0xf0 - __m128i addMask = _mm_and_si128(_mm_slli_si128(maskEsc, 1), _mm_set1_epi8(64)); - addMask = _mm_or_si128(addMask, _mm_and_si128(maskEsc, _mm_set1_epi8('='-42))); - addMask = _mm_add_epi8(addMask, _mm_set1_epi8(42)); - - _mm_store_si128(&(lookups->shufMix[i].mix), addMask); - - - lookups->eolLastChar[i] = ((i == 214+'\t' || i == 214+' ' || i == 214+'\0' || i == 214+'\n' || i == 214+'\r' || i == '='-42) ? (((i+42+64)&0xff)<<8)+0x0a0d003d : ((i+42)&0xff)+0x0a0d00); - lookups->eolFirstMask[i] = ((i == 214+'\t' || i == 214+' ' || i == '.'-42) ? 1 : 0); - - lookups->BitsSetTable256plus8[i] = 8 + ( - (i & 1) + ((i>>1) & 1) + ((i>>2) & 1) + ((i>>3) & 1) + ((i>>4) & 1) + ((i>>5) & 1) + ((i>>6) & 1) + ((i>>7) & 1) - ); - } - for(int i=0; i<33; i++) { - int n = (use_isa & ISA_FEATURE_LZCNT) ? (i == 32 ? 32 : 31-i) : (i == 0 ? 32 : i-1); - for(int j=0; j<32; j++) { - lookups->expandMaskmix[i*64 + j] = (n>j ? -1 : 0); - if(j > 15) // mask part - lookups->expandShufmaskmix[i*64 + j] = (n>=j ? -1 : 0); - else // shuffle part - lookups->expandShufmaskmix[i*64 + j] = (n==j ? -1 : (j-(nexpandMaskmix[i*64 + j + 32] = (n==j ? '=' : 42+64*(n==j-1)); - lookups->expandShufmaskmix[i*64 + j + 32] = (n==j ? '=' : 42+64*(n==j-1)); - } - } -} - - -// for LZCNT/BSF -#ifdef _MSC_VER -# include -# include -static HEDLEY_ALWAYS_INLINE unsigned BSR32(unsigned src) { - unsigned long result; - _BitScanReverse((unsigned long*)&result, src); - return result; -} -#elif defined(__GNUC__) -// have seen Clang not like _bit_scan_reverse -# include // for lzcnt -# define BSR32(src) (31^__builtin_clz(src)) -#else -# include -# define BSR32 _bit_scan_reverse -#endif - -template -static HEDLEY_ALWAYS_INLINE __m128i sse2_expand_bytes(unsigned mask, __m128i data) { - while(mask) { - // get highest bit - unsigned bitIndex; - __m128i mergeMask; -#if defined(__LZCNT__) - if(use_isa & ISA_FEATURE_LZCNT) { - bitIndex = _lzcnt_u32(mask); - mergeMask = _mm_load_si128((const __m128i*)lookups->expandMaskmix + bitIndex*4); - mask &= 0x7fffffffU>>bitIndex; - } else -#endif - { - // TODO: consider LUT for when BSR is slow - bitIndex = BSR32(mask); - mergeMask = _mm_load_si128((const __m128i*)lookups->expandMaskmix + (bitIndex+1)*4); - mask ^= 1< -static HEDLEY_ALWAYS_INLINE uintptr_t sse2_expand_store_vector(__m128i data, unsigned int mask, unsigned maskP1, unsigned maskP2, uint8_t* p, unsigned int& shufLenP1, unsigned int& shufLenP2) { - // TODO: consider 1 bit shortcut (slightly tricky with needing bit counts though) - if(mask) { - __m128i dataA = sse2_expand_bytes(maskP1, data); - __m128i dataB = sse2_expand_bytes(maskP2, _mm_srli_si128(data, 8)); - dataA = _mm_add_epi8(dataA, _mm_load_si128(&(lookups->shufMix[maskP1].mix))); - dataB = _mm_add_epi8(dataB, _mm_load_si128(&(lookups->shufMix[maskP2].mix))); - shufLenP1 = lookups->BitsSetTable256plus8[maskP1]; - shufLenP2 = shufLenP1 + lookups->BitsSetTable256plus8[maskP2]; - STOREU_XMM(p, dataA); - STOREU_XMM(p+shufLenP1, dataB); - return shufLenP2; - } else { - STOREU_XMM(p, _mm_sub_epi8(data, _mm_set1_epi8(-42))); - shufLenP1 = 8; - shufLenP2 = 16; - return XMM_SIZE; - } -} - -namespace RapidYenc { - -template -HEDLEY_ALWAYS_INLINE void do_encode_sse(int line_size, int* colOffset, const uint8_t* HEDLEY_RESTRICT srcEnd, uint8_t* HEDLEY_RESTRICT& dest, size_t& len) { - // offset position to enable simpler loop condition checking - const int INPUT_OFFSET = XMM_SIZE*4+1 -1; // EOL handling reads an additional byte, -1 to change <= to < - if(len <= INPUT_OFFSET || line_size < XMM_SIZE) return; - - // slower CPUs prefer to branch as mispredict penalty is probably small relative to general execution -#if defined(__tune_atom__) || defined(__tune_slm__) || defined(__tune_btver1__) || defined(__tune_btver2__) - const bool _PREFER_BRANCHING = true; -#else - const bool _PREFER_BRANCHING = (use_isa < ISA_LEVEL_SSSE3); -#endif - - uint8_t *p = dest; // destination pointer - intptr_t i = -(intptr_t)len; // input position - intptr_t lineSizeOffset = -line_size +1; - //intptr_t col = *colOffset - line_size +1; // for some reason, this causes GCC-8 to spill an extra register, causing the main loop to run ~5% slower, so use the alternative version below - intptr_t col = *colOffset + lineSizeOffset; - - i += INPUT_OFFSET; - const uint8_t* es = srcEnd - INPUT_OFFSET; - - if(HEDLEY_UNLIKELY(col >= 0)) { - uint8_t c = es[i++]; - if(col == 0) { - uint32_t eolChar = lookups->eolLastChar[c]; - *(uint32_t*)p = eolChar; - p += 3 + (eolChar>>27); - col = -line_size+1; - } else { - if (LIKELIHOOD(0.0273, escapedLUT[c]!=0)) { - *(uint32_t*)p = UINT32_16_PACK(UINT16_PACK('\r', '\n'), (uint32_t)escapedLUT[c]); - p += 4; - col = 2-line_size + 1; - } else { - *(uint32_t*)p = UINT32_PACK('\r', '\n', (uint32_t)(c+42), 0); - p += 3; - col = 2-line_size; - } - } - } - if (HEDLEY_LIKELY(col == -line_size+1)) { - uint8_t c = es[i++]; - if (LIKELIHOOD(0.0273, escapedLUT[c] != 0)) { - *(uint16_t*)p = escapedLUT[c]; - p += 2; - col += 2; - } else { - *(p++) = c + 42; - col += 1; - } - } - do { - __m128i dataA = _mm_loadu_si128((__m128i *)(es + i)); // probably not worth the effort to align - __m128i dataB = _mm_loadu_si128((__m128i *)(es + i) +1); - - i += XMM_SIZE*2; - // search for special chars - __m128i cmpA, cmpB; -#if defined(__SSSE3__) && !defined(__tune_atom__) && !defined(__tune_slm__) && !defined(__tune_btver1__) - if(use_isa >= ISA_LEVEL_SSSE3) { - cmpA = _mm_cmpeq_epi8( - _mm_shuffle_epi8(_mm_set_epi8( - '\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42 - ), _mm_abs_epi8(dataA)), - dataA - ); - } else -#endif - { - cmpA = _mm_or_si128( - _mm_or_si128( - _mm_cmpeq_epi8(dataA, _mm_set1_epi8(-42)), - _mm_cmpeq_epi8(dataA, _mm_set1_epi8('\n'-42)) - ), - _mm_or_si128( - _mm_cmpeq_epi8(dataA, _mm_set1_epi8('='-42)), - _mm_cmpeq_epi8(dataA, _mm_set1_epi8('\r'-42)) - ) - ); - } - - _encode_loop_branchA: - unsigned int maskA = _mm_movemask_epi8(cmpA); - _encode_loop_branchB: - -#if defined(__SSSE3__) && !defined(__tune_atom__) && !defined(__tune_slm__) && !defined(__tune_btver1__) - if(use_isa >= ISA_LEVEL_SSSE3) { - cmpB = _mm_cmpeq_epi8( - _mm_shuffle_epi8(_mm_set_epi8( - '\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42 - ), _mm_abs_epi8(dataB)), - dataB - ); - } else -#endif - { - cmpB = _mm_or_si128( - _mm_or_si128( - _mm_cmpeq_epi8(dataB, _mm_set1_epi8(-42)), - _mm_cmpeq_epi8(dataB, _mm_set1_epi8('\n'-42)) - ), - _mm_or_si128( - _mm_cmpeq_epi8(dataB, _mm_set1_epi8('='-42)), - _mm_cmpeq_epi8(dataB, _mm_set1_epi8('\r'-42)) - ) - ); - } - unsigned int maskB = _mm_movemask_epi8(cmpB); - - uint32_t mask = (maskB<<16) | maskA; - intptr_t bitIndex; // because you can't goto past variable declarations... - intptr_t maskBits, outputBytes; - - bool manyBitsSet; -#if defined(__POPCNT__) && !defined(__tune_btver1__) - if(use_isa & ISA_FEATURE_POPCNT) { - maskBits = popcnt32(mask); - outputBytes = maskBits + XMM_SIZE*2; - manyBitsSet = maskBits > 1; - } else -#endif - { - manyBitsSet = (mask & (mask-1)) != 0; - } - - if (LIKELIHOOD(0.089, manyBitsSet)) { - _encode_loop_branch_slow: - unsigned m1 = maskA & 0xFF; - unsigned m2 = maskA >> 8; - unsigned m3 = maskB & 0xFF; - unsigned m4 = maskB >> 8; - unsigned int shuf1Len, shuf2Len, shuf3Len; - __m128i shuf1A, shuf1B, shuf2A, shuf2B; // only used for SSSE3 path - __m128i data1A, data1B, data2A, data2B; - -#if defined(__AVX512VBMI2__) && defined(__AVX512VL__) && defined(__AVX512BW__) - if(use_isa >= ISA_LEVEL_VBMI2) { - // do +42 and +64 to data - dataA = _mm_sub_epi8(dataA, _mm_set1_epi8(-42)); - dataA = _mm_ternarylogic_epi32(dataA, cmpA, _mm_set1_epi8(64), 0xf8); // data | (cmp & 64) - dataB = _mm_sub_epi8(dataB, _mm_set1_epi8(-42)); - dataB = _mm_ternarylogic_epi32(dataB, cmpB, _mm_set1_epi8(64), 0xf8); - - /* alternative no-LUT 64-bit only version - * LUT generally seems to be faster though - //uint64_t expandMask = _pdep_u64(mask, 0x5555555555555555); // expand bits - //expandMask = ~_pext_u64(expandMask, expandMask|~0x5555555555555555); - uint64_t expandMask = ~_pdep_u64(~mask, 0x5555555555555555); // expand bits, with bits set - expandMask = _pext_u64(expandMask^0x5555555555555555, expandMask); - data2A = _mm_mask_expand_epi8(_mm_set1_epi8('='), expandMask>>16, _mm_srli_si128(dataA, 8)); - data1A = _mm_mask_expand_epi8(_mm_set1_epi8('='), expandMask , dataA); - data2B = _mm_mask_expand_epi8(_mm_set1_epi8('='), expandMask>>48, _mm_srli_si128(dataB, 8)); - data1B = _mm_mask_expand_epi8(_mm_set1_epi8('='), expandMask>>32, dataB); - */ - data2A = _mm_mask_expand_epi8(_mm_set1_epi8('='), KLOAD16(lookups->expandMask, m2), _mm_srli_si128(dataA, 8)); - data1A = _mm_mask_expand_epi8(_mm_set1_epi8('='), KLOAD16(lookups->expandMask, m1), dataA); - data2B = _mm_mask_expand_epi8(_mm_set1_epi8('='), KLOAD16(lookups->expandMask, m4), _mm_srli_si128(dataB, 8)); - data1B = _mm_mask_expand_epi8(_mm_set1_epi8('='), KLOAD16(lookups->expandMask, m3), dataB); - } else -#endif -#ifdef __SSSE3__ - if(use_isa >= ISA_LEVEL_SSSE3) { - // perform lookup for shuffle mask - shuf1A = _mm_load_si128(&(lookups->shufMix[m1].shuf)); - shuf2A = _mm_load_si128(&(lookups->shufMix[m2].shuf)); - shuf1B = _mm_load_si128(&(lookups->shufMix[m3].shuf)); - shuf2B = _mm_load_si128(&(lookups->shufMix[m4].shuf)); - - // second mask processes on second half, so add to the offsets - shuf2A = _mm_or_si128(shuf2A, _mm_set1_epi8(8)); - shuf2B = _mm_or_si128(shuf2B, _mm_set1_epi8(8)); - - // expand halves - data2A = _mm_shuffle_epi8(dataA, shuf2A); - data1A = _mm_shuffle_epi8(dataA, shuf1A); - data2B = _mm_shuffle_epi8(dataB, shuf2B); - data1B = _mm_shuffle_epi8(dataB, shuf1B); - - // add in escaped chars - data1A = _mm_add_epi8(data1A, _mm_load_si128(&(lookups->shufMix[m1].mix))); - data2A = _mm_add_epi8(data2A, _mm_load_si128(&(lookups->shufMix[m2].mix))); - data1B = _mm_add_epi8(data1B, _mm_load_si128(&(lookups->shufMix[m3].mix))); - data2B = _mm_add_epi8(data2B, _mm_load_si128(&(lookups->shufMix[m4].mix))); - } else -#endif - { - p += sse2_expand_store_vector(dataA, maskA, m1, m2, p, shuf1Len, shuf2Len); - unsigned int shuf4Len; - p += sse2_expand_store_vector(dataB, maskB, m3, m4, p, shuf3Len, shuf4Len); - shuf3Len += shuf2Len; -#if !defined(__tune_btver1__) - if(!(use_isa & ISA_FEATURE_POPCNT)) -#endif - outputBytes = shuf2Len + shuf4Len; - } - - if(use_isa >= ISA_LEVEL_SSSE3) { - // store out -#if defined(__POPCNT__) && !defined(__tune_btver1__) - if(use_isa & ISA_FEATURE_POPCNT) { - shuf2Len = popcnt32(maskA) + 16; -# if defined(__tune_znver6__) || defined(__tune_znver5__) || defined(__tune_znver4__) || defined(__tune_znver3__) || defined(__tune_znver2__) || defined(__tune_znver1__) || defined(__tune_btver2__) - shuf1Len = popcnt32(m1) + 8; - shuf3Len = popcnt32(m3) + shuf2Len + 8; -# else - shuf1Len = lookups->BitsSetTable256plus8[m1]; - shuf3Len = lookups->BitsSetTable256plus8[m3] + shuf2Len; -# endif - } else -#endif - { - shuf1Len = lookups->BitsSetTable256plus8[m1]; - shuf2Len = shuf1Len + lookups->BitsSetTable256plus8[m2]; - shuf3Len = shuf2Len + lookups->BitsSetTable256plus8[m3]; - outputBytes = shuf3Len + lookups->BitsSetTable256plus8[m4]; - } - STOREU_XMM(p, data1A); - STOREU_XMM(p+shuf1Len, data2A); - STOREU_XMM(p+shuf2Len, data1B); - STOREU_XMM(p+shuf3Len, data2B); - p += outputBytes; - } - col += outputBytes; - - if(LIKELIHOOD(0.3 /*guess, using 128b lines*/, col >= 0)) { - uintptr_t bitCount; - intptr_t shiftAmt = (outputBytes - shuf2Len) - col -1; - uint32_t eqMask; - if(HEDLEY_UNLIKELY(shiftAmt < 0)) { - shiftAmt += shuf2Len; - i -= 16; - if(use_isa >= ISA_LEVEL_VBMI2 || use_isa < ISA_LEVEL_SSSE3) { - eqMask = - ((uint32_t)lookups->expandMask[m2] << shuf1Len) - | (uint32_t)lookups->expandMask[m1]; - } else { - eqMask = - ((uint32_t)_mm_movemask_epi8(shuf2A) << shuf1Len) - | (uint32_t)_mm_movemask_epi8(shuf1A); - i += outputBytes - shuf2Len; - } - } else { - if(use_isa >= ISA_LEVEL_VBMI2 || use_isa < ISA_LEVEL_SSSE3) { - eqMask = - ((uint32_t)lookups->expandMask[m4] << (shuf3Len-shuf2Len)) - | (uint32_t)lookups->expandMask[m3]; - } else { - eqMask = - ((uint32_t)_mm_movemask_epi8(shuf2B) << (shuf3Len-shuf2Len)) - | (uint32_t)_mm_movemask_epi8(shuf1B); - } - } - - if(use_isa >= ISA_LEVEL_VBMI2 || use_isa < ISA_LEVEL_SSSE3) { -#if defined(__GNUC__) - // be careful to avoid partial flag stalls on Intel P6 CPUs (SHR+ADC will likely stall) -# if !(defined(__tune_amdfam10__) || defined(__tune_k8__)) - if(use_isa >= ISA_LEVEL_VBMI2) -# endif - { - __asm__( - "shrl $1, %[eqMask] \n" - "shrl %%cl, %[eqMask] \n" // TODO: can use shrq to avoid above shift? -# if defined(PLATFORM_AMD64) && !defined(__ILP32__) - "adcq %q[col], %q[p] \n" -# else - "adcl %[col], %[p] \n" -# endif - : [eqMask]"+r"(eqMask), [p]"+r"(p) - : "c"(shiftAmt), [col]"r"(~col) - ); - } -# if !(defined(__tune_amdfam10__) || defined(__tune_k8__)) - else -# else - if(0) -# endif -#endif - { - eqMask >>= shiftAmt; - p -= col; - if(LIKELIHOOD(0.98, (eqMask & 1) != 1)) - p--; - else - i++; - } - } else { - eqMask >>= shiftAmt; - col += eqMask & 1; // revert if escape char - } - -#if defined(__POPCNT__) - if(use_isa & ISA_FEATURE_POPCNT) { - bitCount = popcnt32(eqMask); - } else -#endif - { - unsigned char cnt = lookups->BitsSetTable256plus8[eqMask & 0xff]; - cnt += lookups->BitsSetTable256plus8[(eqMask>>8) & 0xff]; - cnt += lookups->BitsSetTable256plus8[(eqMask>>16) & 0xff]; - cnt += lookups->BitsSetTable256plus8[(eqMask>>24) & 0xff]; - bitCount = (uintptr_t)cnt - 32; - } - - if(use_isa >= ISA_LEVEL_VBMI2 || use_isa < ISA_LEVEL_SSSE3) { - i -= bitCount; - goto _encode_eol_handle_pre; - } else { - i += bitCount; - goto _encode_eol_handle_pre_adjust; - } - } - } else { - if(_PREFER_BRANCHING && LIKELIHOOD(0.663, !mask)) { - _encode_loop_branch_fast_noesc: - dataA = _mm_sub_epi8(dataA, _mm_set1_epi8(-42)); - dataB = _mm_sub_epi8(dataB, _mm_set1_epi8(-42)); - STOREU_XMM(p, dataA); - STOREU_XMM(p+XMM_SIZE, dataB); - p += XMM_SIZE*2; - col += XMM_SIZE*2; - if(LIKELIHOOD(0.15, col >= 0)) - goto _encode_eol_handle_pre_adjust; - continue; - } - // shortcut for common case of only 1 bit set - _encode_loop_branch_fast_1ch: -#if defined(__AVX512VL__) && defined(__AVX512BW__) - if(use_isa >= ISA_LEVEL_AVX3) { - dataA = _mm_sub_epi8(dataA, _mm_set1_epi8(-42)); - dataA = _mm_ternarylogic_epi32(dataA, cmpA, _mm_set1_epi8(64), 0xf8); // data | (cmp & 64) - dataB = _mm_sub_epi8(dataB, _mm_set1_epi8(-42)); - dataB = _mm_ternarylogic_epi32(dataB, cmpB, _mm_set1_epi8(64), 0xf8); - - // store last char - p[XMM_SIZE*2] = _mm_extract_epi8(dataB, 15); - - uint32_t blendMask = (uint32_t)(-(int32_t)mask); - dataB = _mm_mask_alignr_epi8(dataB, blendMask>>16, dataB, dataA, 15); - dataB = _mm_ternarylogic_epi32(dataB, cmpB, _mm_set1_epi8('='), 0xb8); // (data & ~cmp) | (cmp & '=') - -# if defined(__AVX512VBMI2__) - if(use_isa >= ISA_LEVEL_VBMI2) - dataA = _mm_mask_expand_epi8(_mm_set1_epi8('='), ~mask, dataA); - else -# endif - { - dataA = _mm_mask_alignr_epi8(dataA, blendMask, dataA, dataA, 15); // there's no masked shift, so use ALIGNR instead - dataA = _mm_ternarylogic_epi32(dataA, cmpA, _mm_set1_epi8('='), 0xb8); - } - } else -#endif - { - -#if !defined(__tune_btver1__) - if(!(use_isa & ISA_FEATURE_POPCNT)) -#endif - maskBits = (mask != 0); - if(_PREFER_BRANCHING) maskBits = 1; -#if !defined(__tune_btver1__) - if(!(use_isa & ISA_FEATURE_POPCNT)) -#endif - outputBytes = XMM_SIZE*2 + maskBits; - -#if defined(__LZCNT__) - if(use_isa & ISA_FEATURE_LZCNT) - bitIndex = _lzcnt_u32(mask); - else -#endif - { - bitIndex = BSR32(mask); - bitIndex |= maskBits-1; // if(mask == 0) bitIndex = -1; - } - const __m128i* entries; - -#if defined(__SSSE3__) && !defined(__tune_atom__) && !defined(__tune_slm__) && !defined(__tune_btver1__) - if(use_isa >= ISA_LEVEL_SSSE3) { - entries = (const __m128i*)lookups->expandShufmaskmix; - if(!(use_isa & ISA_FEATURE_LZCNT)) - entries += 4; - entries += bitIndex*4; - - __m128i shufMaskA = _mm_load_si128(entries+0); - __m128i mergeMaskB = _mm_load_si128(entries+1); - __m128i dataBShifted = _mm_alignr_epi8(dataB, dataA, 15); - dataB = _mm_andnot_si128(cmpB, dataB); - - dataA = _mm_shuffle_epi8(dataA, shufMaskA); - -# if defined(__SSE4_1__) && !defined(__tune_slm__) && !defined(__tune_goldmont__) && !defined(__tune_goldmont_plus__) && !defined(__tune_tremont__) - if(use_isa >= ISA_LEVEL_SSE41) { - dataB = _mm_blendv_epi8(dataBShifted, dataB, mergeMaskB); - } else -# endif - { - dataB = _mm_or_si128( - _mm_and_si128(mergeMaskB, dataB), - _mm_andnot_si128(mergeMaskB, dataBShifted) - ); - } - } else -#endif - { - - entries = (const __m128i*)lookups->expandMaskmix; - if(!(use_isa & ISA_FEATURE_LZCNT)) - entries += 4; - entries += bitIndex*4; - - __m128i mergeMaskA = _mm_load_si128(entries+0); - __m128i mergeMaskB = _mm_load_si128(entries+1); - // TODO: consider deferring mask operation? (does require an extra ANDN but may help with L1 latency) - __m128i dataAMasked = _mm_andnot_si128(mergeMaskA, dataA); - __m128i dataBMasked = _mm_andnot_si128(mergeMaskB, dataB); - __m128i dataAShifted = _mm_slli_si128(dataAMasked, 1); - __m128i dataBShifted; - -#if defined(__SSSE3__) && !defined(__tune_btver1__) - if(use_isa >= ISA_LEVEL_SSSE3) - dataBShifted = _mm_alignr_epi8(dataBMasked, dataAMasked, 15); - else -#endif - dataBShifted = _mm_or_si128( - _mm_slli_si128(dataBMasked, 1), - _mm_srli_si128(dataAMasked, 15) - ); - - // alternatively `_mm_xor_si128(dataAMasked, dataA)` if compiler wants to load mergeMask* again - dataB = _mm_or_si128( - _mm_and_si128(mergeMaskB, dataB), dataBShifted - ); - dataA = _mm_or_si128( - _mm_and_si128(mergeMaskA, dataA), dataAShifted - ); - } - // add escape chars - dataA = _mm_add_epi8(dataA, _mm_load_si128(entries+2)); - dataB = _mm_add_epi8(dataB, _mm_load_si128(entries+3)); - - // store final char - p[XMM_SIZE*2] = es[i-1] + 42 + (64 & (mask>>(XMM_SIZE*2-1-6))); - } - - // store main part - STOREU_XMM(p, dataA); - STOREU_XMM(p+XMM_SIZE, dataB); - - p += outputBytes; - col += outputBytes; - - if(LIKELIHOOD(0.3, col >= 0)) { -#if defined(__AVX512VL__) - if(use_isa >= ISA_LEVEL_AVX3) - bitIndex = _lzcnt_u32(mask) +1; - else -#endif - if(use_isa & ISA_FEATURE_LZCNT) - bitIndex = bitIndex +1; - else - bitIndex = 31-bitIndex +1; - if(HEDLEY_UNLIKELY(col == bitIndex)) { - // this is an escape character, so line will need to overflow - p--; - } else { - i += (col > bitIndex); - } - _encode_eol_handle_pre_adjust: - p -= col; - i -= col; - - _encode_eol_handle_pre: - uint32_t eolChar = lookups->eolLastChar[es[i]]; - *(uint32_t*)p = eolChar; - p += 3 + (eolChar>>27); - col = lineSizeOffset; - - if(HEDLEY_UNLIKELY(i >= 0)) { // this isn't really a proper check - it's only needed to support short lines; basically, if the line is too short, `i` never gets checked, so we need one somewhere - i++; - break; - } - - dataA = _mm_loadu_si128((__m128i *)(es + i + 1)); - dataB = _mm_loadu_si128((__m128i *)(es + i + 1) + 1); - // search for special chars (EOL) -#if defined(__SSSE3__) && !defined(__tune_atom__) && !defined(__tune_slm__) && !defined(__tune_btver1__) - if(use_isa >= ISA_LEVEL_SSSE3) { - cmpA = _mm_cmpeq_epi8( - _mm_shuffle_epi8(_mm_set_epi8( - '\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42 - ), _mm_adds_epi8( - _mm_abs_epi8(dataA), _mm_cvtsi32_si128(88) - )), - dataA - ); - i += XMM_SIZE*2 + 1; -# if defined(__GNUC__) && !defined(__clang__) - // GCC seems to have trouble keeping track of variable usage and spills many of them if we goto after declarations; Clang9 seems to be fine, or if _PREFER_BRANCHING is used - if(!_PREFER_BRANCHING) - goto _encode_loop_branchA; -# endif - maskA = _mm_movemask_epi8(cmpA); - cmpB = _mm_cmpeq_epi8( - _mm_shuffle_epi8(_mm_set_epi8( - '\0'-42,-42,'\r'-42,'.'-42,'='-42,'\0'-42,'\t'-42,'\n'-42,-42,-42,'\r'-42,-42,'='-42,' '-42,-42,'\n'-42 - ), _mm_abs_epi8(dataB)), - dataB - ); - } else -#endif - { - cmpA = _mm_or_si128( - _mm_or_si128( - _mm_cmpeq_epi8(dataA, _mm_set1_epi8(-42)), - _mm_cmpeq_epi8(dataA, _mm_set1_epi8('\n'-42)) - ), - _mm_or_si128( - _mm_cmpeq_epi8(dataA, _mm_set1_epi8('='-42)), - _mm_cmpeq_epi8(dataA, _mm_set1_epi8('\r'-42)) - ) - ); - maskA = _mm_movemask_epi8(cmpA); - maskA |= lookups->eolFirstMask[es[i+1]]; - i += XMM_SIZE*2 + 1; -#if defined(__GNUC__) && !defined(__clang__) - if(!_PREFER_BRANCHING) - goto _encode_loop_branchB; -#endif - cmpB = _mm_or_si128( - _mm_or_si128( - _mm_cmpeq_epi8(dataB, _mm_set1_epi8(-42)), - _mm_cmpeq_epi8(dataB, _mm_set1_epi8('\n'-42)) - ), - _mm_or_si128( - _mm_cmpeq_epi8(dataB, _mm_set1_epi8('='-42)), - _mm_cmpeq_epi8(dataB, _mm_set1_epi8('\r'-42)) - ) - ); - } - maskB = _mm_movemask_epi8(cmpB); - - mask = (maskB<<16) | maskA; - bool manyBitsSet; // don't retain this across loop cycles -#if defined(__POPCNT__) && !defined(__tune_btver1__) - if(use_isa & ISA_FEATURE_POPCNT) { - maskBits = popcnt32(mask); - outputBytes = maskBits + XMM_SIZE*2; - manyBitsSet = maskBits > 1; - } else -#endif - { - manyBitsSet = (mask & (mask-1)) != 0; - } - - if (LIKELIHOOD(0.089, manyBitsSet)) - goto _encode_loop_branch_slow; - if(_PREFER_BRANCHING && LIKELIHOOD(0.663, !mask)) - goto _encode_loop_branch_fast_noesc; - goto _encode_loop_branch_fast_1ch; - if(0) { // silence unused label warnings - goto _encode_loop_branchA; - goto _encode_loop_branchB; - } - } - - } - } while(i < 0); - - *colOffset = (int)(col + line_size -1); - dest = p; - len = -(i - INPUT_OFFSET); -} -} // namespace - diff --git a/rapidyenc/rapidyenc/src/encoder_ssse3.cc b/rapidyenc/rapidyenc/src/encoder_ssse3.cc deleted file mode 100644 index 9ca27ea..0000000 --- a/rapidyenc/rapidyenc/src/encoder_ssse3.cc +++ /dev/null @@ -1,19 +0,0 @@ -#include "common.h" -#include "encoder_common.h" - -// slightly faster version which improves the worst case scenario significantly; since worst case doesn't happen often, overall speedup is relatively minor -// requires PSHUFB (SSSE3) instruction, but will use POPCNT (SSE4.2 (or AMD's ABM, but Phenom doesn't support SSSE3 so doesn't matter)) if available (these only seem to give minor speedups, so considered optional) -#ifdef __SSSE3__ -#include "encoder_sse_base.h" - -void RapidYenc::encoder_ssse3_init() { - _do_encode = &do_encode_simd< do_encode_sse >; - encoder_sse_lut(); - _encode_isa = ISA_LEVEL_SSSE3; -} -#else -void RapidYenc::encoder_ssse3_init() { - encoder_sse2_init(); -} -#endif - diff --git a/rapidyenc/rapidyenc/src/encoder_vbmi2.cc b/rapidyenc/rapidyenc/src/encoder_vbmi2.cc deleted file mode 100644 index 9ae279d..0000000 --- a/rapidyenc/rapidyenc/src/encoder_vbmi2.cc +++ /dev/null @@ -1,31 +0,0 @@ -#include "common.h" -#include "encoder_common.h" - -#if !defined(__EVEX512__) && (defined(__AVX10_1__) || defined(__EVEX256__)) && defined(__AVX512VL__) && defined(__AVX512VBMI2__) && defined(__AVX512BW__) -const bool RapidYenc::encoder_has_avx10 = true; -#else -const bool RapidYenc::encoder_has_avx10 = false; -#endif - -#if defined(__AVX512VL__) && defined(__AVX512VBMI2__) && defined(__AVX512BW__) -# ifndef YENC_DISABLE_AVX256 -# include "encoder_avx_base.h" - -void RapidYenc::encoder_vbmi2_init() { - _do_encode = &do_encode_simd< do_encode_avx2 >; - encoder_avx2_lut(); - _encode_isa = ISA_LEVEL_VBMI2; -} -# else -# include "encoder_sse_base.h" -void RapidYenc::encoder_vbmi2_init() { - _do_encode = &do_encode_simd< do_encode_sse >; - encoder_sse_lut(); - _encode_isa = ISA_LEVEL_VBMI2; -} -# endif -#else -void RapidYenc::encoder_vbmi2_init() { - encoder_avx2_init(); -} -#endif diff --git a/rapidyenc/rapidyenc/src/hedley.h b/rapidyenc/rapidyenc/src/hedley.h deleted file mode 100644 index 8a713e6..0000000 --- a/rapidyenc/rapidyenc/src/hedley.h +++ /dev/null @@ -1,2042 +0,0 @@ -/* Hedley - https://nemequ.github.io/hedley - * Created by Evan Nemerson - * - * To the extent possible under law, the author(s) have dedicated all - * copyright and related and neighboring rights to this software to - * the public domain worldwide. This software is distributed without - * any warranty. - * - * For details, see . - * SPDX-License-Identifier: CC0-1.0 - */ - -#if !defined(HEDLEY_VERSION) || (HEDLEY_VERSION < 15) -#if defined(HEDLEY_VERSION) -# undef HEDLEY_VERSION -#endif -#define HEDLEY_VERSION 15 - -#if defined(HEDLEY_STRINGIFY_EX) -# undef HEDLEY_STRINGIFY_EX -#endif -#define HEDLEY_STRINGIFY_EX(x) #x - -#if defined(HEDLEY_STRINGIFY) -# undef HEDLEY_STRINGIFY -#endif -#define HEDLEY_STRINGIFY(x) HEDLEY_STRINGIFY_EX(x) - -#if defined(HEDLEY_CONCAT_EX) -# undef HEDLEY_CONCAT_EX -#endif -#define HEDLEY_CONCAT_EX(a,b) a##b - -#if defined(HEDLEY_CONCAT) -# undef HEDLEY_CONCAT -#endif -#define HEDLEY_CONCAT(a,b) HEDLEY_CONCAT_EX(a,b) - -#if defined(HEDLEY_CONCAT3_EX) -# undef HEDLEY_CONCAT3_EX -#endif -#define HEDLEY_CONCAT3_EX(a,b,c) a##b##c - -#if defined(HEDLEY_CONCAT3) -# undef HEDLEY_CONCAT3 -#endif -#define HEDLEY_CONCAT3(a,b,c) HEDLEY_CONCAT3_EX(a,b,c) - -#if defined(HEDLEY_VERSION_ENCODE) -# undef HEDLEY_VERSION_ENCODE -#endif -#define HEDLEY_VERSION_ENCODE(major,minor,revision) (((major) * 1000000) + ((minor) * 1000) + (revision)) - -#if defined(HEDLEY_VERSION_DECODE_MAJOR) -# undef HEDLEY_VERSION_DECODE_MAJOR -#endif -#define HEDLEY_VERSION_DECODE_MAJOR(version) ((version) / 1000000) - -#if defined(HEDLEY_VERSION_DECODE_MINOR) -# undef HEDLEY_VERSION_DECODE_MINOR -#endif -#define HEDLEY_VERSION_DECODE_MINOR(version) (((version) % 1000000) / 1000) - -#if defined(HEDLEY_VERSION_DECODE_REVISION) -# undef HEDLEY_VERSION_DECODE_REVISION -#endif -#define HEDLEY_VERSION_DECODE_REVISION(version) ((version) % 1000) - -#if defined(HEDLEY_GNUC_VERSION) -# undef HEDLEY_GNUC_VERSION -#endif -#if defined(__GNUC__) && defined(__GNUC_PATCHLEVEL__) -# define HEDLEY_GNUC_VERSION HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__) -#elif defined(__GNUC__) -# define HEDLEY_GNUC_VERSION HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, 0) -#endif - -#if defined(HEDLEY_GNUC_VERSION_CHECK) -# undef HEDLEY_GNUC_VERSION_CHECK -#endif -#if defined(HEDLEY_GNUC_VERSION) -# define HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (HEDLEY_GNUC_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_MSVC_VERSION) -# undef HEDLEY_MSVC_VERSION -#endif -#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 140000000) && !defined(__ICL) -# define HEDLEY_MSVC_VERSION HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 10000000, (_MSC_FULL_VER % 10000000) / 100000, (_MSC_FULL_VER % 100000) / 100) -#elif defined(_MSC_FULL_VER) && !defined(__ICL) -# define HEDLEY_MSVC_VERSION HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 1000000, (_MSC_FULL_VER % 1000000) / 10000, (_MSC_FULL_VER % 10000) / 10) -#elif defined(_MSC_VER) && !defined(__ICL) -# define HEDLEY_MSVC_VERSION HEDLEY_VERSION_ENCODE(_MSC_VER / 100, _MSC_VER % 100, 0) -#endif - -#if defined(HEDLEY_MSVC_VERSION_CHECK) -# undef HEDLEY_MSVC_VERSION_CHECK -#endif -#if !defined(HEDLEY_MSVC_VERSION) -# define HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (0) -#elif defined(_MSC_VER) && (_MSC_VER >= 1400) -# define HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 10000000) + (minor * 100000) + (patch))) -#elif defined(_MSC_VER) && (_MSC_VER >= 1200) -# define HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 1000000) + (minor * 10000) + (patch))) -#else -# define HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_VER >= ((major * 100) + (minor))) -#endif - -#if defined(HEDLEY_INTEL_VERSION) -# undef HEDLEY_INTEL_VERSION -#endif -#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && !defined(__ICL) -# define HEDLEY_INTEL_VERSION HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, __INTEL_COMPILER_UPDATE) -#elif defined(__INTEL_COMPILER) && !defined(__ICL) -# define HEDLEY_INTEL_VERSION HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, 0) -#endif - -#if defined(HEDLEY_INTEL_VERSION_CHECK) -# undef HEDLEY_INTEL_VERSION_CHECK -#endif -#if defined(HEDLEY_INTEL_VERSION) -# define HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (HEDLEY_INTEL_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_INTEL_CL_VERSION) -# undef HEDLEY_INTEL_CL_VERSION -#endif -#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && defined(__ICL) -# define HEDLEY_INTEL_CL_VERSION HEDLEY_VERSION_ENCODE(__INTEL_COMPILER, __INTEL_COMPILER_UPDATE, 0) -#endif - -#if defined(HEDLEY_INTEL_CL_VERSION_CHECK) -# undef HEDLEY_INTEL_CL_VERSION_CHECK -#endif -#if defined(HEDLEY_INTEL_CL_VERSION) -# define HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (HEDLEY_INTEL_CL_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_PGI_VERSION) -# undef HEDLEY_PGI_VERSION -#endif -#if defined(__PGI) && defined(__PGIC__) && defined(__PGIC_MINOR__) && defined(__PGIC_PATCHLEVEL__) -# define HEDLEY_PGI_VERSION HEDLEY_VERSION_ENCODE(__PGIC__, __PGIC_MINOR__, __PGIC_PATCHLEVEL__) -#endif - -#if defined(HEDLEY_PGI_VERSION_CHECK) -# undef HEDLEY_PGI_VERSION_CHECK -#endif -#if defined(HEDLEY_PGI_VERSION) -# define HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (HEDLEY_PGI_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_SUNPRO_VERSION) -# undef HEDLEY_SUNPRO_VERSION -#endif -#if defined(__SUNPRO_C) && (__SUNPRO_C > 0x1000) -# define HEDLEY_SUNPRO_VERSION HEDLEY_VERSION_ENCODE((((__SUNPRO_C >> 16) & 0xf) * 10) + ((__SUNPRO_C >> 12) & 0xf), (((__SUNPRO_C >> 8) & 0xf) * 10) + ((__SUNPRO_C >> 4) & 0xf), (__SUNPRO_C & 0xf) * 10) -#elif defined(__SUNPRO_C) -# define HEDLEY_SUNPRO_VERSION HEDLEY_VERSION_ENCODE((__SUNPRO_C >> 8) & 0xf, (__SUNPRO_C >> 4) & 0xf, (__SUNPRO_C) & 0xf) -#elif defined(__SUNPRO_CC) && (__SUNPRO_CC > 0x1000) -# define HEDLEY_SUNPRO_VERSION HEDLEY_VERSION_ENCODE((((__SUNPRO_CC >> 16) & 0xf) * 10) + ((__SUNPRO_CC >> 12) & 0xf), (((__SUNPRO_CC >> 8) & 0xf) * 10) + ((__SUNPRO_CC >> 4) & 0xf), (__SUNPRO_CC & 0xf) * 10) -#elif defined(__SUNPRO_CC) -# define HEDLEY_SUNPRO_VERSION HEDLEY_VERSION_ENCODE((__SUNPRO_CC >> 8) & 0xf, (__SUNPRO_CC >> 4) & 0xf, (__SUNPRO_CC) & 0xf) -#endif - -#if defined(HEDLEY_SUNPRO_VERSION_CHECK) -# undef HEDLEY_SUNPRO_VERSION_CHECK -#endif -#if defined(HEDLEY_SUNPRO_VERSION) -# define HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (HEDLEY_SUNPRO_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_EMSCRIPTEN_VERSION) -# undef HEDLEY_EMSCRIPTEN_VERSION -#endif -#if defined(__EMSCRIPTEN__) -# define HEDLEY_EMSCRIPTEN_VERSION HEDLEY_VERSION_ENCODE(__EMSCRIPTEN_major__, __EMSCRIPTEN_minor__, __EMSCRIPTEN_tiny__) -#endif - -#if defined(HEDLEY_EMSCRIPTEN_VERSION_CHECK) -# undef HEDLEY_EMSCRIPTEN_VERSION_CHECK -#endif -#if defined(HEDLEY_EMSCRIPTEN_VERSION) -# define HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (HEDLEY_EMSCRIPTEN_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_ARM_VERSION) -# undef HEDLEY_ARM_VERSION -#endif -#if defined(__CC_ARM) && defined(__ARMCOMPILER_VERSION) -# define HEDLEY_ARM_VERSION HEDLEY_VERSION_ENCODE(__ARMCOMPILER_VERSION / 1000000, (__ARMCOMPILER_VERSION % 1000000) / 10000, (__ARMCOMPILER_VERSION % 10000) / 100) -#elif defined(__CC_ARM) && defined(__ARMCC_VERSION) -# define HEDLEY_ARM_VERSION HEDLEY_VERSION_ENCODE(__ARMCC_VERSION / 1000000, (__ARMCC_VERSION % 1000000) / 10000, (__ARMCC_VERSION % 10000) / 100) -#endif - -#if defined(HEDLEY_ARM_VERSION_CHECK) -# undef HEDLEY_ARM_VERSION_CHECK -#endif -#if defined(HEDLEY_ARM_VERSION) -# define HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (HEDLEY_ARM_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_IBM_VERSION) -# undef HEDLEY_IBM_VERSION -#endif -#if defined(__ibmxl__) -# define HEDLEY_IBM_VERSION HEDLEY_VERSION_ENCODE(__ibmxl_version__, __ibmxl_release__, __ibmxl_modification__) -#elif defined(__xlC__) && defined(__xlC_ver__) -# define HEDLEY_IBM_VERSION HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, (__xlC_ver__ >> 8) & 0xff) -#elif defined(__xlC__) -# define HEDLEY_IBM_VERSION HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, 0) -#endif - -#if defined(HEDLEY_IBM_VERSION_CHECK) -# undef HEDLEY_IBM_VERSION_CHECK -#endif -#if defined(HEDLEY_IBM_VERSION) -# define HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (HEDLEY_IBM_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_TI_VERSION) -# undef HEDLEY_TI_VERSION -#endif -#if \ - defined(__TI_COMPILER_VERSION__) && \ - ( \ - defined(__TMS470__) || defined(__TI_ARM__) || \ - defined(__MSP430__) || \ - defined(__TMS320C2000__) \ - ) -# if (__TI_COMPILER_VERSION__ >= 16000000) -# define HEDLEY_TI_VERSION HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) -# endif -#endif - -#if defined(HEDLEY_TI_VERSION_CHECK) -# undef HEDLEY_TI_VERSION_CHECK -#endif -#if defined(HEDLEY_TI_VERSION) -# define HEDLEY_TI_VERSION_CHECK(major,minor,patch) (HEDLEY_TI_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_TI_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_TI_CL2000_VERSION) -# undef HEDLEY_TI_CL2000_VERSION -#endif -#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C2000__) -# define HEDLEY_TI_CL2000_VERSION HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) -#endif - -#if defined(HEDLEY_TI_CL2000_VERSION_CHECK) -# undef HEDLEY_TI_CL2000_VERSION_CHECK -#endif -#if defined(HEDLEY_TI_CL2000_VERSION) -# define HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (HEDLEY_TI_CL2000_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_TI_CL430_VERSION) -# undef HEDLEY_TI_CL430_VERSION -#endif -#if defined(__TI_COMPILER_VERSION__) && defined(__MSP430__) -# define HEDLEY_TI_CL430_VERSION HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) -#endif - -#if defined(HEDLEY_TI_CL430_VERSION_CHECK) -# undef HEDLEY_TI_CL430_VERSION_CHECK -#endif -#if defined(HEDLEY_TI_CL430_VERSION) -# define HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (HEDLEY_TI_CL430_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_TI_ARMCL_VERSION) -# undef HEDLEY_TI_ARMCL_VERSION -#endif -#if defined(__TI_COMPILER_VERSION__) && (defined(__TMS470__) || defined(__TI_ARM__)) -# define HEDLEY_TI_ARMCL_VERSION HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) -#endif - -#if defined(HEDLEY_TI_ARMCL_VERSION_CHECK) -# undef HEDLEY_TI_ARMCL_VERSION_CHECK -#endif -#if defined(HEDLEY_TI_ARMCL_VERSION) -# define HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (HEDLEY_TI_ARMCL_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_TI_CL6X_VERSION) -# undef HEDLEY_TI_CL6X_VERSION -#endif -#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C6X__) -# define HEDLEY_TI_CL6X_VERSION HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) -#endif - -#if defined(HEDLEY_TI_CL6X_VERSION_CHECK) -# undef HEDLEY_TI_CL6X_VERSION_CHECK -#endif -#if defined(HEDLEY_TI_CL6X_VERSION) -# define HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (HEDLEY_TI_CL6X_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_TI_CL7X_VERSION) -# undef HEDLEY_TI_CL7X_VERSION -#endif -#if defined(__TI_COMPILER_VERSION__) && defined(__C7000__) -# define HEDLEY_TI_CL7X_VERSION HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) -#endif - -#if defined(HEDLEY_TI_CL7X_VERSION_CHECK) -# undef HEDLEY_TI_CL7X_VERSION_CHECK -#endif -#if defined(HEDLEY_TI_CL7X_VERSION) -# define HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (HEDLEY_TI_CL7X_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_TI_CLPRU_VERSION) -# undef HEDLEY_TI_CLPRU_VERSION -#endif -#if defined(__TI_COMPILER_VERSION__) && defined(__PRU__) -# define HEDLEY_TI_CLPRU_VERSION HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) -#endif - -#if defined(HEDLEY_TI_CLPRU_VERSION_CHECK) -# undef HEDLEY_TI_CLPRU_VERSION_CHECK -#endif -#if defined(HEDLEY_TI_CLPRU_VERSION) -# define HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (HEDLEY_TI_CLPRU_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_CRAY_VERSION) -# undef HEDLEY_CRAY_VERSION -#endif -#if defined(_CRAYC) -# if defined(_RELEASE_PATCHLEVEL) -# define HEDLEY_CRAY_VERSION HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, _RELEASE_PATCHLEVEL) -# else -# define HEDLEY_CRAY_VERSION HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, 0) -# endif -#endif - -#if defined(HEDLEY_CRAY_VERSION_CHECK) -# undef HEDLEY_CRAY_VERSION_CHECK -#endif -#if defined(HEDLEY_CRAY_VERSION) -# define HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (HEDLEY_CRAY_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_IAR_VERSION) -# undef HEDLEY_IAR_VERSION -#endif -#if defined(__IAR_SYSTEMS_ICC__) -# if __VER__ > 1000 -# define HEDLEY_IAR_VERSION HEDLEY_VERSION_ENCODE((__VER__ / 1000000), ((__VER__ / 1000) % 1000), (__VER__ % 1000)) -# else -# define HEDLEY_IAR_VERSION HEDLEY_VERSION_ENCODE(__VER__ / 100, __VER__ % 100, 0) -# endif -#endif - -#if defined(HEDLEY_IAR_VERSION_CHECK) -# undef HEDLEY_IAR_VERSION_CHECK -#endif -#if defined(HEDLEY_IAR_VERSION) -# define HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (HEDLEY_IAR_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_TINYC_VERSION) -# undef HEDLEY_TINYC_VERSION -#endif -#if defined(__TINYC__) -# define HEDLEY_TINYC_VERSION HEDLEY_VERSION_ENCODE(__TINYC__ / 1000, (__TINYC__ / 100) % 10, __TINYC__ % 100) -#endif - -#if defined(HEDLEY_TINYC_VERSION_CHECK) -# undef HEDLEY_TINYC_VERSION_CHECK -#endif -#if defined(HEDLEY_TINYC_VERSION) -# define HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (HEDLEY_TINYC_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_DMC_VERSION) -# undef HEDLEY_DMC_VERSION -#endif -#if defined(__DMC__) -# define HEDLEY_DMC_VERSION HEDLEY_VERSION_ENCODE(__DMC__ >> 8, (__DMC__ >> 4) & 0xf, __DMC__ & 0xf) -#endif - -#if defined(HEDLEY_DMC_VERSION_CHECK) -# undef HEDLEY_DMC_VERSION_CHECK -#endif -#if defined(HEDLEY_DMC_VERSION) -# define HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (HEDLEY_DMC_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_COMPCERT_VERSION) -# undef HEDLEY_COMPCERT_VERSION -#endif -#if defined(__COMPCERT_VERSION__) -# define HEDLEY_COMPCERT_VERSION HEDLEY_VERSION_ENCODE(__COMPCERT_VERSION__ / 10000, (__COMPCERT_VERSION__ / 100) % 100, __COMPCERT_VERSION__ % 100) -#endif - -#if defined(HEDLEY_COMPCERT_VERSION_CHECK) -# undef HEDLEY_COMPCERT_VERSION_CHECK -#endif -#if defined(HEDLEY_COMPCERT_VERSION) -# define HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (HEDLEY_COMPCERT_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_PELLES_VERSION) -# undef HEDLEY_PELLES_VERSION -#endif -#if defined(__POCC__) -# define HEDLEY_PELLES_VERSION HEDLEY_VERSION_ENCODE(__POCC__ / 100, __POCC__ % 100, 0) -#endif - -#if defined(HEDLEY_PELLES_VERSION_CHECK) -# undef HEDLEY_PELLES_VERSION_CHECK -#endif -#if defined(HEDLEY_PELLES_VERSION) -# define HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (HEDLEY_PELLES_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_MCST_LCC_VERSION) -# undef HEDLEY_MCST_LCC_VERSION -#endif -#if defined(__LCC__) && defined(__LCC_MINOR__) -# define HEDLEY_MCST_LCC_VERSION HEDLEY_VERSION_ENCODE(__LCC__ / 100, __LCC__ % 100, __LCC_MINOR__) -#endif - -#if defined(HEDLEY_MCST_LCC_VERSION_CHECK) -# undef HEDLEY_MCST_LCC_VERSION_CHECK -#endif -#if defined(HEDLEY_MCST_LCC_VERSION) -# define HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (HEDLEY_MCST_LCC_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_GCC_VERSION) -# undef HEDLEY_GCC_VERSION -#endif -#if \ - defined(HEDLEY_GNUC_VERSION) && \ - !defined(__clang__) && \ - !defined(HEDLEY_INTEL_VERSION) && \ - !defined(HEDLEY_PGI_VERSION) && \ - !defined(HEDLEY_ARM_VERSION) && \ - !defined(HEDLEY_CRAY_VERSION) && \ - !defined(HEDLEY_TI_VERSION) && \ - !defined(HEDLEY_TI_ARMCL_VERSION) && \ - !defined(HEDLEY_TI_CL430_VERSION) && \ - !defined(HEDLEY_TI_CL2000_VERSION) && \ - !defined(HEDLEY_TI_CL6X_VERSION) && \ - !defined(HEDLEY_TI_CL7X_VERSION) && \ - !defined(HEDLEY_TI_CLPRU_VERSION) && \ - !defined(__COMPCERT__) && \ - !defined(HEDLEY_MCST_LCC_VERSION) -# define HEDLEY_GCC_VERSION HEDLEY_GNUC_VERSION -#endif - -#if defined(HEDLEY_GCC_VERSION_CHECK) -# undef HEDLEY_GCC_VERSION_CHECK -#endif -#if defined(HEDLEY_GCC_VERSION) -# define HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (HEDLEY_GCC_VERSION >= HEDLEY_VERSION_ENCODE(major, minor, patch)) -#else -# define HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (0) -#endif - -#if defined(HEDLEY_HAS_ATTRIBUTE) -# undef HEDLEY_HAS_ATTRIBUTE -#endif -#if \ - defined(__has_attribute) && \ - ( \ - (!defined(HEDLEY_IAR_VERSION) || HEDLEY_IAR_VERSION_CHECK(8,5,9)) \ - ) -# define HEDLEY_HAS_ATTRIBUTE(attribute) __has_attribute(attribute) -#else -# define HEDLEY_HAS_ATTRIBUTE(attribute) (0) -#endif - -#if defined(HEDLEY_GNUC_HAS_ATTRIBUTE) -# undef HEDLEY_GNUC_HAS_ATTRIBUTE -#endif -#if defined(__has_attribute) -# define HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) HEDLEY_HAS_ATTRIBUTE(attribute) -#else -# define HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) -#endif - -#if defined(HEDLEY_GCC_HAS_ATTRIBUTE) -# undef HEDLEY_GCC_HAS_ATTRIBUTE -#endif -#if defined(__has_attribute) -# define HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) HEDLEY_HAS_ATTRIBUTE(attribute) -#else -# define HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) HEDLEY_GCC_VERSION_CHECK(major,minor,patch) -#endif - -#if defined(HEDLEY_HAS_CPP_ATTRIBUTE) -# undef HEDLEY_HAS_CPP_ATTRIBUTE -#endif -#if \ - defined(__has_cpp_attribute) && \ - defined(__cplusplus) && \ - (!defined(HEDLEY_SUNPRO_VERSION) || HEDLEY_SUNPRO_VERSION_CHECK(5,15,0)) -# define HEDLEY_HAS_CPP_ATTRIBUTE(attribute) __has_cpp_attribute(attribute) -#else -# define HEDLEY_HAS_CPP_ATTRIBUTE(attribute) (0) -#endif - -#if defined(HEDLEY_HAS_CPP_ATTRIBUTE_NS) -# undef HEDLEY_HAS_CPP_ATTRIBUTE_NS -#endif -#if !defined(__cplusplus) || !defined(__has_cpp_attribute) -# define HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0) -#elif \ - !defined(HEDLEY_PGI_VERSION) && \ - !defined(HEDLEY_IAR_VERSION) && \ - (!defined(HEDLEY_SUNPRO_VERSION) || HEDLEY_SUNPRO_VERSION_CHECK(5,15,0)) && \ - (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,20,0)) -# define HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) HEDLEY_HAS_CPP_ATTRIBUTE(ns::attribute) -#else -# define HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0) -#endif - -#if defined(HEDLEY_GNUC_HAS_CPP_ATTRIBUTE) -# undef HEDLEY_GNUC_HAS_CPP_ATTRIBUTE -#endif -#if defined(__has_cpp_attribute) && defined(__cplusplus) -# define HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute) -#else -# define HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) -#endif - -#if defined(HEDLEY_GCC_HAS_CPP_ATTRIBUTE) -# undef HEDLEY_GCC_HAS_CPP_ATTRIBUTE -#endif -#if defined(__has_cpp_attribute) && defined(__cplusplus) -# define HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute) -#else -# define HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) HEDLEY_GCC_VERSION_CHECK(major,minor,patch) -#endif - -#if defined(HEDLEY_HAS_BUILTIN) -# undef HEDLEY_HAS_BUILTIN -#endif -#if defined(__has_builtin) -# define HEDLEY_HAS_BUILTIN(builtin) __has_builtin(builtin) -#else -# define HEDLEY_HAS_BUILTIN(builtin) (0) -#endif - -#if defined(HEDLEY_GNUC_HAS_BUILTIN) -# undef HEDLEY_GNUC_HAS_BUILTIN -#endif -#if defined(__has_builtin) -# define HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin) -#else -# define HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) -#endif - -#if defined(HEDLEY_GCC_HAS_BUILTIN) -# undef HEDLEY_GCC_HAS_BUILTIN -#endif -#if defined(__has_builtin) -# define HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin) -#else -# define HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) HEDLEY_GCC_VERSION_CHECK(major,minor,patch) -#endif - -#if defined(HEDLEY_HAS_FEATURE) -# undef HEDLEY_HAS_FEATURE -#endif -#if defined(__has_feature) -# define HEDLEY_HAS_FEATURE(feature) __has_feature(feature) -#else -# define HEDLEY_HAS_FEATURE(feature) (0) -#endif - -#if defined(HEDLEY_GNUC_HAS_FEATURE) -# undef HEDLEY_GNUC_HAS_FEATURE -#endif -#if defined(__has_feature) -# define HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature) -#else -# define HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) -#endif - -#if defined(HEDLEY_GCC_HAS_FEATURE) -# undef HEDLEY_GCC_HAS_FEATURE -#endif -#if defined(__has_feature) -# define HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature) -#else -# define HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) HEDLEY_GCC_VERSION_CHECK(major,minor,patch) -#endif - -#if defined(HEDLEY_HAS_EXTENSION) -# undef HEDLEY_HAS_EXTENSION -#endif -#if defined(__has_extension) -# define HEDLEY_HAS_EXTENSION(extension) __has_extension(extension) -#else -# define HEDLEY_HAS_EXTENSION(extension) (0) -#endif - -#if defined(HEDLEY_GNUC_HAS_EXTENSION) -# undef HEDLEY_GNUC_HAS_EXTENSION -#endif -#if defined(__has_extension) -# define HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension) -#else -# define HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) -#endif - -#if defined(HEDLEY_GCC_HAS_EXTENSION) -# undef HEDLEY_GCC_HAS_EXTENSION -#endif -#if defined(__has_extension) -# define HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension) -#else -# define HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) HEDLEY_GCC_VERSION_CHECK(major,minor,patch) -#endif - -#if defined(HEDLEY_HAS_DECLSPEC_ATTRIBUTE) -# undef HEDLEY_HAS_DECLSPEC_ATTRIBUTE -#endif -#if defined(__has_declspec_attribute) -# define HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) __has_declspec_attribute(attribute) -#else -# define HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) (0) -#endif - -#if defined(HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE) -# undef HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE -#endif -#if defined(__has_declspec_attribute) -# define HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute) -#else -# define HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) -#endif - -#if defined(HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE) -# undef HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE -#endif -#if defined(__has_declspec_attribute) -# define HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute) -#else -# define HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) HEDLEY_GCC_VERSION_CHECK(major,minor,patch) -#endif - -#if defined(HEDLEY_HAS_WARNING) -# undef HEDLEY_HAS_WARNING -#endif -#if defined(__has_warning) -# define HEDLEY_HAS_WARNING(warning) __has_warning(warning) -#else -# define HEDLEY_HAS_WARNING(warning) (0) -#endif - -#if defined(HEDLEY_GNUC_HAS_WARNING) -# undef HEDLEY_GNUC_HAS_WARNING -#endif -#if defined(__has_warning) -# define HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning) -#else -# define HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) -#endif - -#if defined(HEDLEY_GCC_HAS_WARNING) -# undef HEDLEY_GCC_HAS_WARNING -#endif -#if defined(__has_warning) -# define HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning) -#else -# define HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) HEDLEY_GCC_VERSION_CHECK(major,minor,patch) -#endif - -#if \ - (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \ - defined(__clang__) || \ - HEDLEY_GCC_VERSION_CHECK(3,0,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_IAR_VERSION_CHECK(8,0,0) || \ - HEDLEY_PGI_VERSION_CHECK(18,4,0) || \ - HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - HEDLEY_TI_VERSION_CHECK(15,12,0) || \ - HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \ - HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \ - HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(7,0,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ - HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ - HEDLEY_CRAY_VERSION_CHECK(5,0,0) || \ - HEDLEY_TINYC_VERSION_CHECK(0,9,17) || \ - HEDLEY_SUNPRO_VERSION_CHECK(8,0,0) || \ - (HEDLEY_IBM_VERSION_CHECK(10,1,0) && defined(__C99_PRAGMA_OPERATOR)) -# define HEDLEY_PRAGMA(value) _Pragma(#value) -#elif HEDLEY_MSVC_VERSION_CHECK(15,0,0) -# define HEDLEY_PRAGMA(value) __pragma(value) -#else -# define HEDLEY_PRAGMA(value) -#endif - -#if defined(HEDLEY_DIAGNOSTIC_PUSH) -# undef HEDLEY_DIAGNOSTIC_PUSH -#endif -#if defined(HEDLEY_DIAGNOSTIC_POP) -# undef HEDLEY_DIAGNOSTIC_POP -#endif -#if defined(__clang__) -# define HEDLEY_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push") -# define HEDLEY_DIAGNOSTIC_POP _Pragma("clang diagnostic pop") -#elif HEDLEY_INTEL_VERSION_CHECK(13,0,0) -# define HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") -# define HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") -#elif HEDLEY_GCC_VERSION_CHECK(4,6,0) -# define HEDLEY_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push") -# define HEDLEY_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop") -#elif \ - HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \ - HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) -# define HEDLEY_DIAGNOSTIC_PUSH __pragma(warning(push)) -# define HEDLEY_DIAGNOSTIC_POP __pragma(warning(pop)) -#elif HEDLEY_ARM_VERSION_CHECK(5,6,0) -# define HEDLEY_DIAGNOSTIC_PUSH _Pragma("push") -# define HEDLEY_DIAGNOSTIC_POP _Pragma("pop") -#elif \ - HEDLEY_TI_VERSION_CHECK(15,12,0) || \ - HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ - HEDLEY_TI_CL430_VERSION_CHECK(4,4,0) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ - HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) -# define HEDLEY_DIAGNOSTIC_PUSH _Pragma("diag_push") -# define HEDLEY_DIAGNOSTIC_POP _Pragma("diag_pop") -#elif HEDLEY_PELLES_VERSION_CHECK(2,90,0) -# define HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") -# define HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") -#else -# define HEDLEY_DIAGNOSTIC_PUSH -# define HEDLEY_DIAGNOSTIC_POP -#endif - -/* HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ is for - HEDLEY INTERNAL USE ONLY. API subject to change without notice. */ -#if defined(HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_) -# undef HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ -#endif -#if defined(__cplusplus) -# if HEDLEY_HAS_WARNING("-Wc++98-compat") -# if HEDLEY_HAS_WARNING("-Wc++17-extensions") -# if HEDLEY_HAS_WARNING("-Wc++1z-extensions") -# define HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ - HEDLEY_DIAGNOSTIC_PUSH \ - _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ - _Pragma("clang diagnostic ignored \"-Wc++17-extensions\"") \ - _Pragma("clang diagnostic ignored \"-Wc++1z-extensions\"") \ - xpr \ - HEDLEY_DIAGNOSTIC_POP -# else -# define HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ - HEDLEY_DIAGNOSTIC_PUSH \ - _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ - _Pragma("clang diagnostic ignored \"-Wc++17-extensions\"") \ - xpr \ - HEDLEY_DIAGNOSTIC_POP -# endif -# else -# define HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ - HEDLEY_DIAGNOSTIC_PUSH \ - _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ - xpr \ - HEDLEY_DIAGNOSTIC_POP -# endif -# endif -#endif -#if !defined(HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_) -# define HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(x) x -#endif - -#if defined(HEDLEY_CONST_CAST) -# undef HEDLEY_CONST_CAST -#endif -#if defined(__cplusplus) -# define HEDLEY_CONST_CAST(T, expr) (const_cast(expr)) -#elif \ - HEDLEY_HAS_WARNING("-Wcast-qual") || \ - HEDLEY_GCC_VERSION_CHECK(4,6,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) -# define HEDLEY_CONST_CAST(T, expr) (__extension__ ({ \ - HEDLEY_DIAGNOSTIC_PUSH \ - HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL \ - ((T) (expr)); \ - HEDLEY_DIAGNOSTIC_POP \ - })) -#else -# define HEDLEY_CONST_CAST(T, expr) ((T) (expr)) -#endif - -#if defined(HEDLEY_REINTERPRET_CAST) -# undef HEDLEY_REINTERPRET_CAST -#endif -#if defined(__cplusplus) -# define HEDLEY_REINTERPRET_CAST(T, expr) (reinterpret_cast(expr)) -#else -# define HEDLEY_REINTERPRET_CAST(T, expr) ((T) (expr)) -#endif - -#if defined(HEDLEY_STATIC_CAST) -# undef HEDLEY_STATIC_CAST -#endif -#if defined(__cplusplus) -# define HEDLEY_STATIC_CAST(T, expr) (static_cast(expr)) -#else -# define HEDLEY_STATIC_CAST(T, expr) ((T) (expr)) -#endif - -#if defined(HEDLEY_CPP_CAST) -# undef HEDLEY_CPP_CAST -#endif -#if defined(__cplusplus) -# if HEDLEY_HAS_WARNING("-Wold-style-cast") -# define HEDLEY_CPP_CAST(T, expr) \ - HEDLEY_DIAGNOSTIC_PUSH \ - _Pragma("clang diagnostic ignored \"-Wold-style-cast\"") \ - ((T) (expr)) \ - HEDLEY_DIAGNOSTIC_POP -# elif HEDLEY_IAR_VERSION_CHECK(8,3,0) -# define HEDLEY_CPP_CAST(T, expr) \ - HEDLEY_DIAGNOSTIC_PUSH \ - _Pragma("diag_suppress=Pe137") \ - HEDLEY_DIAGNOSTIC_POP -# else -# define HEDLEY_CPP_CAST(T, expr) ((T) (expr)) -# endif -#else -# define HEDLEY_CPP_CAST(T, expr) (expr) -#endif - -#if defined(HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED) -# undef HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED -#endif -#if HEDLEY_HAS_WARNING("-Wdeprecated-declarations") -# define HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"") -#elif HEDLEY_INTEL_VERSION_CHECK(13,0,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warning(disable:1478 1786)") -#elif HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:1478 1786)) -#elif HEDLEY_PGI_VERSION_CHECK(20,7,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1216,1444,1445") -#elif HEDLEY_PGI_VERSION_CHECK(17,10,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444") -#elif HEDLEY_GCC_VERSION_CHECK(4,3,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") -#elif HEDLEY_MSVC_VERSION_CHECK(15,0,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:4996)) -#elif HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444") -#elif \ - HEDLEY_TI_VERSION_CHECK(15,12,0) || \ - (HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ - (HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ - (HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ - (HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ - HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1291,1718") -#elif HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && !defined(__cplusplus) -# define HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,E_DEPRECATED_ATT,E_DEPRECATED_ATT_MESS)") -#elif HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && defined(__cplusplus) -# define HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,symdeprecated,symdeprecated2)") -#elif HEDLEY_IAR_VERSION_CHECK(8,0,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress=Pe1444,Pe1215") -#elif HEDLEY_PELLES_VERSION_CHECK(2,90,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warn(disable:2241)") -#else -# define HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED -#endif - -#if defined(HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS) -# undef HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS -#endif -#if HEDLEY_HAS_WARNING("-Wunknown-pragmas") -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("clang diagnostic ignored \"-Wunknown-pragmas\"") -#elif HEDLEY_INTEL_VERSION_CHECK(13,0,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("warning(disable:161)") -#elif HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:161)) -#elif HEDLEY_PGI_VERSION_CHECK(17,10,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 1675") -#elif HEDLEY_GCC_VERSION_CHECK(4,3,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("GCC diagnostic ignored \"-Wunknown-pragmas\"") -#elif HEDLEY_MSVC_VERSION_CHECK(15,0,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:4068)) -#elif \ - HEDLEY_TI_VERSION_CHECK(16,9,0) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ - HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163") -#elif HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163") -#elif HEDLEY_IAR_VERSION_CHECK(8,0,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress=Pe161") -#elif HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 161") -#else -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS -#endif - -#if defined(HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES) -# undef HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES -#endif -#if HEDLEY_HAS_WARNING("-Wunknown-attributes") -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("clang diagnostic ignored \"-Wunknown-attributes\"") -#elif HEDLEY_GCC_VERSION_CHECK(4,6,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") -#elif HEDLEY_INTEL_VERSION_CHECK(17,0,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("warning(disable:1292)") -#elif HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:1292)) -#elif HEDLEY_MSVC_VERSION_CHECK(19,0,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:5030)) -#elif HEDLEY_PGI_VERSION_CHECK(20,7,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097,1098") -#elif HEDLEY_PGI_VERSION_CHECK(17,10,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097") -#elif HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("error_messages(off,attrskipunsup)") -#elif \ - HEDLEY_TI_VERSION_CHECK(18,1,0) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1173") -#elif HEDLEY_IAR_VERSION_CHECK(8,0,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress=Pe1097") -#elif HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097") -#else -# define HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES -#endif - -#if defined(HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL) -# undef HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL -#endif -#if HEDLEY_HAS_WARNING("-Wcast-qual") -# define HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("clang diagnostic ignored \"-Wcast-qual\"") -#elif HEDLEY_INTEL_VERSION_CHECK(13,0,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("warning(disable:2203 2331)") -#elif HEDLEY_GCC_VERSION_CHECK(3,0,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("GCC diagnostic ignored \"-Wcast-qual\"") -#else -# define HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL -#endif - -#if defined(HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION) -# undef HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION -#endif -#if HEDLEY_HAS_WARNING("-Wunused-function") -# define HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("clang diagnostic ignored \"-Wunused-function\"") -#elif HEDLEY_GCC_VERSION_CHECK(3,4,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("GCC diagnostic ignored \"-Wunused-function\"") -#elif HEDLEY_MSVC_VERSION_CHECK(1,0,0) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION __pragma(warning(disable:4505)) -#elif HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("diag_suppress 3142") -#else -# define HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION -#endif - -#if defined(HEDLEY_DEPRECATED) -# undef HEDLEY_DEPRECATED -#endif -#if defined(HEDLEY_DEPRECATED_FOR) -# undef HEDLEY_DEPRECATED_FOR -#endif -#if \ - HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ - HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) -# define HEDLEY_DEPRECATED(since) __declspec(deprecated("Since " # since)) -# define HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated("Since " #since "; use " #replacement)) -#elif \ - (HEDLEY_HAS_EXTENSION(attribute_deprecated_with_message) && !defined(HEDLEY_IAR_VERSION)) || \ - HEDLEY_GCC_VERSION_CHECK(4,5,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_ARM_VERSION_CHECK(5,6,0) || \ - HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) || \ - HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ - HEDLEY_TI_VERSION_CHECK(18,1,0) || \ - HEDLEY_TI_ARMCL_VERSION_CHECK(18,1,0) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ - HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_DEPRECATED(since) __attribute__((__deprecated__("Since " #since))) -# define HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__("Since " #since "; use " #replacement))) -#elif defined(__cplusplus) && (__cplusplus >= 201402L) -# define HEDLEY_DEPRECATED(since) HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated("Since " #since)]]) -# define HEDLEY_DEPRECATED_FOR(since, replacement) HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated("Since " #since "; use " #replacement)]]) -#elif \ - HEDLEY_HAS_ATTRIBUTE(deprecated) || \ - HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ - HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - HEDLEY_TI_VERSION_CHECK(15,12,0) || \ - (HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ - (HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ - (HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ - (HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ - HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ - HEDLEY_IAR_VERSION_CHECK(8,10,0) -# define HEDLEY_DEPRECATED(since) __attribute__((__deprecated__)) -# define HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__)) -#elif \ - HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ - HEDLEY_PELLES_VERSION_CHECK(6,50,0) || \ - HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) -# define HEDLEY_DEPRECATED(since) __declspec(deprecated) -# define HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated) -#elif HEDLEY_IAR_VERSION_CHECK(8,0,0) -# define HEDLEY_DEPRECATED(since) _Pragma("deprecated") -# define HEDLEY_DEPRECATED_FOR(since, replacement) _Pragma("deprecated") -#else -# define HEDLEY_DEPRECATED(since) -# define HEDLEY_DEPRECATED_FOR(since, replacement) -#endif - -#if defined(HEDLEY_UNAVAILABLE) -# undef HEDLEY_UNAVAILABLE -#endif -#if \ - HEDLEY_HAS_ATTRIBUTE(warning) || \ - HEDLEY_GCC_VERSION_CHECK(4,3,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_UNAVAILABLE(available_since) __attribute__((__warning__("Not available until " #available_since))) -#else -# define HEDLEY_UNAVAILABLE(available_since) -#endif - -#if defined(HEDLEY_WARN_UNUSED_RESULT) -# undef HEDLEY_WARN_UNUSED_RESULT -#endif -#if defined(HEDLEY_WARN_UNUSED_RESULT_MSG) -# undef HEDLEY_WARN_UNUSED_RESULT_MSG -#endif -#if \ - HEDLEY_HAS_ATTRIBUTE(warn_unused_result) || \ - HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_TI_VERSION_CHECK(15,12,0) || \ - (HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ - (HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ - (HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ - (HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ - HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ - (HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \ - HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__)) -# define HEDLEY_WARN_UNUSED_RESULT_MSG(msg) __attribute__((__warn_unused_result__)) -#elif (HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard) >= 201907L) -# define HEDLEY_WARN_UNUSED_RESULT HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) -# define HEDLEY_WARN_UNUSED_RESULT_MSG(msg) HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard(msg)]]) -#elif HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard) -# define HEDLEY_WARN_UNUSED_RESULT HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) -# define HEDLEY_WARN_UNUSED_RESULT_MSG(msg) HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) -#elif defined(_Check_return_) /* SAL */ -# define HEDLEY_WARN_UNUSED_RESULT _Check_return_ -# define HEDLEY_WARN_UNUSED_RESULT_MSG(msg) _Check_return_ -#else -# define HEDLEY_WARN_UNUSED_RESULT -# define HEDLEY_WARN_UNUSED_RESULT_MSG(msg) -#endif - -#if defined(HEDLEY_SENTINEL) -# undef HEDLEY_SENTINEL -#endif -#if \ - HEDLEY_HAS_ATTRIBUTE(sentinel) || \ - HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_ARM_VERSION_CHECK(5,4,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_SENTINEL(position) __attribute__((__sentinel__(position))) -#else -# define HEDLEY_SENTINEL(position) -#endif - -#if defined(HEDLEY_NO_RETURN) -# undef HEDLEY_NO_RETURN -#endif -#if HEDLEY_IAR_VERSION_CHECK(8,0,0) -# define HEDLEY_NO_RETURN __noreturn -#elif \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_NO_RETURN __attribute__((__noreturn__)) -#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L -# define HEDLEY_NO_RETURN _Noreturn -#elif defined(__cplusplus) && (__cplusplus >= 201103L) -# define HEDLEY_NO_RETURN HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[noreturn]]) -#elif \ - HEDLEY_HAS_ATTRIBUTE(noreturn) || \ - HEDLEY_GCC_VERSION_CHECK(3,2,0) || \ - HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ - HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ - HEDLEY_TI_VERSION_CHECK(15,12,0) || \ - (HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ - (HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ - (HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ - (HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ - HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ - HEDLEY_IAR_VERSION_CHECK(8,10,0) -# define HEDLEY_NO_RETURN __attribute__((__noreturn__)) -#elif HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) -# define HEDLEY_NO_RETURN _Pragma("does_not_return") -#elif \ - HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ - HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) -# define HEDLEY_NO_RETURN __declspec(noreturn) -#elif HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus) -# define HEDLEY_NO_RETURN _Pragma("FUNC_NEVER_RETURNS;") -#elif HEDLEY_COMPCERT_VERSION_CHECK(3,2,0) -# define HEDLEY_NO_RETURN __attribute((noreturn)) -#elif HEDLEY_PELLES_VERSION_CHECK(9,0,0) -# define HEDLEY_NO_RETURN __declspec(noreturn) -#else -# define HEDLEY_NO_RETURN -#endif - -#if defined(HEDLEY_NO_ESCAPE) -# undef HEDLEY_NO_ESCAPE -#endif -#if HEDLEY_HAS_ATTRIBUTE(noescape) -# define HEDLEY_NO_ESCAPE __attribute__((__noescape__)) -#else -# define HEDLEY_NO_ESCAPE -#endif - -#if defined(HEDLEY_UNREACHABLE) -# undef HEDLEY_UNREACHABLE -#endif -#if defined(HEDLEY_UNREACHABLE_RETURN) -# undef HEDLEY_UNREACHABLE_RETURN -#endif -#if defined(HEDLEY_ASSUME) -# undef HEDLEY_ASSUME -#endif -#if \ - HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) -# define HEDLEY_ASSUME(expr) __assume(expr) -#elif HEDLEY_HAS_BUILTIN(__builtin_assume) -# define HEDLEY_ASSUME(expr) __builtin_assume(expr) -#elif \ - HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) -# if defined(__cplusplus) -# define HEDLEY_ASSUME(expr) std::_nassert(expr) -# else -# define HEDLEY_ASSUME(expr) _nassert(expr) -# endif -#endif -#if \ - (HEDLEY_HAS_BUILTIN(__builtin_unreachable) && (!defined(HEDLEY_ARM_VERSION))) || \ - HEDLEY_GCC_VERSION_CHECK(4,5,0) || \ - HEDLEY_PGI_VERSION_CHECK(18,10,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_IBM_VERSION_CHECK(13,1,5) || \ - HEDLEY_CRAY_VERSION_CHECK(10,0,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_UNREACHABLE() __builtin_unreachable() -#elif defined(HEDLEY_ASSUME) -# define HEDLEY_UNREACHABLE() HEDLEY_ASSUME(0) -#endif -#if !defined(HEDLEY_ASSUME) -# if defined(HEDLEY_UNREACHABLE) -# define HEDLEY_ASSUME(expr) HEDLEY_STATIC_CAST(void, ((expr) ? 1 : (HEDLEY_UNREACHABLE(), 1))) -# else -# define HEDLEY_ASSUME(expr) HEDLEY_STATIC_CAST(void, expr) -# endif -#endif -#if defined(HEDLEY_UNREACHABLE) -# if \ - HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) -# define HEDLEY_UNREACHABLE_RETURN(value) return (HEDLEY_STATIC_CAST(void, HEDLEY_ASSUME(0)), (value)) -# else -# define HEDLEY_UNREACHABLE_RETURN(value) HEDLEY_UNREACHABLE() -# endif -#else -# define HEDLEY_UNREACHABLE_RETURN(value) return (value) -#endif -#if !defined(HEDLEY_UNREACHABLE) -# define HEDLEY_UNREACHABLE() HEDLEY_ASSUME(0) -#endif - -HEDLEY_DIAGNOSTIC_PUSH -#if HEDLEY_HAS_WARNING("-Wpedantic") -# pragma clang diagnostic ignored "-Wpedantic" -#endif -#if HEDLEY_HAS_WARNING("-Wc++98-compat-pedantic") && defined(__cplusplus) -# pragma clang diagnostic ignored "-Wc++98-compat-pedantic" -#endif -#if HEDLEY_GCC_HAS_WARNING("-Wvariadic-macros",4,0,0) -# if defined(__clang__) -# pragma clang diagnostic ignored "-Wvariadic-macros" -# elif defined(HEDLEY_GCC_VERSION) -# pragma GCC diagnostic ignored "-Wvariadic-macros" -# endif -#endif -#if defined(HEDLEY_NON_NULL) -# undef HEDLEY_NON_NULL -#endif -#if \ - HEDLEY_HAS_ATTRIBUTE(nonnull) || \ - HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_ARM_VERSION_CHECK(4,1,0) -# define HEDLEY_NON_NULL(...) __attribute__((__nonnull__(__VA_ARGS__))) -#else -# define HEDLEY_NON_NULL(...) -#endif -HEDLEY_DIAGNOSTIC_POP - -#if defined(HEDLEY_PRINTF_FORMAT) -# undef HEDLEY_PRINTF_FORMAT -#endif -#if defined(__MINGW32__) && HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && !defined(__USE_MINGW_ANSI_STDIO) -# define HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(ms_printf, string_idx, first_to_check))) -#elif defined(__MINGW32__) && HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && defined(__USE_MINGW_ANSI_STDIO) -# define HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(gnu_printf, string_idx, first_to_check))) -#elif \ - HEDLEY_HAS_ATTRIBUTE(format) || \ - HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_ARM_VERSION_CHECK(5,6,0) || \ - HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ - HEDLEY_TI_VERSION_CHECK(15,12,0) || \ - (HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ - (HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ - (HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ - (HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ - HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(__printf__, string_idx, first_to_check))) -#elif HEDLEY_PELLES_VERSION_CHECK(6,0,0) -# define HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __declspec(vaformat(printf,string_idx,first_to_check)) -#else -# define HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) -#endif - -#if defined(HEDLEY_CONSTEXPR) -# undef HEDLEY_CONSTEXPR -#endif -#if defined(__cplusplus) -# if __cplusplus >= 201103L -# define HEDLEY_CONSTEXPR HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(constexpr) -# endif -#endif -#if !defined(HEDLEY_CONSTEXPR) -# define HEDLEY_CONSTEXPR -#endif - -#if defined(HEDLEY_PREDICT) -# undef HEDLEY_PREDICT -#endif -#if defined(HEDLEY_LIKELY) -# undef HEDLEY_LIKELY -#endif -#if defined(HEDLEY_UNLIKELY) -# undef HEDLEY_UNLIKELY -#endif -#if defined(HEDLEY_UNPREDICTABLE) -# undef HEDLEY_UNPREDICTABLE -#endif -#if HEDLEY_HAS_BUILTIN(__builtin_unpredictable) -# define HEDLEY_UNPREDICTABLE(expr) __builtin_unpredictable((expr)) -#endif -#if \ - (HEDLEY_HAS_BUILTIN(__builtin_expect_with_probability) && !defined(HEDLEY_PGI_VERSION)) || \ - HEDLEY_GCC_VERSION_CHECK(9,0,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_PREDICT(expr, value, probability) __builtin_expect_with_probability( (expr), (value), (probability)) -# define HEDLEY_PREDICT_TRUE(expr, probability) __builtin_expect_with_probability(!!(expr), 1 , (probability)) -# define HEDLEY_PREDICT_FALSE(expr, probability) __builtin_expect_with_probability(!!(expr), 0 , (probability)) -# define HEDLEY_LIKELY(expr) __builtin_expect (!!(expr), 1 ) -# define HEDLEY_UNLIKELY(expr) __builtin_expect (!!(expr), 0 ) -#elif \ - (HEDLEY_HAS_BUILTIN(__builtin_expect) && !defined(HEDLEY_INTEL_CL_VERSION)) || \ - HEDLEY_GCC_VERSION_CHECK(3,0,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - (HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \ - HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ - HEDLEY_TI_VERSION_CHECK(15,12,0) || \ - HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \ - HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \ - HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ - HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ - HEDLEY_TINYC_VERSION_CHECK(0,9,27) || \ - HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_PREDICT(expr, expected, probability) \ - (((probability) >= 0.9) ? __builtin_expect((expr), (expected)) : (HEDLEY_STATIC_CAST(void, expected), (expr))) -# define HEDLEY_PREDICT_TRUE(expr, probability) \ - (__extension__ ({ \ - double hedley_probability_ = (probability); \ - ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 1) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 0) : !!(expr))); \ - })) -# define HEDLEY_PREDICT_FALSE(expr, probability) \ - (__extension__ ({ \ - double hedley_probability_ = (probability); \ - ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 0) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 1) : !!(expr))); \ - })) -# define HEDLEY_LIKELY(expr) __builtin_expect(!!(expr), 1) -# define HEDLEY_UNLIKELY(expr) __builtin_expect(!!(expr), 0) -#else -# define HEDLEY_PREDICT(expr, expected, probability) (HEDLEY_STATIC_CAST(void, expected), (expr)) -# define HEDLEY_PREDICT_TRUE(expr, probability) (!!(expr)) -# define HEDLEY_PREDICT_FALSE(expr, probability) (!!(expr)) -# define HEDLEY_LIKELY(expr) (!!(expr)) -# define HEDLEY_UNLIKELY(expr) (!!(expr)) -#endif -#if !defined(HEDLEY_UNPREDICTABLE) -# define HEDLEY_UNPREDICTABLE(expr) HEDLEY_PREDICT(expr, 1, 0.5) -#endif - -#if defined(HEDLEY_MALLOC) -# undef HEDLEY_MALLOC -#endif -#if \ - HEDLEY_HAS_ATTRIBUTE(malloc) || \ - HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ - HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - HEDLEY_IBM_VERSION_CHECK(12,1,0) || \ - HEDLEY_TI_VERSION_CHECK(15,12,0) || \ - (HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ - (HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ - (HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ - (HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ - HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_MALLOC __attribute__((__malloc__)) -#elif HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) -# define HEDLEY_MALLOC _Pragma("returns_new_memory") -#elif \ - HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ - HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) -# define HEDLEY_MALLOC __declspec(restrict) -#else -# define HEDLEY_MALLOC -#endif - -#if defined(HEDLEY_PURE) -# undef HEDLEY_PURE -#endif -#if \ - HEDLEY_HAS_ATTRIBUTE(pure) || \ - HEDLEY_GCC_VERSION_CHECK(2,96,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ - HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ - HEDLEY_TI_VERSION_CHECK(15,12,0) || \ - (HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ - (HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ - (HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ - (HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ - HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ - HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_PURE __attribute__((__pure__)) -#elif HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) -# define HEDLEY_PURE _Pragma("does_not_write_global_data") -#elif defined(__cplusplus) && \ - ( \ - HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) \ - ) -# define HEDLEY_PURE _Pragma("FUNC_IS_PURE;") -#else -# define HEDLEY_PURE -#endif - -#if defined(HEDLEY_CONST) -# undef HEDLEY_CONST -#endif -#if \ - HEDLEY_HAS_ATTRIBUTE(const) || \ - HEDLEY_GCC_VERSION_CHECK(2,5,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ - HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ - HEDLEY_TI_VERSION_CHECK(15,12,0) || \ - (HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ - (HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ - (HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ - (HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ - HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ - HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_CONST __attribute__((__const__)) -#elif \ - HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) -# define HEDLEY_CONST _Pragma("no_side_effect") -#else -# define HEDLEY_CONST HEDLEY_PURE -#endif - -#if defined(HEDLEY_RESTRICT) -# undef HEDLEY_RESTRICT -#endif -#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && !defined(__cplusplus) -# define HEDLEY_RESTRICT restrict -#elif \ - HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ - HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ - HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ - HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ - HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ - HEDLEY_TI_CL2000_VERSION_CHECK(6,2,4) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ - (HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus)) || \ - HEDLEY_IAR_VERSION_CHECK(8,0,0) || \ - defined(__clang__) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_RESTRICT __restrict -#elif HEDLEY_SUNPRO_VERSION_CHECK(5,3,0) && !defined(__cplusplus) -# define HEDLEY_RESTRICT _Restrict -#else -# define HEDLEY_RESTRICT -#endif - -#if defined(HEDLEY_INLINE) -# undef HEDLEY_INLINE -#endif -#if \ - (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \ - (defined(__cplusplus) && (__cplusplus >= 199711L)) -# define HEDLEY_INLINE inline -#elif \ - defined(HEDLEY_GCC_VERSION) || \ - HEDLEY_ARM_VERSION_CHECK(6,2,0) -# define HEDLEY_INLINE __inline__ -#elif \ - HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \ - HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ - HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - HEDLEY_TI_ARMCL_VERSION_CHECK(5,1,0) || \ - HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \ - HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ - HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_INLINE __inline -#else -# define HEDLEY_INLINE -#endif - -#if defined(HEDLEY_ALWAYS_INLINE) -# undef HEDLEY_ALWAYS_INLINE -#endif -#if \ - HEDLEY_HAS_ATTRIBUTE(always_inline) || \ - HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ - HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ - HEDLEY_TI_VERSION_CHECK(15,12,0) || \ - (HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ - (HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ - (HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ - (HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ - HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ - HEDLEY_IAR_VERSION_CHECK(8,10,0) -# define HEDLEY_ALWAYS_INLINE __attribute__((__always_inline__)) HEDLEY_INLINE -#elif \ - HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \ - HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) -# define HEDLEY_ALWAYS_INLINE __forceinline -#elif defined(__cplusplus) && \ - ( \ - HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ - HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ - HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ - HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) \ - ) -# define HEDLEY_ALWAYS_INLINE _Pragma("FUNC_ALWAYS_INLINE;") -#elif HEDLEY_IAR_VERSION_CHECK(8,0,0) -# define HEDLEY_ALWAYS_INLINE _Pragma("inline=forced") -#else -# define HEDLEY_ALWAYS_INLINE HEDLEY_INLINE -#endif - -#if defined(HEDLEY_NEVER_INLINE) -# undef HEDLEY_NEVER_INLINE -#endif -#if \ - HEDLEY_HAS_ATTRIBUTE(noinline) || \ - HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ - HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ - HEDLEY_TI_VERSION_CHECK(15,12,0) || \ - (HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ - (HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ - (HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ - (HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ - HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ - HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ - HEDLEY_IAR_VERSION_CHECK(8,10,0) -# define HEDLEY_NEVER_INLINE __attribute__((__noinline__)) -#elif \ - HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ - HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) -# define HEDLEY_NEVER_INLINE __declspec(noinline) -#elif HEDLEY_PGI_VERSION_CHECK(10,2,0) -# define HEDLEY_NEVER_INLINE _Pragma("noinline") -#elif HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus) -# define HEDLEY_NEVER_INLINE _Pragma("FUNC_CANNOT_INLINE;") -#elif HEDLEY_IAR_VERSION_CHECK(8,0,0) -# define HEDLEY_NEVER_INLINE _Pragma("inline=never") -#elif HEDLEY_COMPCERT_VERSION_CHECK(3,2,0) -# define HEDLEY_NEVER_INLINE __attribute((noinline)) -#elif HEDLEY_PELLES_VERSION_CHECK(9,0,0) -# define HEDLEY_NEVER_INLINE __declspec(noinline) -#else -# define HEDLEY_NEVER_INLINE -#endif - -#if defined(HEDLEY_PRIVATE) -# undef HEDLEY_PRIVATE -#endif -#if defined(HEDLEY_PUBLIC) -# undef HEDLEY_PUBLIC -#endif -#if defined(HEDLEY_IMPORT) -# undef HEDLEY_IMPORT -#endif -#if defined(_WIN32) || defined(__CYGWIN__) -# define HEDLEY_PRIVATE -# define HEDLEY_PUBLIC __declspec(dllexport) -# define HEDLEY_IMPORT __declspec(dllimport) -#else -# if \ - HEDLEY_HAS_ATTRIBUTE(visibility) || \ - HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ - HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ - ( \ - defined(__TI_EABI__) && \ - ( \ - (HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) \ - ) \ - ) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_PRIVATE __attribute__((__visibility__("hidden"))) -# define HEDLEY_PUBLIC __attribute__((__visibility__("default"))) -# else -# define HEDLEY_PRIVATE -# define HEDLEY_PUBLIC -# endif -# define HEDLEY_IMPORT extern -#endif - -#if defined(HEDLEY_NO_THROW) -# undef HEDLEY_NO_THROW -#endif -#if \ - HEDLEY_HAS_ATTRIBUTE(nothrow) || \ - HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_NO_THROW __attribute__((__nothrow__)) -#elif \ - HEDLEY_MSVC_VERSION_CHECK(13,1,0) || \ - HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ - HEDLEY_ARM_VERSION_CHECK(4,1,0) -# define HEDLEY_NO_THROW __declspec(nothrow) -#else -# define HEDLEY_NO_THROW -#endif - -#if defined(HEDLEY_FALL_THROUGH) -# undef HEDLEY_FALL_THROUGH -#endif -#if \ - HEDLEY_HAS_ATTRIBUTE(fallthrough) || \ - HEDLEY_GCC_VERSION_CHECK(7,0,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_FALL_THROUGH __attribute__((__fallthrough__)) -#elif HEDLEY_HAS_CPP_ATTRIBUTE_NS(clang,fallthrough) -# define HEDLEY_FALL_THROUGH HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[clang::fallthrough]]) -#elif HEDLEY_HAS_CPP_ATTRIBUTE(fallthrough) -# define HEDLEY_FALL_THROUGH HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[fallthrough]]) -#elif defined(__fallthrough) /* SAL */ -# define HEDLEY_FALL_THROUGH __fallthrough -#else -# define HEDLEY_FALL_THROUGH -#endif - -#if defined(HEDLEY_RETURNS_NON_NULL) -# undef HEDLEY_RETURNS_NON_NULL -#endif -#if \ - HEDLEY_HAS_ATTRIBUTE(returns_nonnull) || \ - HEDLEY_GCC_VERSION_CHECK(4,9,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_RETURNS_NON_NULL __attribute__((__returns_nonnull__)) -#elif defined(_Ret_notnull_) /* SAL */ -# define HEDLEY_RETURNS_NON_NULL _Ret_notnull_ -#else -# define HEDLEY_RETURNS_NON_NULL -#endif - -#if defined(HEDLEY_ARRAY_PARAM) -# undef HEDLEY_ARRAY_PARAM -#endif -#if \ - defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \ - !defined(__STDC_NO_VLA__) && \ - !defined(__cplusplus) && \ - !defined(HEDLEY_PGI_VERSION) && \ - !defined(HEDLEY_TINYC_VERSION) -# define HEDLEY_ARRAY_PARAM(name) (name) -#else -# define HEDLEY_ARRAY_PARAM(name) -#endif - -#if defined(HEDLEY_IS_CONSTANT) -# undef HEDLEY_IS_CONSTANT -#endif -#if defined(HEDLEY_REQUIRE_CONSTEXPR) -# undef HEDLEY_REQUIRE_CONSTEXPR -#endif -/* HEDLEY_IS_CONSTEXPR_ is for - HEDLEY INTERNAL USE ONLY. API subject to change without notice. */ -#if defined(HEDLEY_IS_CONSTEXPR_) -# undef HEDLEY_IS_CONSTEXPR_ -#endif -#if \ - HEDLEY_HAS_BUILTIN(__builtin_constant_p) || \ - HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_TINYC_VERSION_CHECK(0,9,19) || \ - HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ - HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ - (HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) && !defined(__cplusplus)) || \ - HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ - HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) -# define HEDLEY_IS_CONSTANT(expr) __builtin_constant_p(expr) -#endif -#if !defined(__cplusplus) -# if \ - HEDLEY_HAS_BUILTIN(__builtin_types_compatible_p) || \ - HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ - HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ - HEDLEY_ARM_VERSION_CHECK(5,4,0) || \ - HEDLEY_TINYC_VERSION_CHECK(0,9,24) -# if defined(__INTPTR_TYPE__) -# define HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0)), int*) -# else -# include -# define HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((intptr_t) ((expr) * 0)) : (int*) 0)), int*) -# endif -# elif \ - ( \ - defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && \ - !defined(HEDLEY_SUNPRO_VERSION) && \ - !defined(HEDLEY_PGI_VERSION) && \ - !defined(HEDLEY_IAR_VERSION)) || \ - (HEDLEY_HAS_EXTENSION(c_generic_selections) && !defined(HEDLEY_IAR_VERSION)) || \ - HEDLEY_GCC_VERSION_CHECK(4,9,0) || \ - HEDLEY_INTEL_VERSION_CHECK(17,0,0) || \ - HEDLEY_IBM_VERSION_CHECK(12,1,0) || \ - HEDLEY_ARM_VERSION_CHECK(5,3,0) -# if defined(__INTPTR_TYPE__) -# define HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0), int*: 1, void*: 0) -# else -# include -# define HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((intptr_t) * 0) : (int*) 0), int*: 1, void*: 0) -# endif -# elif \ - defined(HEDLEY_GCC_VERSION) || \ - defined(HEDLEY_INTEL_VERSION) || \ - defined(HEDLEY_TINYC_VERSION) || \ - defined(HEDLEY_TI_ARMCL_VERSION) || \ - HEDLEY_TI_CL430_VERSION_CHECK(18,12,0) || \ - defined(HEDLEY_TI_CL2000_VERSION) || \ - defined(HEDLEY_TI_CL6X_VERSION) || \ - defined(HEDLEY_TI_CL7X_VERSION) || \ - defined(HEDLEY_TI_CLPRU_VERSION) || \ - defined(__clang__) -# define HEDLEY_IS_CONSTEXPR_(expr) ( \ - sizeof(void) != \ - sizeof(*( \ - 1 ? \ - ((void*) ((expr) * 0L) ) : \ - ((struct { char v[sizeof(void) * 2]; } *) 1) \ - ) \ - ) \ - ) -# endif -#endif -#if defined(HEDLEY_IS_CONSTEXPR_) -# if !defined(HEDLEY_IS_CONSTANT) -# define HEDLEY_IS_CONSTANT(expr) HEDLEY_IS_CONSTEXPR_(expr) -# endif -# define HEDLEY_REQUIRE_CONSTEXPR(expr) (HEDLEY_IS_CONSTEXPR_(expr) ? (expr) : (-1)) -#else -# if !defined(HEDLEY_IS_CONSTANT) -# define HEDLEY_IS_CONSTANT(expr) (0) -# endif -# define HEDLEY_REQUIRE_CONSTEXPR(expr) (expr) -#endif - -#if defined(HEDLEY_BEGIN_C_DECLS) -# undef HEDLEY_BEGIN_C_DECLS -#endif -#if defined(HEDLEY_END_C_DECLS) -# undef HEDLEY_END_C_DECLS -#endif -#if defined(HEDLEY_C_DECL) -# undef HEDLEY_C_DECL -#endif -#if defined(__cplusplus) -# define HEDLEY_BEGIN_C_DECLS extern "C" { -# define HEDLEY_END_C_DECLS } -# define HEDLEY_C_DECL extern "C" -#else -# define HEDLEY_BEGIN_C_DECLS -# define HEDLEY_END_C_DECLS -# define HEDLEY_C_DECL -#endif - -#if defined(HEDLEY_STATIC_ASSERT) -# undef HEDLEY_STATIC_ASSERT -#endif -#if \ - !defined(__cplusplus) && ( \ - (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) || \ - (HEDLEY_HAS_FEATURE(c_static_assert) && !defined(HEDLEY_INTEL_CL_VERSION)) || \ - HEDLEY_GCC_VERSION_CHECK(6,0,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - defined(_Static_assert) \ - ) -# define HEDLEY_STATIC_ASSERT(expr, message) _Static_assert(expr, message) -#elif \ - (defined(__cplusplus) && (__cplusplus >= 201103L)) || \ - HEDLEY_MSVC_VERSION_CHECK(16,0,0) || \ - HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) -# define HEDLEY_STATIC_ASSERT(expr, message) HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(static_assert(expr, message)) -#else -# define HEDLEY_STATIC_ASSERT(expr, message) -#endif - -#if defined(HEDLEY_NULL) -# undef HEDLEY_NULL -#endif -#if defined(__cplusplus) -# if __cplusplus >= 201103L -# define HEDLEY_NULL HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(nullptr) -# elif defined(NULL) -# define HEDLEY_NULL NULL -# else -# define HEDLEY_NULL HEDLEY_STATIC_CAST(void*, 0) -# endif -#elif defined(NULL) -# define HEDLEY_NULL NULL -#else -# define HEDLEY_NULL ((void*) 0) -#endif - -#if defined(HEDLEY_MESSAGE) -# undef HEDLEY_MESSAGE -#endif -#if HEDLEY_HAS_WARNING("-Wunknown-pragmas") -# define HEDLEY_MESSAGE(msg) \ - HEDLEY_DIAGNOSTIC_PUSH \ - HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ - HEDLEY_PRAGMA(message msg) \ - HEDLEY_DIAGNOSTIC_POP -#elif \ - HEDLEY_GCC_VERSION_CHECK(4,4,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) -# define HEDLEY_MESSAGE(msg) HEDLEY_PRAGMA(message msg) -#elif HEDLEY_CRAY_VERSION_CHECK(5,0,0) -# define HEDLEY_MESSAGE(msg) HEDLEY_PRAGMA(_CRI message msg) -#elif HEDLEY_IAR_VERSION_CHECK(8,0,0) -# define HEDLEY_MESSAGE(msg) HEDLEY_PRAGMA(message(msg)) -#elif HEDLEY_PELLES_VERSION_CHECK(2,0,0) -# define HEDLEY_MESSAGE(msg) HEDLEY_PRAGMA(message(msg)) -#else -# define HEDLEY_MESSAGE(msg) -#endif - -#if defined(HEDLEY_WARNING) -# undef HEDLEY_WARNING -#endif -#if HEDLEY_HAS_WARNING("-Wunknown-pragmas") -# define HEDLEY_WARNING(msg) \ - HEDLEY_DIAGNOSTIC_PUSH \ - HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ - HEDLEY_PRAGMA(clang warning msg) \ - HEDLEY_DIAGNOSTIC_POP -#elif \ - HEDLEY_GCC_VERSION_CHECK(4,8,0) || \ - HEDLEY_PGI_VERSION_CHECK(18,4,0) || \ - HEDLEY_INTEL_VERSION_CHECK(13,0,0) -# define HEDLEY_WARNING(msg) HEDLEY_PRAGMA(GCC warning msg) -#elif \ - HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \ - HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) -# define HEDLEY_WARNING(msg) HEDLEY_PRAGMA(message(msg)) -#else -# define HEDLEY_WARNING(msg) HEDLEY_MESSAGE(msg) -#endif - -#if defined(HEDLEY_REQUIRE) -# undef HEDLEY_REQUIRE -#endif -#if defined(HEDLEY_REQUIRE_MSG) -# undef HEDLEY_REQUIRE_MSG -#endif -#if HEDLEY_HAS_ATTRIBUTE(diagnose_if) -# if HEDLEY_HAS_WARNING("-Wgcc-compat") -# define HEDLEY_REQUIRE(expr) \ - HEDLEY_DIAGNOSTIC_PUSH \ - _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") \ - __attribute__((diagnose_if(!(expr), #expr, "error"))) \ - HEDLEY_DIAGNOSTIC_POP -# define HEDLEY_REQUIRE_MSG(expr,msg) \ - HEDLEY_DIAGNOSTIC_PUSH \ - _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") \ - __attribute__((diagnose_if(!(expr), msg, "error"))) \ - HEDLEY_DIAGNOSTIC_POP -# else -# define HEDLEY_REQUIRE(expr) __attribute__((diagnose_if(!(expr), #expr, "error"))) -# define HEDLEY_REQUIRE_MSG(expr,msg) __attribute__((diagnose_if(!(expr), msg, "error"))) -# endif -#else -# define HEDLEY_REQUIRE(expr) -# define HEDLEY_REQUIRE_MSG(expr,msg) -#endif - -#if defined(HEDLEY_FLAGS) -# undef HEDLEY_FLAGS -#endif -#if HEDLEY_HAS_ATTRIBUTE(flag_enum) && (!defined(__cplusplus) || HEDLEY_HAS_WARNING("-Wbitfield-enum-conversion")) -# define HEDLEY_FLAGS __attribute__((__flag_enum__)) -#else -# define HEDLEY_FLAGS -#endif - -#if defined(HEDLEY_FLAGS_CAST) -# undef HEDLEY_FLAGS_CAST -#endif -#if HEDLEY_INTEL_VERSION_CHECK(19,0,0) -# define HEDLEY_FLAGS_CAST(T, expr) (__extension__ ({ \ - HEDLEY_DIAGNOSTIC_PUSH \ - _Pragma("warning(disable:188)") \ - ((T) (expr)); \ - HEDLEY_DIAGNOSTIC_POP \ - })) -#else -# define HEDLEY_FLAGS_CAST(T, expr) HEDLEY_STATIC_CAST(T, expr) -#endif - -#if defined(HEDLEY_EMPTY_BASES) -# undef HEDLEY_EMPTY_BASES -#endif -#if \ - (HEDLEY_MSVC_VERSION_CHECK(19,0,23918) && !HEDLEY_MSVC_VERSION_CHECK(20,0,0)) || \ - HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) -# define HEDLEY_EMPTY_BASES __declspec(empty_bases) -#else -# define HEDLEY_EMPTY_BASES -#endif - -/* Remaining macros are deprecated. */ - -#if defined(HEDLEY_GCC_NOT_CLANG_VERSION_CHECK) -# undef HEDLEY_GCC_NOT_CLANG_VERSION_CHECK -#endif -#if defined(__clang__) -# define HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) (0) -#else -# define HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) HEDLEY_GCC_VERSION_CHECK(major,minor,patch) -#endif - -#if defined(HEDLEY_CLANG_HAS_ATTRIBUTE) -# undef HEDLEY_CLANG_HAS_ATTRIBUTE -#endif -#define HEDLEY_CLANG_HAS_ATTRIBUTE(attribute) HEDLEY_HAS_ATTRIBUTE(attribute) - -#if defined(HEDLEY_CLANG_HAS_CPP_ATTRIBUTE) -# undef HEDLEY_CLANG_HAS_CPP_ATTRIBUTE -#endif -#define HEDLEY_CLANG_HAS_CPP_ATTRIBUTE(attribute) HEDLEY_HAS_CPP_ATTRIBUTE(attribute) - -#if defined(HEDLEY_CLANG_HAS_BUILTIN) -# undef HEDLEY_CLANG_HAS_BUILTIN -#endif -#define HEDLEY_CLANG_HAS_BUILTIN(builtin) HEDLEY_HAS_BUILTIN(builtin) - -#if defined(HEDLEY_CLANG_HAS_FEATURE) -# undef HEDLEY_CLANG_HAS_FEATURE -#endif -#define HEDLEY_CLANG_HAS_FEATURE(feature) HEDLEY_HAS_FEATURE(feature) - -#if defined(HEDLEY_CLANG_HAS_EXTENSION) -# undef HEDLEY_CLANG_HAS_EXTENSION -#endif -#define HEDLEY_CLANG_HAS_EXTENSION(extension) HEDLEY_HAS_EXTENSION(extension) - -#if defined(HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE) -# undef HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE -#endif -#define HEDLEY_CLANG_HAS_DECLSPEC_ATTRIBUTE(attribute) HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) - -#if defined(HEDLEY_CLANG_HAS_WARNING) -# undef HEDLEY_CLANG_HAS_WARNING -#endif -#define HEDLEY_CLANG_HAS_WARNING(warning) HEDLEY_HAS_WARNING(warning) - -#endif /* !defined(HEDLEY_VERSION) || (HEDLEY_VERSION < X) */ diff --git a/rapidyenc/rapidyenc/src/platform.cc b/rapidyenc/rapidyenc/src/platform.cc deleted file mode 100644 index 1b5bddd..0000000 --- a/rapidyenc/rapidyenc/src/platform.cc +++ /dev/null @@ -1,213 +0,0 @@ -#include "common.h" -#ifdef PLATFORM_ARM -# ifdef __ANDROID__ -# include -# elif defined(_WIN32) -# define WIN32_LEAN_AND_MEAN -# define NOMINMAX -# include -# elif defined(__APPLE__) -# include -# include -# elif defined(__has_include) -# if __has_include() -# include -# if __has_include() -# include -# endif -# endif -# endif -bool RapidYenc::cpu_supports_neon() { -# if defined(AT_HWCAP) -# if defined(__FreeBSD__) || defined(__OpenBSD__) - unsigned long supported; - elf_aux_info(AT_HWCAP, &supported, sizeof(supported)); -# ifdef __aarch64__ - return supported & HWCAP_ASIMD; -# else - return supported & HWCAP_NEON; -# endif -# else -# ifdef __aarch64__ - return getauxval(AT_HWCAP) & HWCAP_ASIMD; -# else - return getauxval(AT_HWCAP) & HWCAP_NEON; -# endif -# endif -# elif defined(ANDROID_CPU_FAMILY_ARM) -# ifdef __aarch64__ - return android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_ASIMD; -# else - return android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_NEON; -# endif -# elif defined(_WIN32) - return IsProcessorFeaturePresent(PF_ARM_NEON_INSTRUCTIONS_AVAILABLE); -# elif defined(__APPLE__) - int supported = 0; - size_t len = sizeof(supported); - if(sysctlbyname("hw.optional.neon", &supported, &len, NULL, 0)) - return false; - return (bool)supported; -# endif -# ifdef __aarch64__ - return true; // assume NEON support on AArch64 -# else - return false; -# endif -} -#endif - - -#ifdef PLATFORM_X86 -#ifdef _MSC_VER -# define _cpuid1(ar) __cpuid(ar, 1) -# define _cpuid1x(ar) __cpuid(ar, 0x80000001) -# if _MSC_VER >= 1600 -# define _cpuidX __cpuidex -# include -# define _GET_XCR() _xgetbv(_XCR_XFEATURE_ENABLED_MASK) -# else -// not supported -# define _cpuidX(ar, eax, ecx) ar[0]=0, ar[1]=0, ar[2]=0, ar[3]=0 -# define _GET_XCR() 0 -# endif -#else -# include -# define _cpuid1(ar) __cpuid(1, ar[0], ar[1], ar[2], ar[3]) -# define _cpuid1x(ar) __cpuid(0x80000001, ar[0], ar[1], ar[2], ar[3]) -# define _cpuidX(ar, eax, ecx) __cpuid_count(eax, ecx, ar[0], ar[1], ar[2], ar[3]) -static inline int _GET_XCR() { - int xcr0; - __asm__ __volatile__("xgetbv" : "=a" (xcr0) : "c" (0) : "%edx"); - return xcr0; -} -#endif -// checks if CPU has 128-bit AVX units; currently not used as AVX2 is beneficial even on Zen1 -// static bool cpu_has_slow_avx(cpuid1flag0) { - // int family = ((cpuid1flag0>>8) & 0xf) + ((cpuid1flag0>>16) & 0xff0), - // model = ((cpuid1flag0>>4) & 0xf) + ((cpuid1flag0>>12) & 0xf0); - // return ( - // family == 0x6f // AMD Bulldozer family - // || family == 0x7f // AMD Jaguar/Puma family - // || (family == 0x8f && (model == 0 /*Summit Ridge ES*/ || model == 1 /*Zen*/ || model == 8 /*Zen+*/ || model == 0x11 /*Zen APU*/ || model == 0x18 /*Zen+ APU*/ || model == 0x50 /*Subor Z+*/)) // AMD Zen1 family - // || (family == 6 && model == 0xf) // Centaur/Zhaoxin; overlaps with Intel Core 2, but they don't support AVX - // ); -// } - - -int RapidYenc::cpu_supports_isa() { - int flags[4]; - _cpuid1(flags); - int ret = 0; - - if(flags[2] & 0x800000) - ret |= ISA_FEATURE_POPCNT; - int flags2[4]; - _cpuid1x(flags2); - if(flags2[2] & 0x20) // ABM - ret |= ISA_FEATURE_LZCNT | ISA_FEATURE_POPCNT; - - int family = ((flags[0]>>8) & 0xf) + ((flags[0]>>16) & 0xff0); - int model = ((flags[0]>>4) & 0xf) + ((flags[0]>>12) & 0xf0); - - if(family == 6 && ( - model == 0x1C || model == 0x26 || model == 0x27 || model == 0x35 || model == 0x36 || model == 0x37 || model == 0x4A || model == 0x4C || model == 0x4D || model == 0x5A || model == 0x5D - )) - // Intel Bonnell/Silvermont CPU with very slow PSHUFB and PBLENDVB - pretend SSSE3 doesn't exist - return ret | ISA_LEVEL_SSE2; - - if(family == 0x5f && (model == 0 || model == 1 || model == 2)) - // AMD Bobcat with slow SSSE3 instructions - pretend it doesn't exist - return ret | ISA_LEVEL_SSE2; - - if((flags[2] & 0x200) == 0x200) { // SSSE3 - if(family == 6 && (model == 0x5c || model == 0x5f || model == 0x7a || model == 0x9c)) - // Intel Goldmont/plus / Tremont with slow PBLENDVB - return ret | ISA_LEVEL_SSSE3; - - if(flags[2] & 0x80000) { // SSE4.1 - if((flags[2] & 0x1C800000) == 0x1C800000) { // POPCNT + OSXSAVE + XSAVE + AVX - int xcr = _GET_XCR() & 0xff; // ignore unused bits - if((xcr & 6) == 6) { // AVX enabled - int cpuInfo[4]; - _cpuidX(cpuInfo, 7, 0); - if((cpuInfo[1] & 0x128) == 0x128 && (ret & ISA_FEATURE_LZCNT)) { // BMI2 + AVX2 + BMI1 - if((xcr & 0xE0) == 0xE0) { // AVX512 XSTATE (also applies to AVX10) - // check AVX10 - int cpuInfo2[4]; - _cpuidX(cpuInfo2, 7, 1); - if(cpuInfo2[3] & 0x80000) { - _cpuidX(cpuInfo2, 0x24, 0); - if((cpuInfo2[1] & 0xff) >= 1 && ( // minimum AVX10.1 - cpuInfo2[1] & 0x20000 // AVX10/256 (AVX10/128 is now invalid) - )) { - if(cpuInfo2[1] & 0x40000) ret |= ISA_FEATURE_EVEX512; - return ret | ISA_LEVEL_VBMI2; - } - } - - if((cpuInfo[1] & 0xC0010000) == 0xC0010000) { // AVX512BW + AVX512VL + AVX512F - ret |= ISA_FEATURE_EVEX512; - if(cpuInfo[2] & 0x40) - return ret | ISA_LEVEL_VBMI2; - return ret | ISA_LEVEL_AVX3; - } - } - // AVX2 is beneficial even on Zen1 - return ret | ISA_LEVEL_AVX2; - } - return ret | ISA_LEVEL_AVX; - } - } - return ret | ISA_LEVEL_SSE41; - } - return ret | ISA_LEVEL_SSSE3; - } - return ret | ISA_LEVEL_SSE2; -} - -int RapidYenc::cpu_supports_crc_isa() { - int flags[4]; - _cpuid1(flags); - - if((flags[2] & 0x80202) == 0x80202) { // SSE4.1 + SSSE3 + CLMUL - if((flags[2] & 0x1C000000) == 0x1C000000) { // AVX + OSXSAVE + XSAVE - int xcr = _GET_XCR() & 0xff; // ignore unused bits - if((xcr & 6) == 6) { // AVX enabled - int cpuInfo[4]; - _cpuidX(cpuInfo, 7, 0); - if((cpuInfo[1] & 0x20) == 0x20 && (cpuInfo[2] & 0x400) == 0x400) { // AVX2 + VPCLMULQDQ - return 2; - } - } - } - return 1; - } - return 0; -} - -#endif // PLATFORM_X86 - -#ifdef __riscv -# if defined(__has_include) -# if __has_include() -# include -# if __has_include() -# include -# endif -# endif -# endif -bool RapidYenc::cpu_supports_rvv() { -# if defined(AT_HWCAP) - unsigned long ret; -# if defined(__FreeBSD__) || defined(__OpenBSD__) - elf_aux_info(AT_HWCAP, &ret, sizeof(ret)); -# else - ret = getauxval(AT_HWCAP); -# endif - return (ret & 0x20112D) == 0x20112D; // IMAFDCV; TODO: how to detect Z* features of 'G'? -# endif - return false; -} -#endif - diff --git a/rapidyenc/rapidyenc/src/stdint.h b/rapidyenc/rapidyenc/src/stdint.h deleted file mode 100644 index 94c8b3c..0000000 --- a/rapidyenc/rapidyenc/src/stdint.h +++ /dev/null @@ -1,251 +0,0 @@ -// ISO C9x compliant stdint.h for Microsoft Visual Studio -// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 -// -// Copyright (c) 2006-2008 Alexander Chemeris -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. The name of the author may be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -/////////////////////////////////////////////////////////////////////////////// - -#if !defined(_MSC_VER) || defined(_STDINT) || _MSC_VER >= 1900 - -#include - -#else - -#ifndef _MSC_STDINT_H_ // [ -#define _MSC_STDINT_H_ - -#if _MSC_VER > 1000 -#pragma once -#endif - -#include - -// For Visual Studio 6 in C++ mode and for many Visual Studio versions when -// compiling for ARM we should wrap include with 'extern "C++" {}' -// or compiler give many errors like this: -// error C2733: second C linkage of overloaded function 'wmemchr' not allowed -#ifdef __cplusplus -extern "C" { -#endif -# include -#ifdef __cplusplus -} -#endif - -// Define _W64 macros to mark types changing their size, like intptr_t. -#ifndef _W64 -# if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 -# define _W64 __w64 -# else -# define _W64 -# endif -#endif - - -// 7.18.1 Integer types - -// 7.18.1.1 Exact-width integer types - -// Visual Studio 6 and Embedded Visual C++ 4 doesn't -// realize that, e.g. char has the same size as __int8 -// so we give up on __intX for them. -#if (_MSC_VER < 1300) - typedef signed char int8_t; - typedef signed short int16_t; - typedef signed int int32_t; - typedef unsigned char uint8_t; - typedef unsigned short uint16_t; - typedef unsigned int uint32_t; -#else - typedef signed __int8 int8_t; - typedef signed __int16 int16_t; - typedef signed __int32 int32_t; - typedef unsigned __int8 uint8_t; - typedef unsigned __int16 uint16_t; - typedef unsigned __int32 uint32_t; -#endif -typedef signed __int64 int64_t; -typedef unsigned __int64 uint64_t; - - -// 7.18.1.2 Minimum-width integer types -typedef int8_t int_least8_t; -typedef int16_t int_least16_t; -typedef int32_t int_least32_t; -typedef int64_t int_least64_t; -typedef uint8_t uint_least8_t; -typedef uint16_t uint_least16_t; -typedef uint32_t uint_least32_t; -typedef uint64_t uint_least64_t; - -// 7.18.1.3 Fastest minimum-width integer types -typedef int8_t int_fast8_t; -typedef int16_t int_fast16_t; -typedef int32_t int_fast32_t; -typedef int64_t int_fast64_t; -typedef uint8_t uint_fast8_t; -typedef uint16_t uint_fast16_t; -typedef uint32_t uint_fast32_t; -typedef uint64_t uint_fast64_t; - -// 7.18.1.4 Integer types capable of holding object pointers -#ifdef _WIN64 // [ - typedef signed __int64 intptr_t; - typedef unsigned __int64 uintptr_t; -#else // _WIN64 ][ - typedef _W64 signed int intptr_t; - typedef _W64 unsigned int uintptr_t; -#endif // _WIN64 ] - -// 7.18.1.5 Greatest-width integer types -typedef int64_t intmax_t; -typedef uint64_t uintmax_t; - - -// 7.18.2 Limits of specified-width integer types - -#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 - -// 7.18.2.1 Limits of exact-width integer types -#define INT8_MIN ((int8_t)_I8_MIN) -#define INT8_MAX _I8_MAX -#define INT16_MIN ((int16_t)_I16_MIN) -#define INT16_MAX _I16_MAX -#define INT32_MIN ((int32_t)_I32_MIN) -#define INT32_MAX _I32_MAX -#define INT64_MIN ((int64_t)_I64_MIN) -#define INT64_MAX _I64_MAX -#define UINT8_MAX _UI8_MAX -#define UINT16_MAX _UI16_MAX -#define UINT32_MAX _UI32_MAX -#define UINT64_MAX _UI64_MAX - -// 7.18.2.2 Limits of minimum-width integer types -#define INT_LEAST8_MIN INT8_MIN -#define INT_LEAST8_MAX INT8_MAX -#define INT_LEAST16_MIN INT16_MIN -#define INT_LEAST16_MAX INT16_MAX -#define INT_LEAST32_MIN INT32_MIN -#define INT_LEAST32_MAX INT32_MAX -#define INT_LEAST64_MIN INT64_MIN -#define INT_LEAST64_MAX INT64_MAX -#define UINT_LEAST8_MAX UINT8_MAX -#define UINT_LEAST16_MAX UINT16_MAX -#define UINT_LEAST32_MAX UINT32_MAX -#define UINT_LEAST64_MAX UINT64_MAX - -// 7.18.2.3 Limits of fastest minimum-width integer types -#define INT_FAST8_MIN INT8_MIN -#define INT_FAST8_MAX INT8_MAX -#define INT_FAST16_MIN INT16_MIN -#define INT_FAST16_MAX INT16_MAX -#define INT_FAST32_MIN INT32_MIN -#define INT_FAST32_MAX INT32_MAX -#define INT_FAST64_MIN INT64_MIN -#define INT_FAST64_MAX INT64_MAX -#define UINT_FAST8_MAX UINT8_MAX -#define UINT_FAST16_MAX UINT16_MAX -#define UINT_FAST32_MAX UINT32_MAX -#define UINT_FAST64_MAX UINT64_MAX - -// 7.18.2.4 Limits of integer types capable of holding object pointers -#ifdef _WIN64 // [ -# define INTPTR_MIN INT64_MIN -# define INTPTR_MAX INT64_MAX -# define UINTPTR_MAX UINT64_MAX -#else // _WIN64 ][ -# define INTPTR_MIN INT32_MIN -# define INTPTR_MAX INT32_MAX -# define UINTPTR_MAX UINT32_MAX -#endif // _WIN64 ] - -// 7.18.2.5 Limits of greatest-width integer types -#define INTMAX_MIN INT64_MIN -#define INTMAX_MAX INT64_MAX -#define UINTMAX_MAX UINT64_MAX - -// 7.18.3 Limits of other integer types - -#ifdef _WIN64 // [ -# define PTRDIFF_MIN _I64_MIN -# define PTRDIFF_MAX _I64_MAX -#else // _WIN64 ][ -# define PTRDIFF_MIN _I32_MIN -# define PTRDIFF_MAX _I32_MAX -#endif // _WIN64 ] - -#define SIG_ATOMIC_MIN INT_MIN -#define SIG_ATOMIC_MAX INT_MAX - -#ifndef SIZE_MAX // [ -# ifdef _WIN64 // [ -# define SIZE_MAX _UI64_MAX -# else // _WIN64 ][ -# define SIZE_MAX _UI32_MAX -# endif // _WIN64 ] -#endif // SIZE_MAX ] - -// WCHAR_MIN and WCHAR_MAX are also defined in -#ifndef WCHAR_MIN // [ -# define WCHAR_MIN 0 -#endif // WCHAR_MIN ] -#ifndef WCHAR_MAX // [ -# define WCHAR_MAX _UI16_MAX -#endif // WCHAR_MAX ] - -#define WINT_MIN 0 -#define WINT_MAX _UI16_MAX - -#endif // __STDC_LIMIT_MACROS ] - - -// 7.18.4 Limits of other integer types - -#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 - -// 7.18.4.1 Macros for minimum-width integer constants - -#define INT8_C(val) val##i8 -#define INT16_C(val) val##i16 -#define INT32_C(val) val##i32 -#define INT64_C(val) val##i64 - -#define UINT8_C(val) val##ui8 -#define UINT16_C(val) val##ui16 -#define UINT32_C(val) val##ui32 -#define UINT64_C(val) val##ui64 - -// 7.18.4.2 Macros for greatest-width integer constants -#define INTMAX_C INT64_C -#define UINTMAX_C UINT64_C - -#endif // __STDC_CONSTANT_MACROS ] - - -#endif // _MSC_STDINT_H_ ] - -#endif // MSVC stuff diff --git a/rapidyenc/rapidyenc/tool/bench.cc b/rapidyenc/rapidyenc/tool/bench.cc deleted file mode 100644 index 0a1e0a3..0000000 --- a/rapidyenc/rapidyenc/tool/bench.cc +++ /dev/null @@ -1,130 +0,0 @@ -#include -#include -#include - -#include "../rapidyenc.h" - -static const char* kernel_to_str(int k) { - if(k == RYKERN_GENERIC) return "generic"; - if(k == RYKERN_SSE2) return "SSE2"; - if(k == RYKERN_SSSE3) return "SSSE3"; - if(k == RYKERN_AVX) return "AVX"; - if(k == RYKERN_AVX2) return "AVX2"; - if(k == RYKERN_VBMI2) return "VBMI2"; - if(k == RYKERN_NEON) return "NEON"; - if(k == RYKERN_PCLMUL) return "PCLMUL"; - if(k == RYKERN_VPCLMUL) return "VPCLMUL"; - if(k == RYKERN_ARMCRC) return "ARM-CRC"; - if(k == RYKERN_RVV) return "RVV"; - if(k == RYKERN_ARMPMULL) return "ARM-CRC + PMULL"; - if(k == RYKERN_ZBC) return "Zbkc"; - return "unknown"; -} -#define ARTICLE_SIZE 768000ULL -#define SINGLE_OP_NUM 100 -#define REPETITIONS 1000 - -int main(int, char**) { - std::vector data(ARTICLE_SIZE); - std::vector article(rapidyenc_encode_max_length(ARTICLE_SIZE, 128)); - size_t article_length; - - // fill with random data - for(auto& c : data) - c = rand() & 0xff; - - - // encode benchmark -#ifndef RAPIDYENC_DISABLE_ENCODE - rapidyenc_encode_init(); - { - auto kernel = rapidyenc_encode_kernel(); - auto start = std::chrono::high_resolution_clock::now(); - for(int i=0; i(stop - start).count(); - double speed = ARTICLE_SIZE * REPETITIONS; - speed = speed / us / 1.048576; - std::cerr << "Encode (" << kernel_to_str(kernel) << "): " << speed << " MB/s" << std::endl; - } -#else - { - // do a pseudo yEnc encode to get a valid-ish article - unsigned char* pOut = article.data(); - int col = 0; - for(unsigned i=0; i= 128) { - *pOut++ = '\r'; - *pOut++ = '\n'; - col = 0; - } - } - article_length = pOut - article.data(); - } -#endif - - // decode -#ifndef RAPIDYENC_DISABLE_DECODE - rapidyenc_decode_init(); - { - auto kernel = rapidyenc_decode_kernel(); - auto start = std::chrono::high_resolution_clock::now(); - for(int i=0; i(stop - start).count(); - double speed = article_length * REPETITIONS; - speed = speed / us / 1.048576; - std::cerr << "Decode (" << kernel_to_str(kernel) << "): " << speed << " MB/s" << std::endl; - } -#endif - - // CRC -#ifndef RAPIDYENC_DISABLE_CRC - rapidyenc_crc_init(); - { - auto kernel = rapidyenc_crc_kernel(); - auto start = std::chrono::high_resolution_clock::now(); - for(int i=0; i(stop - start).count(); - double speed = ARTICLE_SIZE * REPETITIONS; - speed = speed / us / 1.048576; - std::cerr << "CRC32 (" << kernel_to_str(kernel) << "): " << speed << " MB/s" << std::endl; - - // CRC 256pow - std::vector rnd_n(SINGLE_OP_NUM); - std::vector rnd_out(SINGLE_OP_NUM); - for(auto& c : rnd_n) - c = ((rand() & 0xffff) << 20) | (rand() & 0xfffff); // 36-bit random numbers - - start = std::chrono::high_resolution_clock::now(); - for(int i=0; i(stop - start).count(); - speed = SINGLE_OP_NUM * REPETITIONS; - speed = speed / us; - std::cerr << "CRC32 256^n: " << speed << " Mop/s" << std::endl; - } -#endif - - return 0; -} diff --git a/rapidyenc/rapidyenc/tool/cli.c b/rapidyenc/rapidyenc/tool/cli.c deleted file mode 100644 index 4857155..0000000 --- a/rapidyenc/rapidyenc/tool/cli.c +++ /dev/null @@ -1,157 +0,0 @@ -#include -#include -#include -#include - -#include "../rapidyenc.h" - -static int print_usage(const char *app) { - fprintf(stderr, "Sample rapidyenc application\n"); - fprintf(stderr, "Usage: %s {e|d}\n", app); - fprintf(stderr, " (e)ncodes or (d)ecodes stdin to stdout\n"); - return 1; -} - -#define BUFFER_SIZE 65536 -#define LINE_SIZE 128 - -int main(int argc, char **argv) { - if(argc < 2) - return print_usage(argv[0]); - if(argv[1][0] != 'e' && argv[1][0] != 'd') - return print_usage(argv[0]); - -#ifdef RAPIDYENC_DISABLE_ENCODE - if(argv[1][0] == 'e') { - fprintf(stderr, "encoder has been disabled in this build\n"); - return 1; - } -#endif -#ifdef RAPIDYENC_DISABLE_DECODE - if(argv[1][0] == 'd') { - fprintf(stderr, "decoder has been disabled in this build\n"); - return 1; - } -#endif - - FILE* infile = stdin; // fopen("", "rb"); - if(!infile) { - fprintf(stderr, "error opening input: %s\n", strerror(errno)); - return 1; - } - FILE* outfile = stdout; // fopen("", "rb"); - if(!outfile) { - fprintf(stderr, "error opening output: %s\n", strerror(errno)); - fclose(infile); - return 1; - } - - void* data = malloc(BUFFER_SIZE); - if(!data) { - fprintf(stderr, "error allocating input buffer\n"); - fclose(infile); - fclose(outfile); - return 1; - } - -#ifndef RAPIDYENC_DISABLE_CRC - rapidyenc_crc_init(); - uint32_t crc = 0; -#endif - int has_error = 0; - -#ifndef RAPIDYENC_DISABLE_ENCODE - if(argv[1][0] == 'e') { - void* output = malloc(rapidyenc_encode_max_length(BUFFER_SIZE, LINE_SIZE)); - if(!output) { - fprintf(stderr, "error allocating output buffer\n"); - fclose(infile); - fclose(outfile); - free(data); - return 1; - } - rapidyenc_encode_init(); - - int column = 0; - while(1) { - size_t read = fread(data, 1, BUFFER_SIZE, infile); - int eof = feof(infile); - if(read < BUFFER_SIZE && !eof) { - if(ferror(infile)) { - fprintf(stderr, "error reading input\n"); - } else { - fprintf(stderr, "error: got zero bytes when reading input\n"); - } - has_error = 1; - break; - } - size_t out_len = rapidyenc_encode_ex(LINE_SIZE, &column, data, output, read, eof); -#ifndef RAPIDYENC_DISABLE_CRC - crc = rapidyenc_crc(data, read, crc); -#endif - if(fwrite(output, 1, out_len, outfile) != out_len) { - fprintf(stderr, "error writing output\n"); - has_error = 1; - break; - } - if(eof) break; - } - free(output); - } -#endif -#ifndef RAPIDYENC_DISABLE_DECODE - if(argv[1][0] == 'd') { - rapidyenc_decode_init(); - - RapidYencDecoderState state = RYDEC_STATE_CRLF; - while(1) { - size_t read = fread(data, 1, BUFFER_SIZE, infile); - RapidYencDecoderEnd ended; - void* in_ptr = data; - void* out_ptr = data; - int eof = feof(infile); - if(read < BUFFER_SIZE && !eof) { - if(ferror(infile)) { - fprintf(stderr, "error reading input\n"); - } else { - fprintf(stderr, "error: got zero bytes when reading input\n"); - } - has_error = 1; - break; - } - ended = rapidyenc_decode_incremental((const void**)&in_ptr, &out_ptr, read, &state); - size_t out_len = (uintptr_t)out_ptr - (uintptr_t)data; -#ifndef RAPIDYENC_DISABLE_CRC - crc = rapidyenc_crc(data, out_len, crc); -#endif - if(fwrite(data, 1, out_len, outfile) != out_len) { - fprintf(stderr, "error writing output\n"); - has_error = 1; - break; - } - - if(ended != RYDEC_END_NONE || eof) { - if(ended == RYDEC_END_CONTROL) - fprintf(stderr, "yEnc control line found\n"); - else if(ended == RYDEC_END_ARTICLE) - fprintf(stderr, "End-of-article marker found\n"); - else - fprintf(stderr, "End of input reached\n"); - break; - } - } - } -#endif - - fclose(infile); - fclose(outfile); - free(data); - - if(!has_error) { -#ifndef RAPIDYENC_DISABLE_CRC - fprintf(stderr, "Computed CRC32: %08x\n", crc); -#endif - } - - return 0; -} From 671b79387f10faa92c01714b410d118ba14ad20a Mon Sep 17 00:00:00 2001 From: go-while Date: Sat, 17 Oct 2026 12:09:42 +0000 Subject: [PATCH 118/152] WIP: web server job queue + race band-aid locks (Oct 2025, unreviewed) Preserves uncommitted working-tree changes from 16-18 Oct 2025 that were never committed. Origin is uncertain: most likely an AI coding assistant session from that time (not Claude Code). Committed as-is, unreviewed, so they are recoverable before the fixes in review-the-codebase-for-jolly-petal.md. 16 Oct 2025: web server with NZB job queue - webserver.go, web/templates/*, WEB_JOB_QUEUE.md (new) - Flags.go: -web, -webaddr, -webssl, -webssl-crt, -webssl-key; re-enable -nzbdir - main.go: -web boot branch - go.mod/go.sum: gin and transitive deps, testify 1.11.1 18 Oct 2025: lock band-aids for race reports - ConnPool.go, Session.go, Utils.go Known issues are listed in the review plan (Phase 3.13, Phase 4, Phase 6). Builds and vets clean; not race-tested. Co-Authored-By: Claude Opus 5.5 --- ConnPool.go | 2 + Flags.go | 7 +- Session.go | 6 +- Utils.go | 3 +- WEB_JOB_QUEUE.md | 207 +++++++++ go.mod | 32 +- go.sum | 80 +++- main.go | 30 ++ web/templates/base.html | 59 +++ web/templates/index.html | 198 ++++++++ web/templates/jobs.html | 521 +++++++++++++++++++++ web/templates/stats.html | 220 +++++++++ web/templates/style.css | 44 ++ webserver.go | 978 +++++++++++++++++++++++++++++++++++++++ 14 files changed, 2381 insertions(+), 6 deletions(-) create mode 100644 WEB_JOB_QUEUE.md create mode 100644 web/templates/base.html create mode 100644 web/templates/index.html create mode 100644 web/templates/jobs.html create mode 100644 web/templates/stats.html create mode 100644 web/templates/style.css create mode 100644 webserver.go diff --git a/ConnPool.go b/ConnPool.go index 1c9e11c..fecbc8d 100644 --- a/ConnPool.go +++ b/ConnPool.go @@ -566,7 +566,9 @@ func (c *ConnPool) GetStats() (openconns int, idle int) { // It will run until the global stopChan is closed or the provider is killed. // It will also print the status of the mutexes for debugging purposes. func (c *ConnPool) WatchOpenConnsThread(workerWGconnReady *sync.WaitGroup) { + globalmux.Lock() start := time.Now() + globalmux.Unlock() workerWGconnReady.Wait() dlog(cfg.opt.Debug, "WatchOpenConnsThread started for '%s' waited=%v", c.provider.Name, time.Since(start)) defer dlog(cfg.opt.Debug, "WatchOpenConnsThread for '%s' defer returned", c.provider.Name) diff --git a/Flags.go b/Flags.go index dc25e99..5abd775 100644 --- a/Flags.go +++ b/Flags.go @@ -25,7 +25,7 @@ func ParseFlags() { flag.BoolVar(&version, "version", false, "prints app version") // essentials flag.StringVar(&nzbfile, "nzb", "nzbs/ubuntu-24.04-live-server-amd64.iso.nzb.gz", "/path/file.nzb(.gz)") - //flag.StringVar(&cfg.opt.NzbDir, "nzbdir", "", "/path/to/watch/dir/for/nzbfiles") + flag.StringVar(&cfg.opt.NzbDir, "nzbdir", "", "/path/to/watch/dir/for/nzbfiles") flag.StringVar(&cfg.opt.ProvFile, "provider", "provider.json", "/path/provider.json") flag.BoolVar(&cfg.opt.CheckOnly, "checkonly", false, "[true|false] check online status only: no downs/reups (default: false)") flag.BoolVar(&cfg.opt.CheckFirst, "checkfirst", false, "[true|false] if false: starts downs/reups asap as segments are checked (default: false)") @@ -91,6 +91,11 @@ func ParseFlags() { flag.StringVar(&cfg.opt.TLSPrivKey, "tlskey", "privkey.pem", "path to TLS private key file for proxy (default: privkey.pem in current dir)") flag.StringVar(&AddUserDataToProxy, "proxyadduser", "", "if set: adds this user to proxy ( \ne.g.: -proxyadduser='username1234|password4567|maxconns|expires|(no)post'\n ) \nPassword entered MUST be cleartext, hashing is done when writing to .passwd file!\n Expiration: you can use a 'number of days' with 'd' at the end (e.g. 7d, 30d, 365d) or 'h' for hours (1h, 12h, ...) or 'm' minutes (1m, 30m, ...), it will be converted to a valid unixtimestamp expiring in the future from now on! (default: empty = no user added)\nExample -proxyadduser \"HelloWorld|NotAsecurePassword|5|42d|nopost\" creates a user with 5 conns and 42 days to expiration") flag.StringVar(&cfg.opt.ProxyPasswdFile, "proxypasswdfile", ".proxypasswd", "if set: use this file for proxy (default: .proxypasswd in current dir)") + flag.BoolVar(&webServerEnabled, "web", false, "[true|false] enable HTTP web server for NZB upload and statistics (default: false)") + flag.StringVar(&webServerAddr, "webaddr", "127.0.0.1:11911", "web server address (e.g.: '127.0.0.1:1234' or '[::1]:1234' or public ':1234') (default: 127.0.0.1:11911)") + flag.BoolVar(&webServerSSL, "webssl", false, "[true|false] enable HTTPS/TLS for web server (default: false)") + flag.StringVar(&webServerCert, "webssl-crt", "fullchain.pem", "path to TLS certificate file for web server (default: fullchain.pem in current dir)") + flag.StringVar(&webServerKey, "webssl-key", "privkey.pem", "path to TLS private key file for web server (default: privkey.pem in current dir)") // cosmetics: segmentBar needs fixing: only when everything else works! //flag.BoolVar(&cfg.opt.Bar, "bar", false, "show progress bars") // FIXME TODO //flag.BoolVar(&cfg.opt.Colors, "colors", false, "adds colors to s") // FIXME TODO diff --git a/Session.go b/Session.go index 2a80637..d69841d 100644 --- a/Session.go +++ b/Session.go @@ -268,7 +268,9 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s item.segment.Id = "<" + segment.Id + ">" // ensure segment.Id is always in the format "" for consistency } item.hashedId = SHA256str(item.segment.Id) + s.mux.Lock() s.segmentList = append(s.segmentList, item) + s.mux.Unlock() } } @@ -333,6 +335,7 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s } } globalmux.Unlock() + if s.proxy { // infinite block here for proxy server. if a wg is supplied it will never be released _, ok := <-s.stopChan @@ -348,7 +351,7 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s var waitDivider sync.WaitGroup // waitDivider is used to block GoWorkDivider() until all workers are ready var waitDividerDone sync.WaitGroup // waitDividerDone is used to signal that GoWorkDivider() has quit var waitPool sync.WaitGroup // waitPool waits until all workers closed their connections and routines - + globalmux.Lock() // prepare wait groups waitDivider.Add(1) waitDividerDone.Add(1) @@ -362,6 +365,7 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s } } // run the go routines + globalmux.Unlock() s.GoBootWorkers(&waitDivider, &workerWGconnReady, &waitWorker, &waitPool, s.nzbFile.Bytes) workerWGconnReady.Wait() // waits for all connections to be established before starting the work diff --git a/Utils.go b/Utils.go index 4035634..02136c8 100644 --- a/Utils.go +++ b/Utils.go @@ -146,6 +146,8 @@ func (cfg *Config) loadProviderList(s *SESSION, workerWGconnEstablish *sync.Wait if file, err := os.ReadFile(cfg.opt.ProvFile); err != nil { return err } else { + s.mux.Lock() + defer s.mux.Unlock() if err := json.Unmarshal(file, &cfg.providers); err != nil { return err } @@ -203,7 +205,6 @@ func (cfg *Config) loadProviderList(s *SESSION, workerWGconnEstablish *sync.Wait id++ } } - time.Sleep(time.Second * 3) return nil } // end func loadProviderList diff --git a/WEB_JOB_QUEUE.md b/WEB_JOB_QUEUE.md new file mode 100644 index 0000000..370f53f --- /dev/null +++ b/WEB_JOB_QUEUE.md @@ -0,0 +1,207 @@ +# NZBreX Web Server Job Queue Implementation + +## Overview +The web server now includes a job queue system that allows users to upload NZB files, configure processing options, and manage job execution. + +## Features + +### Job Queue System +- **Queue Management**: Upload NZB files and add them to a processing queue +- **Job Configuration**: Set per-job options for processing +- **Job Control**: Start, stop, and delete jobs +- **Status Tracking**: Monitor job progress and status + +### Configurable Options (Per Job) +- `-checkonly`: Check article availability only (no download/upload) +- `-checkfirst`: Check all articles before starting downloads +- `-crc32`: Enable CRC32 verification for articles +- `-yenctest`: Select yenc decoding mode (1-4) +- `-yenccpu`: Set CPU threads for yenc processing +- `-cd`: Cache directory path + +## API Endpoints + +### Web Pages +- `GET /` - Upload page with file drop zone +- `GET /jobs` - Job queue management page +- `GET /stats` - Statistics dashboard + +### API Routes +- `POST /upload` - Upload NZB file and add to queue +- `GET /api/jobs` - List all jobs (queued + running) +- `POST /api/jobs/:id/start` - Start a queued job +- `DELETE /api/jobs/:id` - Delete a queued job +- `PATCH /api/jobs/:id/config` - Update job configuration +- `GET /api/stats` - Get statistics (JSON) + +## Data Structures + +### JobQueue +Manages all queued and running jobs with thread-safe operations. + +### QueuedJob +```go +type QueuedJob struct { + ID string // Unique job identifier + NZBPath string // Path to uploaded NZB file + NZBName string // Original filename + Config JobConfig // Job-specific configuration + Status string // "queued", "running", "completed", "failed" + AddedAt time.Time // When job was added + StartedAt time.Time // When job started processing + FinishedAt time.Time // When job finished + Error string // Error message if failed + Progress float64 // Progress percentage (0-100) +} +``` + +### JobConfig +```go +type JobConfig struct { + CheckOnly bool // Only check availability + CheckFirst bool // Check all before downloading + YencCRC bool // Enable CRC32 verification + YencTest int // Yenc mode (1-4) + YencCpu int // CPU threads (0=auto) + Cachedir string // Cache directory +} +``` + +## Workflow + +### 1. Upload NZB File +User uploads an NZB file via web interface: +- File is validated (.nzb or .nzb.gz) +- File is saved to temporary directory +- Job is created with default or specified configuration +- Job is added to queue with status="queued" +- Job ID is returned to user + +### 2. Configure Job (Optional) +User can modify job configuration before starting: +- Update any config options via API +- Configuration is validated +- Only queued jobs can be configured + +### 3. Start Job +User starts a queued job: +- Job moves from queue to running state +- Processor launches session with job-specific config +- Job runs in background goroutine +- Status updates to "running" + +### 4. Monitor Progress +User can view job status: +- Auto-refreshing job list (3 second intervals) +- Status badges (queued/running/completed/failed) +- Configuration summary displayed +- Error messages shown if failed + +### 5. Job Completion +When job finishes: +- Status updates to "completed" or "failed" +- Job is removed from running map +- Timestamp recorded +- Original config restored + +## Usage Examples + +### Start Web Server +```bash +# HTTP (default) +./NZBreX -web -webaddr "127.0.0.1:11911" -cd /cache -provider provider.json + +# HTTPS +./NZBreX -web -webssl \ + -webssl-crt "/etc/letsencrypt/fullchain.pem" \ + -webssl-key "/etc/letsencrypt/privkey.pem" \ + -webaddr ":443" -cd /cache -provider provider.json +``` + +### Upload File with Configuration (cURL) +```bash +# Upload with default config +curl -F "file=@myfile.nzb" http://127.0.0.1:11911/upload + +# Upload with custom config +curl -F "file=@myfile.nzb" \ + -F "checkonly=true" \ + -F "crc32=true" \ + -F "cachedir=/custom/cache" \ + http://127.0.0.1:11911/upload +``` + +### Manage Jobs (API) +```bash +# List all jobs +curl http://127.0.0.1:11911/api/jobs + +# Start job +curl -X POST http://127.0.0.1:11911/api/jobs/job-1/start + +# Update job config +curl -X PATCH http://127.0.0.1:11911/api/jobs/job-1/config \ + -H "Content-Type: application/json" \ + -d '{"CheckOnly":true,"YencCRC":true}' + +# Delete job +curl -X DELETE http://127.0.0.1:11911/api/jobs/job-1 +``` + +## Web Interface Features + +### Upload Page (/) +- Drag-and-drop file upload +- File validation (.nzb, .nzb.gz) +- Upload progress indicator +- Links to job queue and statistics + +### Job Queue Page (/jobs) +- Table view of all jobs +- Status badges with color coding +- Configuration summary per job +- Action buttons: + - ▶ Start (queued jobs only) + - ⚙ Config (queued jobs only) + - 🗑 Delete (queued jobs only) +- Auto-refresh every 3 seconds +- Modal dialog for configuration + +### Statistics Page (/stats) +- Real-time provider statistics +- Bandwidth visualization +- Job progress tracking +- Auto-refresh every 5 seconds + +## Thread Safety +- All job queue operations use mutex locks +- Safe concurrent access to job data +- Original config is restored after job completion +- Job-specific sessions prevent conflicts + +## Error Handling +- File upload validation +- Job not found errors +- Cannot delete running jobs +- Cannot configure non-queued jobs +- Configuration validation +- Session launch errors captured + +## Files Modified/Created + +### Modified +- `webserver.go` - Added job queue system and handlers +- `Flags.go` - Added SSL/TLS flags for web server +- `web/templates/index.html` - Added job queue link +- `web/templates/stats.html` - Added job queue link + +### Created +- `web/templates/jobs.html` - Job queue management page + +## Benefits +1. **Better Control**: Queue jobs and start them when ready +2. **Per-Job Configuration**: Different settings for different NZBs +3. **Monitoring**: Track job status and progress +4. **Safety**: Can't accidentally overload system with simultaneous jobs +5. **Flexibility**: Mix check-only and full processing jobs +6. **Reliability**: Failed jobs don't block the queue diff --git a/go.mod b/go.mod index bf76bc1..f53772a 100644 --- a/go.mod +++ b/go.mod @@ -4,13 +4,43 @@ go 1.25.3 require ( github.com/Tensai75/nzbparser v0.1.0 + github.com/gin-gonic/gin v1.11.0 github.com/go-while/go-cpu-mem-profiler v0.0.0-20240612221627-856954a5fc83 golang.org/x/crypto v0.43.0 ) require ( + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.14.1 // indirect + github.com/bytedance/sonic/loader v0.3.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/gabriel-vasile/mimetype v1.4.10 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.28.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/goccy/go-yaml v1.18.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/quic-go/qpack v0.5.1 // indirect + github.com/quic-go/quic-go v0.55.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.3.0 // indirect + go.uber.org/mock v0.6.0 // indirect + golang.org/x/arch v0.22.0 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/sys v0.37.0 // indirect + golang.org/x/tools v0.38.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) @@ -18,7 +48,7 @@ require ( github.com/Tensai75/subjectparser v0.1.1 // indirect github.com/go-while/go-loggedrwmutex v0.0.0-20250601032232-b0aa20cdcb2c github.com/gorilla/mux v1.8.1 // indirect - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 golang.org/x/net v0.46.0 // indirect golang.org/x/text v0.30.0 ) diff --git a/go.sum b/go.sum index aa98f0d..1352c5d 100644 --- a/go.sum +++ b/go.sum @@ -2,25 +2,101 @@ github.com/Tensai75/nzbparser v0.1.0 h1:6RppAuWFahqu/kKjWO5Br0xuEYcxGz+XBTxYc+qv github.com/Tensai75/nzbparser v0.1.0/go.mod h1:IUIIaeGaYp2dLAAF29BWYeKTfI4COvXaeQAzQiTOfMY= github.com/Tensai75/subjectparser v0.1.1 h1:SAlaEKUmaalt4QH+UFBzEP6iHqA994iouFqvFPSM9y0= github.com/Tensai75/subjectparser v0.1.1/go.mod h1:PNBFBnkOGbVDfX+56ZmC4GKSpqoRMCF1Y44xYd7NLGI= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.14.1 h1:FBMC0zVz5XUmE4z9wF4Jey0An5FueFvOsTKKKtwIl7w= +github.com/bytedance/sonic v1.14.1/go.mod h1:gi6uhQLMbTdeP0muCnrjHLeCUPyb70ujhnNlhOylAFc= +github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= +github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gabriel-vasile/mimetype v1.4.10 h1:zyueNbySn/z8mJZHLt6IPw0KoZsiQNszIpU+bX4+ZK0= +github.com/gabriel-vasile/mimetype v1.4.10/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk= +github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.28.0 h1:Q7ibns33JjyW48gHkuFT91qX48KG0ktULL6FgHdG688= +github.com/go-playground/validator/v10 v10.28.0/go.mod h1:GoI6I1SjPBh9p7ykNE/yj3fFYbyDOpwMn5KXd+m2hUU= github.com/go-while/go-cpu-mem-profiler v0.0.0-20240612221627-856954a5fc83 h1:vehfiL7LsK8bJQZdVuJxcMem4AP2HwPEQ8orUUnIA+E= github.com/go-while/go-cpu-mem-profiler v0.0.0-20240612221627-856954a5fc83/go.mod h1:31AY7BccUb7qvKi4LQCwLOYsvcza9TdQtl/zHEIOcRE= github.com/go-while/go-loggedrwmutex v0.0.0-20250601032232-b0aa20cdcb2c h1:sDQuzVfH/JKw6k06m85cWNKsB27QSH496GZeImozJic= github.com/go-while/go-loggedrwmutex v0.0.0-20250601032232-b0aa20cdcb2c/go.mod h1:PqQxwEostGAr7ARrGKLQATdk6t2CZBOSDNg8vRCmydw= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= +github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= +github.com/quic-go/quic-go v0.55.0 h1:zccPQIqYCXDt5NmcEabyYvOnomjs8Tlwl7tISjJh9Mk= +github.com/quic-go/quic-go v0.55.0/go.mod h1:DR51ilwU1uE164KuWXhinFcKWGlEjzys2l8zUl5Ss1U= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= +github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI= +golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index c9e042c..9b11ab3 100644 --- a/main.go +++ b/main.go @@ -111,6 +111,36 @@ func main() { }(wg) dlog(cfg.opt.Debug, "main: wg.Wait()") wg.Wait() + } else if webServerEnabled { + // boot web server + wg.Add(1) // waitSession + go func(wg *sync.WaitGroup) { + globalmux.Lock() + // set up a session to load providers for web server + // similar to proxy server setup + dlog(cfg.opt.Debug, "pre:thisProcessor.LaunchSession: web server") + s := &SESSION{ + proc: thisProcessor, + mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "WEBSESSION"}, + proxy: true, // Set proxy flag to initialize providers without processing NZB + } + webProcessor = thisProcessor + webWg = wg + globalmux.Unlock() + go func() { + if err := StartWebServer(webServerAddr, webProcessor, wg); err != nil { + dlog(always, "ERROR StartWebServer: err='%v'", err) + os.Exit(1) + } + }() + // Initialize session to load providers + if err := thisProcessor.LaunchSession(s, "", wg); err != nil { + dlog(always, "ERROR NewProcessor Boot Web Server: err='%v'", err) + os.Exit(1) + } + }(wg) + dlog(cfg.opt.Debug, "main: wg.Wait()") + wg.Wait() } else if cfg.opt.NzbDir == "" && nzbfile != "" { wg.Add(1) // waitSession go func(nzbfile string, wg *sync.WaitGroup) { diff --git a/web/templates/base.html b/web/templates/base.html new file mode 100644 index 0000000..3f1258c --- /dev/null +++ b/web/templates/base.html @@ -0,0 +1,59 @@ + + + + + + {{.Title}} - NZBreX + + + {{block "extra_head" .}}{{end}} + + + + + + +
+ {{block "content" .}}{{end}} +
+ + +
+ + +
+ NZBreX - NZB Refresh X | Usenet Article Restoration Tool +
+ + + {{block "extra_scripts" .}}{{end}} + + diff --git a/web/templates/index.html b/web/templates/index.html new file mode 100644 index 0000000..b192617 --- /dev/null +++ b/web/templates/index.html @@ -0,0 +1,198 @@ + + + + + + NZBreX - NZB Refresh X + + + + +
+
+ 📋 Jobs + 📊 Stats +
+ +
+
+

NZBreX

+

NZB Refresh X - Upload and process NZB files

+ +
+
📁
+
Drag & Drop NZB files here
+
or click to browse (supports multiple .nzb or .nzb.gz files)
+
+ + + +
+
+
+
+ + + + + + diff --git a/web/templates/jobs.html b/web/templates/jobs.html new file mode 100644 index 0000000..c6ad046 --- /dev/null +++ b/web/templates/jobs.html @@ -0,0 +1,521 @@ + + + + + + ⚡ NZBreX Job Queue + + + + +
+
+ ← Back to Upload + 📊 Statistics +
+ +
+
+

📋 Job Queue

+

Manage jobs

+ +
+ + +
+ +
+ +
+ +
+ Auto-refreshing every 3 seconds +
+
+
+
+ + +
+ + + + + + + + + + + diff --git a/web/templates/stats.html b/web/templates/stats.html new file mode 100644 index 0000000..907c0ee --- /dev/null +++ b/web/templates/stats.html @@ -0,0 +1,220 @@ + + + + + + ⚡ NZBreX Statistics + + + + +
+
+ ← Back to Upload + 📋 Jobs +
+ +
+
+

📊 Statistics

+

Real-time processing statistics and provider information

+ + +
+
+
+
+
Total Jobs
+
0
+
+
+
+
+
+
+
Active Providers
+
0
+
+
+
+
+
+
+
Total Upload
+
0MB
+
+
+
+
+
+
+
Total Download
+
0MB
+
+
+
+
+ + +
+

Active Jobs

+
+
No active jobs
+
+
+ + +
+

Provider Statistics

+
+
No provider data available
+
+
+ +
+ Auto-refreshing every 5 seconds +
+
+
+
+ + + + + diff --git a/web/templates/style.css b/web/templates/style.css new file mode 100644 index 0000000..e6c06a4 --- /dev/null +++ b/web/templates/style.css @@ -0,0 +1,44 @@ +/* Custom variables to override Bootstrap defaults */ +:root { + --bs-primary: #0d6efd; + --bs-border-radius: 0.5rem; +} + +/* Upload zone */ +.upload-zone { + border: 2px dashed #dee2e6; + border-radius: var(--bs-border-radius); + padding: 3rem 1rem; + text-align: center; + cursor: pointer; + transition: all 0.2s ease; + background-color: #f8f9fa; +} + +.upload-zone:hover, +.upload-zone.drag-over { + border-color: var(--bs-primary); + background-color: #e7f3ff; +} + +.upload-icon { + font-size: 3rem; + opacity: 0.6; + margin-bottom: 1rem; +} + +/* Spinner animation */ +@keyframes spin { + 0% { transform: rotate(0deg); } + 100% { transform: rotate(360deg); } +} + +.spinner { + border: 3px solid #f3f3f3; + border-top: 3px solid var(--bs-primary); + border-radius: 50%; + width: 20px; + height: 20px; + animation: spin 1s linear infinite; + display: inline-block; +} diff --git a/webserver.go b/webserver.go new file mode 100644 index 0000000..7ca8c0f --- /dev/null +++ b/webserver.go @@ -0,0 +1,978 @@ +package main + +import ( + "embed" + "fmt" + "html/template" + "log" + "net/http" + "os" + "path/filepath" + "sync" + "time" + + "github.com/gin-gonic/gin" +) + +//go:embed web/templates/* +var templatesFS embed.FS + +var ( + webServerEnabled bool + webServerAddr string + webServerSSL bool + webServerCert string + webServerKey string + webProcessor *PROCESSOR + webWg *sync.WaitGroup + jobQueue *JobQueue +) + +// JobQueue manages queued NZB processing jobs +type JobQueue struct { + mu sync.RWMutex + jobs []*QueuedJob + running map[string]*QueuedJob // jobID -> job + completed []*QueuedJob // completed/failed jobs + nextID int + processingMu sync.Mutex // Ensures only one job processes at a time (prevents global cfg conflicts) + defaultConfig JobConfig // Default configuration for new jobs +} + +// QueuedJob represents a job in the queue +type QueuedJob struct { + ID string + NZBPath string + NZBName string + Config JobConfig + Status string // "queued", "running", "completed", "failed" + AddedAt time.Time + StartedAt time.Time + FinishedAt time.Time + Error string + Progress float64 + TotalArticles uint64 `json:"total_articles"` + ArticlesProcessed uint64 `json:"articles_processed"` // done count + ArticlesDone uint64 `json:"articles_done"` // done count (same as ArticlesProcessed) + ArticlesCached uint64 `json:"articles_cached"` // cached count + ArticlesDead uint64 `json:"articles_dead"` // dead count + TotalSize int64 `json:"total_size"` // Total size in bytes + SessionID uint64 `json:"session_id"` // Link to SESSION +} + +// JobConfig holds configuration options for a job +type JobConfig struct { + CheckOnly bool + CheckFirst bool + YencCRC bool + YencTest int + YencCpu int + Cachedir string +} + +// SessionStats holds real-time statistics extracted from a SESSION +type SessionStats struct { + TotalArticles uint64 + ArticlesDone uint64 + ArticlesCached uint64 + ArticlesDead uint64 + ArticlesChecked uint64 +} + +// NewJobQueue creates a new job queue +func NewJobQueue() *JobQueue { + globalmux.Lock() + defer globalmux.Unlock() + + // Initialize with default config from current global config + defaultConfig := JobConfig{ + CheckOnly: false, + CheckFirst: false, + YencCRC: false, + YencTest: 4, + YencCpu: 0, + Cachedir: "", + } + + // If cfg is already initialized, use its values as defaults + if cfg != nil && cfg.opt != nil { + defaultConfig.CheckOnly = cfg.opt.CheckOnly + defaultConfig.CheckFirst = cfg.opt.CheckFirst + defaultConfig.YencCRC = cfg.opt.YencCRC + defaultConfig.YencTest = cfg.opt.YencTest + defaultConfig.YencCpu = cfg.opt.YencCpu + defaultConfig.Cachedir = cfg.opt.Cachedir + } + + return &JobQueue{ + jobs: make([]*QueuedJob, 0), + running: make(map[string]*QueuedJob), + completed: make([]*QueuedJob, 0), + nextID: 1, + defaultConfig: defaultConfig, + } +} + +// AddJob adds a new job to the queue +func (jq *JobQueue) AddJob(nzbPath, nzbName string, config JobConfig) string { + jq.mu.Lock() + defer jq.mu.Unlock() + + jobID := fmt.Sprintf("job-%d", jq.nextID) + jq.nextID++ + + job := &QueuedJob{ + ID: jobID, + NZBPath: nzbPath, + NZBName: nzbName, + Config: config, + Status: "queued", + AddedAt: time.Now(), + } + + // Try to parse NZB to get size and article count + nzbfile, err := loadNzbFile(nzbPath) + if err == nil && nzbfile != nil { + var totalArticles uint64 + + for _, file := range nzbfile.Files { + totalArticles += uint64(file.TotalSegments) + } + + job.TotalSize = nzbfile.Bytes + job.TotalArticles = totalArticles + } + + jq.jobs = append(jq.jobs, job) + return jobID +} + +// GetJob retrieves a job by ID +func (jq *JobQueue) GetJob(jobID string) *QueuedJob { + jq.mu.RLock() + defer jq.mu.RUnlock() + + // Check running jobs first + if job, ok := jq.running[jobID]; ok { + return job + } + + // Check queued jobs + for _, job := range jq.jobs { + if job.ID == jobID { + return job + } + } + + // Check completed jobs + for _, job := range jq.completed { + if job.ID == jobID { + return job + } + } + + return nil +} + +// ListJobs returns all jobs (queued, running, and completed) +func (jq *JobQueue) ListJobs() []*QueuedJob { + jq.mu.RLock() + defer jq.mu.RUnlock() + + allJobs := make([]*QueuedJob, 0, len(jq.jobs)+len(jq.running)+len(jq.completed)) + + // Add running jobs with real-time stats + for _, job := range jq.running { + // Extract session stats for running jobs + if webProcessor != nil { + webProcessor.mux.RLock() + for _, session := range webProcessor.sessMap { + if session != nil && session.nzbPath == job.NZBPath { + sessionStats := extractSessionStats(session) + if sessionStats != nil { + // Update job with real-time statistics + job.ArticlesDone = sessionStats.ArticlesDone + job.ArticlesProcessed = sessionStats.ArticlesDone + job.ArticlesCached = sessionStats.ArticlesCached + job.ArticlesDead = sessionStats.ArticlesDead + job.TotalArticles = sessionStats.TotalArticles + if sessionStats.TotalArticles > 0 { + job.Progress = float64(sessionStats.ArticlesDone) / float64(sessionStats.TotalArticles) + } + } + break + } + } + webProcessor.mux.RUnlock() + } + allJobs = append(allJobs, job) + } + + // Add queued jobs + allJobs = append(allJobs, jq.jobs...) + + // Add completed jobs + allJobs = append(allJobs, jq.completed...) + + return allJobs +} + +// GetDefaultConfig returns a copy of the default job configuration +func (jq *JobQueue) GetDefaultConfig() JobConfig { + jq.mu.RLock() + defer jq.mu.RUnlock() + return jq.defaultConfig +} + +// SetDefaultConfig updates the default job configuration +func (jq *JobQueue) SetDefaultConfig(config JobConfig) { + jq.mu.Lock() + defer jq.mu.Unlock() + jq.defaultConfig = config +} + +// StartAllQueued starts all queued jobs (they will process serially) +func (jq *JobQueue) StartAllQueued() (int, error) { + jq.mu.Lock() + + // Get all queued job IDs + queuedIDs := make([]string, 0, len(jq.jobs)) + for _, job := range jq.jobs { + if job.Status == "queued" { + queuedIDs = append(queuedIDs, job.ID) + } + } + jq.mu.Unlock() + + // Start each job + started := 0 + for _, jobID := range queuedIDs { + if err := jq.StartJob(jobID); err != nil { + dlog(always, "Warning: Failed to start job %s: %v", jobID, err) + } else { + started++ + } + } + + return started, nil +} + +// StartJob starts a queued job +func (jq *JobQueue) StartJob(jobID string) error { + jq.mu.Lock() + + // Find the job in queue + var jobIndex = -1 + var job *QueuedJob + for i, j := range jq.jobs { + if j.ID == jobID { + jobIndex = i + job = j + break + } + } + + if jobIndex == -1 { + jq.mu.Unlock() + return fmt.Errorf("job %s not found in queue", jobID) + } + + // Remove from queue + jq.jobs = append(jq.jobs[:jobIndex], jq.jobs[jobIndex+1:]...) + + // Mark as running + job.Status = "running" + job.StartedAt = time.Now() + jq.running[jobID] = job + jq.mu.Unlock() + + // Start processing in background + go func() { + err := jq.processJob(job) + + jq.mu.Lock() + if err != nil { + job.Status = "failed" + job.Error = err.Error() + dlog(always, "ERROR: Job %s failed: %v", jobID, err) + } else { + job.Status = "completed" + dlog(always, "Job %s completed successfully", jobID) + } + job.FinishedAt = time.Now() + delete(jq.running, jobID) + + // Move to completed list + jq.completed = append(jq.completed, job) + + // Keep only last 50 completed jobs to prevent memory leak + if len(jq.completed) > 50 { + jq.completed = jq.completed[len(jq.completed)-50:] + } + + jq.mu.Unlock() + }() + + return nil +} + +// processJob processes a single job +func (jq *JobQueue) processJob(job *QueuedJob) error { + if webProcessor == nil { + return fmt.Errorf("web processor not initialized") + } + + // CRITICAL: Lock to prevent concurrent job execution + // This is necessary because cfg.opt is a GLOBAL variable shared across all jobs. + // Without this lock, concurrent jobs would overwrite each other's configuration. + // TODO: Refactor to pass per-job config to LaunchSession instead of using global cfg + jq.processingMu.Lock() + defer jq.processingMu.Unlock() + + // Apply job-specific configuration to global cfg + /* disabled: data race! + oldConfig := cfg.opt + defer func() { + cfg.opt = oldConfig // Restore original config + }() + + // Override with job config + cfg.opt.CheckOnly = job.Config.CheckOnly + cfg.opt.CheckFirst = job.Config.CheckFirst + cfg.opt.YencCRC = job.Config.YencCRC + cfg.opt.YencTest = job.Config.YencTest + cfg.opt.YencCpu = job.Config.YencCpu + if job.Config.Cachedir != "" { + cfg.opt.Cachedir = job.Config.Cachedir + } + */ + wg := &sync.WaitGroup{} + wg.Add(1) + + dlog(cfg.opt.Verbose, "Starting job %s with NZB: %s (serialized execution due to global cfg)", job.ID, job.NZBPath) + + // Launch session with NZB file (pass nil for session, file path for NZB) + err := webProcessor.LaunchSession(nil, job.NZBPath, wg) + if err != nil { + return fmt.Errorf("failed to launch session: %w", err) + } + + wg.Wait() + return nil +} + +// DeleteJob removes a job from the queue (only if not running) +func (jq *JobQueue) DeleteJob(jobID string) error { + jq.mu.Lock() + defer jq.mu.Unlock() + + // Can't delete running jobs + if _, ok := jq.running[jobID]; ok { + return fmt.Errorf("cannot delete running job %s", jobID) + } + + // Find and remove from queued jobs + for i, job := range jq.jobs { + if job.ID == jobID { + jq.jobs = append(jq.jobs[:i], jq.jobs[i+1:]...) + return nil + } + } + + // Find and remove from completed jobs + for i, job := range jq.completed { + if job.ID == jobID { + jq.completed = append(jq.completed[:i], jq.completed[i+1:]...) + return nil + } + } + + return fmt.Errorf("job %s not found", jobID) +} + +// WebStats holds statistics for the web interface +type WebStats struct { + Uptime string `json:"uptime"` + TotalSegments uint64 `json:"total_segments"` + CheckedSeg uint64 `json:"checked_segments"` + DownloadedSeg uint64 `json:"downloaded_segments"` + UploadedSeg uint64 `json:"uploaded_segments"` + CachedSeg uint64 `json:"cached_segments"` + DeadSeg uint64 `json:"dead_segments"` + MemUsed int `json:"mem_used"` + MemMax int `json:"mem_max"` + + // Bandwidth tracking + TotalRXbytes uint64 `json:"total_download_bytes"` + TotalTXbytes uint64 `json:"total_upload_bytes"` + + // Queue stats + DLQueue uint64 `json:"dl_queue"` + UPQueue uint64 `json:"up_queue"` + TotalDLQueue uint64 `json:"total_dl_queue"` + TotalUPQueue uint64 `json:"total_up_queue"` + + // Routine counts + CheckRoutines uint64 `json:"check_routines"` + DownsRoutines uint64 `json:"downs_routines"` + ReupsRoutines uint64 `json:"reups_routines"` + + // Connection stats + OpenConns uint64 `json:"open_connections"` + TotalNewConns uint64 `json:"total_new_connections"` + TotalDisConns uint64 `json:"total_disconnections"` + + Providers []ProviderStats `json:"providers"` + ActiveJobs []JobInfo `json:"jobs"` + RecentLogs []string `json:"recent_logs"` + + // Computed fields for frontend + TotalJobs int `json:"total_jobs"` + ActiveProviders int `json:"active_providers"` +} + +// ProviderStats holds per-provider statistics +type ProviderStats struct { + Name string `json:"name"` + Group string `json:"group"` + DownloadSpeed float64 `json:"download_speed"` // bytes/sec + UploadSpeed float64 `json:"upload_speed"` // bytes/sec + TotalDownload uint64 `json:"download_bytes"` // total bytes + TotalUpload uint64 `json:"upload_bytes"` // total bytes + OpenConns int `json:"open_connections"` + IdleConns int `json:"idle_connections"` + Checks uint64 `json:"checks"` + Downloads uint64 `json:"downloads"` + Uploads uint64 `json:"uploads"` + + // Article stats + Checked uint64 `json:"checked"` + Available uint64 `json:"available"` + Missing uint64 `json:"missing"` + Downloaded uint64 `json:"downloaded"` + Uploaded uint64 `json:"uploaded"` + + // Total bandwidth + TotalRXbyte uint64 `json:"total_rx_bytes"` + TotalTXbyte uint64 `json:"total_tx_bytes"` +} + +// JobInfo holds information about active processing jobs +type JobInfo struct { + ID string `json:"id"` + NZBName string `json:"name"` + NZBFile string `json:"nzb_file"` + StartTime time.Time `json:"start_time"` + StartedAt time.Time `json:"started_at"` + Status string `json:"status"` + TotalSegments uint64 `json:"total_articles"` + Progress float64 `json:"progress"` + + // Additional stats for display + ArticlesProcessed uint64 `json:"articles_processed"` + UploadBytes uint64 `json:"upload_bytes"` + DownloadBytes uint64 `json:"download_bytes"` +} + +// StartWebServer initializes and starts the Gin web server +func StartWebServer(addr string, processor *PROCESSOR, wg *sync.WaitGroup) error { + webServerEnabled = true + webServerAddr = addr + webProcessor = processor + webWg = wg + + // Initialize job queue + jobQueue = NewJobQueue() + + // Set Gin to release mode for production + if !cfg.opt.Debug { + gin.SetMode(gin.ReleaseMode) + } + + router := gin.Default() + + // Parse embedded templates + tmpl, err := template.ParseFS(templatesFS, "web/templates/*.html") + if err != nil { + return fmt.Errorf("failed to parse templates: %w", err) + } + router.SetHTMLTemplate(tmpl) + + // Serve static CSS file from embedded filesystem + router.GET("/static/style.css", func(c *gin.Context) { + data, err := templatesFS.ReadFile("web/templates/style.css") + if err != nil { + c.String(http.StatusNotFound, "CSS file not found") + return + } + c.Data(http.StatusOK, "text/css", data) + }) + + // Routes + router.GET("/", handleIndex) + router.GET("/stats", handleStats) + router.POST("/upload", handleUpload) + router.GET("/api/stats", handleAPIStats) + + // Job queue routes + router.GET("/jobs", handleJobsList) + router.GET("/api/jobs", handleAPIJobsList) + router.POST("/api/jobs/:id/start", handleJobStart) + router.POST("/api/jobs/start-all", handleJobStartAll) + router.DELETE("/api/jobs/:id", handleJobDelete) + router.PATCH("/api/jobs/:id/config", handleJobUpdateConfig) + router.GET("/api/jobs/default-config", handleGetDefaultConfig) + router.PUT("/api/jobs/default-config", handleSetDefaultConfig) + + // Determine protocol for logging + protocol := "http" + if webServerSSL { + protocol = "https" + } + dlog(always, "Starting web server on %s://%s", protocol, addr) + + // Run server in a goroutine + go func() { + var err error + if webServerSSL { + // Start HTTPS server with TLS + err = router.RunTLS(addr, webServerCert, webServerKey) + } else { + // Start HTTP server + err = router.Run(addr) + } + if err != nil { + dlog(always, "ERROR: Web server failed to start: %v", err) + } + }() + + return nil +} + +// handleIndex serves the main page +func handleIndex(c *gin.Context) { + c.HTML(http.StatusOK, "index.html", gin.H{ + "title": "NZBreX - NZB Refresh", + "version": appVersion, + }) +} + +// handleStats serves the statistics page +func handleStats(c *gin.Context) { + stats := collectStats() + c.HTML(http.StatusOK, "stats.html", gin.H{ + "title": "NZBreX Statistics", + "stats": stats, + }) +} + +// handleUpload processes NZB file uploads and adds them to the job queue +func handleUpload(c *gin.Context) { + file, err := c.FormFile("file") + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "No file uploaded"}) + return + } + + // Validate file extension + ext := filepath.Ext(file.Filename) + if ext != ".nzb" && ext != ".gz" { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file type. Only .nzb and .nzb.gz files are allowed"}) + return + } + + // Create temporary directory for uploads if it doesn't exist + uploadDir := filepath.Join(os.TempDir(), "nzbrex-uploads") + if err := os.MkdirAll(uploadDir, 0755); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create upload directory"}) + return + } + + // Save uploaded file + dst := filepath.Join(uploadDir, file.Filename) + if err := c.SaveUploadedFile(file, dst); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to save file"}) + return + } + + dlog(always, "Web: Uploaded NZB file: %s (size: %d bytes)", file.Filename, file.Size) + + // Start with default config from queue, then override with form data if provided + config := jobQueue.GetDefaultConfig() + + // Override with form data if provided + if checkonly := c.PostForm("checkonly"); checkonly != "" { + config.CheckOnly = checkonly == "true" + } + if checkfirst := c.PostForm("checkfirst"); checkfirst != "" { + config.CheckFirst = checkfirst == "true" + } + if crc32 := c.PostForm("crc32"); crc32 != "" { + config.YencCRC = crc32 == "true" + } + if cachedir := c.PostForm("cachedir"); cachedir != "" { + config.Cachedir = cachedir + } + + // Parse optional int fields + if yencTest := c.PostForm("yenctest"); yencTest != "" { + if val, err := fmt.Sscanf(yencTest, "%d", &config.YencTest); err == nil && val == 1 { + if config.YencTest < 1 || config.YencTest > 4 { + config.YencTest = 4 + } + } + } + if yencCpu := c.PostForm("yenccpu"); yencCpu != "" { + fmt.Sscanf(yencCpu, "%d", &config.YencCpu) + } + + // Add job to queue + jobID := jobQueue.AddJob(dst, file.Filename, config) + + c.JSON(http.StatusOK, gin.H{ + "message": "File uploaded and added to queue", + "filename": file.Filename, + "size": file.Size, + "job_id": jobID, + "status": "queued", + }) +} + +// handleJobsList displays the jobs queue page +func handleJobsList(c *gin.Context) { + jobs := jobQueue.ListJobs() + c.HTML(http.StatusOK, "jobs.html", gin.H{ + "title": "Job Queue", + "jobs": jobs, + }) +} + +// handleAPIJobsList returns all jobs as JSON +func handleAPIJobsList(c *gin.Context) { + jobs := jobQueue.ListJobs() + c.JSON(http.StatusOK, gin.H{ + "jobs": jobs, + }) +} + +// handleJobStart starts a queued job +func handleJobStart(c *gin.Context) { + jobID := c.Param("id") + + if err := jobQueue.StartJob(jobID); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, gin.H{ + "message": "Job started successfully", + "job_id": jobID, + }) +} + +// handleJobDelete deletes a queued job +func handleJobDelete(c *gin.Context) { + jobID := c.Param("id") + + if err := jobQueue.DeleteJob(jobID); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, gin.H{ + "message": "Job deleted successfully", + "job_id": jobID, + }) +} + +// handleJobUpdateConfig updates a job's configuration +func handleJobUpdateConfig(c *gin.Context) { + jobID := c.Param("id") + + job := jobQueue.GetJob(jobID) + if job == nil { + c.JSON(http.StatusNotFound, gin.H{"error": "Job not found"}) + return + } + + if job.Status != "queued" { + c.JSON(http.StatusBadRequest, gin.H{"error": "Cannot update config of non-queued job"}) + return + } + + // Update configuration from JSON body + var newConfig JobConfig + if err := c.ShouldBindJSON(&newConfig); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid configuration"}) + return + } + + jobQueue.mu.Lock() + job.Config = newConfig + jobQueue.mu.Unlock() + + c.JSON(http.StatusOK, gin.H{ + "message": "Job configuration updated", + "job_id": jobID, + "config": newConfig, + }) +} + +// handleJobStartAll starts all queued jobs +func handleJobStartAll(c *gin.Context) { + started, err := jobQueue.StartAllQueued() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, gin.H{ + "message": fmt.Sprintf("Started %d job(s)", started), + "count": started, + }) +} + +// handleGetDefaultConfig returns the default job configuration +func handleGetDefaultConfig(c *gin.Context) { + config := jobQueue.GetDefaultConfig() + c.JSON(http.StatusOK, gin.H{ + "config": config, + }) +} + +// handleSetDefaultConfig updates the default job configuration +func handleSetDefaultConfig(c *gin.Context) { + var newConfig JobConfig + if err := c.ShouldBindJSON(&newConfig); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid configuration"}) + return + } + + jobQueue.SetDefaultConfig(newConfig) + + c.JSON(http.StatusOK, gin.H{ + "message": "Default configuration updated", + "config": newConfig, + }) +} + +// handleAPIStats returns statistics as JSON for AJAX requests +func handleAPIStats(c *gin.Context) { + stats := collectStats() + c.JSON(http.StatusOK, stats) +} + +// extractSessionStats extracts real-time statistics from a SESSION +// This function counts the various states of segments in the session +func extractSessionStats(s *SESSION) *SessionStats { + if s == nil || s.segmentList == nil { + return nil + } + s.mux.RLock() + defer s.mux.RUnlock() + + stats := &SessionStats{ + TotalArticles: uint64(len(s.segmentList)), + } + providersCnt := len(s.providerList) + + // Count articles by state + for _, item := range s.segmentList { + item.mux.RLock() + + // Count cached articles + if item.flagCache { + stats.ArticlesCached++ + } + + // Count checked articles + if item.checkedOn > 0 { + stats.ArticlesChecked++ + } + + // Count done articles (available on all providers or uploaded) + if len(item.missingOn) == 0 && len(item.availableOn) == providersCnt { + stats.ArticlesDone++ + } + + // Count dead articles (not available anywhere) + if len(item.availableOn) == 0 && len(item.missingOn) == providersCnt { + stats.ArticlesDead++ + } + + item.mux.RUnlock() + } + + return stats +} + +// collectStats gathers current statistics for display +func collectStats() WebStats { + stats := WebStats{ + Uptime: formatDuration(time.Since(booted)), + RecentLogs: []string{}, + Providers: []ProviderStats{}, + ActiveJobs: []JobInfo{}, + } + + // Get memory usage + if memlim != nil { + stats.MemUsed, stats.MemMax = memlim.Usage() + } + + // Get global counters if available + if GCounter != nil { + // Download/Upload bandwidth tracking + stats.TotalRXbytes = GCounter.GetValue("TOTAL_RXbytes") + stats.TotalTXbytes = GCounter.GetValue("TOTAL_TXbytes") + + // Queue counters + stats.DLQueue = GCounter.GetValue("dlQueueCnt") + stats.UPQueue = GCounter.GetValue("upQueueCnt") + stats.TotalDLQueue = GCounter.GetValue("TOTAL_dlQueueCnt") + stats.TotalUPQueue = GCounter.GetValue("TOTAL_upQueueCnt") + + // Routine counters + stats.CheckRoutines = GCounter.GetValue("GoCheckRoutines") + stats.DownsRoutines = GCounter.GetValue("GoDownsRoutines") + stats.ReupsRoutines = GCounter.GetValue("GoReupsRoutines") + + // Connection counters + stats.TotalNewConns = GCounter.GetValue("TOTAL_NewConns") + stats.TotalDisConns = GCounter.GetValue("TOTAL_DisConns") + stats.OpenConns = stats.TotalNewConns - stats.TotalDisConns + } + + // Collect provider statistics from running/completed jobs + globalmux.RLock() + defer globalmux.RUnlock() + + if webProcessor != nil { + webProcessor.mux.RLock() + // Iterate through all active sessions + for _, session := range webProcessor.sessMap { + if session != nil && session.active && session.providerList != nil { + for _, prov := range session.providerList { + openConns, idleConns := prov.ConnPool.GetStats() + + // Get bandwidth from provider counter + provRX := prov.ConnPool.Counter.GetValue("TOTAL_RXbytes") + provTX := prov.ConnPool.Counter.GetValue("TOTAL_TXbytes") + + // Get speed values stored by Speedmeter function + prov.mux.RLock() + downloadSpeed := prov.speed.downloadSpeed + uploadSpeed := prov.speed.uploadSpeed + prov.mux.RUnlock() + + provStats := ProviderStats{ + Name: prov.Name, + Group: prov.Group, + OpenConns: openConns, + IdleConns: idleConns, + Checks: prov.articles.checked, + Downloads: prov.articles.downloaded, + Uploads: prov.articles.refreshed, + Checked: prov.articles.checked, + Available: prov.articles.available, + Missing: prov.articles.missing, + Downloaded: prov.articles.downloaded, + Uploaded: prov.articles.refreshed, + TotalRXbyte: provRX, + TotalTXbyte: provTX, + TotalDownload: provRX, // For JSON compatibility + TotalUpload: provTX, // For JSON compatibility + DownloadSpeed: downloadSpeed, // bytes/sec (from Speedmeter) + UploadSpeed: uploadSpeed, // bytes/sec (from Speedmeter) + } + stats.Providers = append(stats.Providers, provStats) + } + } + } + webProcessor.mux.RUnlock() + } + + // Collect active job information + if jobQueue != nil { + jobQueue.mu.RLock() + for _, job := range jobQueue.running { + // Try to find the session for this job and get real-time stats + var sessionStats *SessionStats + if webProcessor != nil { + webProcessor.mux.RLock() + for _, session := range webProcessor.sessMap { + if session != nil && session.nzbPath == job.NZBPath { + sessionStats = extractSessionStats(session) + break + } + } + webProcessor.mux.RUnlock() + } + + jobInfo := JobInfo{ + ID: job.ID, + NZBName: job.NZBName, + Status: job.Status, + StartedAt: job.StartedAt, + StartTime: job.StartedAt, // For JSON compatibility + Progress: job.Progress, + } + + // Update job with session stats if available + if sessionStats != nil { + jobInfo.TotalSegments = sessionStats.TotalArticles + jobInfo.ArticlesProcessed = sessionStats.ArticlesDone + // Update the job object itself with real-time progress + job.ArticlesDone = sessionStats.ArticlesDone + job.ArticlesProcessed = sessionStats.ArticlesDone + job.ArticlesCached = sessionStats.ArticlesCached + job.ArticlesDead = sessionStats.ArticlesDead + job.TotalArticles = sessionStats.TotalArticles + if sessionStats.TotalArticles > 0 { + job.Progress = float64(sessionStats.ArticlesDone) / float64(sessionStats.TotalArticles) + } + } + + stats.ActiveJobs = append(stats.ActiveJobs, jobInfo) + } + jobQueue.mu.RUnlock() + } + + // Calculate computed fields + stats.TotalJobs = len(stats.ActiveJobs) + stats.ActiveProviders = 0 + for _, prov := range stats.Providers { + if prov.Checks > 0 || prov.Downloads > 0 || prov.Uploads > 0 || prov.OpenConns > 0 { + stats.ActiveProviders++ + } + } + + return stats +} + +// formatDuration formats a duration into a human-readable string +func formatDuration(d time.Duration) string { + days := int(d.Hours() / 24) + hours := int(d.Hours()) % 24 + minutes := int(d.Minutes()) % 60 + seconds := int(d.Seconds()) % 60 + + if days > 0 { + return fmt.Sprintf("%dd %dh %dm %ds", days, hours, minutes, seconds) + } else if hours > 0 { + return fmt.Sprintf("%dh %dm %ds", hours, minutes, seconds) + } else if minutes > 0 { + return fmt.Sprintf("%dm %ds", minutes, seconds) + } + return fmt.Sprintf("%ds", seconds) +} + +// WriteWebLog adds a log entry to the web interface (for display in UI) +func WriteWebLog(msg string) { + // This could be expanded to maintain a ring buffer of recent logs + // for display in the web interface + if webServerEnabled { + // Store log in a circular buffer or similar + log.Printf("[WEB] %s", msg) + } +} From bb7a616aa8e5a92c06a1760f12b67aef19d17859 Mon Sep 17 00:00:00 2001 From: go-while Date: Sat, 17 Oct 2026 12:13:56 +0000 Subject: [PATCH 119/152] Create review-the-codebase-for-jolly-petal.md --- review-the-codebase-for-jolly-petal.md | 438 +++++++++++++++++++++++++ 1 file changed, 438 insertions(+) create mode 100644 review-the-codebase-for-jolly-petal.md diff --git a/review-the-codebase-for-jolly-petal.md b/review-the-codebase-for-jolly-petal.md new file mode 100644 index 0000000..765fa78 --- /dev/null +++ b/review-the-codebase-for-jolly-petal.md @@ -0,0 +1,438 @@ +# NZBreX `testing`: deadlock and logic fixes, Go 1.27.2 bump, web server and proxy hardening + +## Status of this plan (revised 2026-10-17) + +The first draft was checked claim by claim against the working tree by four independent reviews. Almost every bug it lists is real and most line numbers are accurate to ±3. But several proposed fixes would have caused new hangs or panics, some cause-and-effect claims were wrong, and some bugs were missing. This revision fixes all of that. Items marked **(corrected)** changed in substance; **(new)** were missing; **(dropped)** turned out to be wrong. + +Done since the first draft: **Phase 0** (rapidyenc C library from upstream), commit `c136df9`. + +Decisions (answered 2026-10-17, see [Decisions](#decisions)): **D1** separate web passwd file; **D2** show per-job settings disabled; **D3** reject broken mode combinations at startup. + +## Context + +The `testing` branch carries three large in-flight changes: +- a new NNTP proxy (`Proxy.go`, `ProxyHelper.go`, `Server.go`); +- a new web server with an NZB job queue (`webserver.go`, `web/templates/*`, `WEB_JOB_QUEUE.md`); +- uncommitted race-detector band-aid locks in `ConnPool.go`, `Session.go`, `Utils.go`. + +The review found two deterministic deadlocks, one unauthenticated remote process kill (`CAPABILITIES` before login), several unbounded-resource paths, and a set of data races, including one that can raise the unrecoverable `fatal error: concurrent map read and map write`. `go vet ./...` and `gofmt -l` are clean and catch none of it. The repo has no `recover()` anywhere and no `_test.go` outside `rapidyenc/`. + +**Important for severity:** `main.go` runs the proxy, the web server and NZB processing as **mutually exclusive modes** (`if proxy {…} else if webServerEnabled {…} else if nzbfile`). The proxy has its own session and its own connection pools, so proxy bugs cannot stall NZB download workers. The exception is `NetConn.go`: `readDotLines` is shared, so proxy-motivated changes there must be gated on `proxy`. + +Baseline (verified 2026-10-17): the tree builds and vets clean and `go test ./rapidyenc/` passes under go1.27.2 (`GOROOT=/tank0/efs/GO/.bin/go1.27.2`; go1.27.1 also still installed). Nothing in the dependency set blocks the bump. + +--- + +## Phase 0: rapidyenc C library from upstream (DONE, `c136df9`) + +- `rapidyenc/rapidyenc/` was a vendored copy of animetosho/rapidyenc, byte-identical to upstream `47f67f5` (2024-08-06). It is now a git submodule (`rapidyenc-c`) pinned to upstream `d223ecc` (v1.1.1-18). That brings in 14 upstream commits, including a scalar encoder overflow fix (sabnzbd/sabctools#209) and an unaligned-load fix in the decoder (nzbgetcom). +- `rapidyenc.h` did not change, so the Go wrapper (our fork of mnightingale/rapidyenc at `2494683`, with local changes) is untouched. +- The stale `.gitmodules` entry for mnightingale/rapidyenc was removed. CI checks out submodules. `clone_rapidyenc.sh` initialises the submodule; before, it was a single commented-out line and did nothing. +- Verified on linux/amd64: `go test -race ./rapidyenc/`, `./NZBreX -testrapidyenc`, `go vet ./...`. +- Leftovers, not touched: `.git/modules/rapidyenc` and `submodule.rapidyenc` in `.git/config`, from the 2025-06-06 submodule attempt. Harmless; remove by hand if wanted. +- The windows and arm64 `.a` files were not rebuilt locally (no cross-compilers installed). CI builds them from source. + +To bump later: `git -C rapidyenc/rapidyenc fetch && git -C rapidyenc/rapidyenc checkout && git add rapidyenc/rapidyenc`, then rebuild with `rapidyenc/build_rapidyenc_linux-amd64.sh`. + +--- + +## Phase 1: Go 1.27.2 bump + +| File | Change | +|---|---| +| `go.mod:3` | `go 1.25.3` → `go 1.27.2` | +| `.github/workflows/build_and_publish_test.yml:133` | `go-version: '1.25.3'` → `'1.27.2'` (line moved +2 by the submodule checkout change in `c136df9`) | + +`codeql.yml` and the `goreleaser_*.off` workflows use `go-version-file: 'go.mod'`, so no edit is needed. Commit the dependency updates already in the worktree (`gin` and its transitive deps, `testify` 1.10.0 → 1.11.1) in the same change. + +--- + +## Phase 2: Deterministic deadlocks and remote crashes (fix first) + +### 2.1 `watchDirThread` deadlock + +`watchDirThread` takes `p.mux.RLock()` at `Session.go:642` and calls `refreshDir()` at `:644`. `refreshDir` takes `p.mux.Lock()` at `:601`, and `newSession`/`newssid` lock again at `:440`/`:456`. `LoggedSyncRWMutex` wraps a plain `sync.RWMutex`, which is not reentrant, so this deadlocks on the first tick. + +**Trigger (corrected):** `-nzbdir` with any `DirRefresh > 0` (`SetDirRefresh`, `Session.go:405-412`, clamps 1–14 up to 15) and one `.nzb` in the directory. + +**Fix:** read `p.refresh` into a local under a short RLock, release, then call `refreshDir()` with no lock held. + +**Note:** even after the fix, `refreshDir` only *registers* sessions. `processorThread` is a TODO stub, so directory NZBs are never processed. See D3. + +### 2.2 `CAPABILITIES` before login kills the process + +`ps` is built at `Server.go:27-31` without `CliTp`. `CliTp` is only assigned at `:235`, after `AUTHINFO PASS`. The pre-auth switch calls `ps.printCapabilities()` at `:129`, and its `PrintfLine` at `ProxyHelper.go:69` dereferences nil. With no `recover()` in the repo, `printf 'CAPABILITIES\r\n' | nc host 1119` kills the process. `Server.go:302-303` also dereferences `ps.CliTp` inside its own nil guard. + +**Fix:** assign `ps.CliTp = cliTp` at construction (the cleanup defer at `:54-58` already handles a non-nil `CliTp`), and fix `:302-303`. Add a `recover()` in `handleConnection` that logs and closes the connection as a safety net, not as the fix. It only covers that goroutine, and a panic after `GetConn` or while `proxyMutex` is held (`:178-247`) would still leak the connection or leave the mutex locked. + +### 2.3 Nil mutex panic on a proxied article with `-crc32` (corrected) + +The proxy allocates `segmentChanItem` with `mux == nil` at **two** sites: `Server.go:347-351` and `:359-363` (the article-number path). `NetConn.go:889-892` locks `item.mux` with no guard; `:903` has the guard. + +A third path also crashes: with `-crc32 -yencwrite` and a cache, `cache.WriteYenc` (`NetConn.go:726/816/881`) locks `item.mux` at `Cache.go:307`, and the yEnc writer goroutine then dereferences `item.s.D`, `item.file.Filename` and `*item.nzbhashname` (`Cache.go:288-290`), all nil for proxy items. That panic is in another goroutine, so no `recover()` in `handleConnection` can catch it. + +**Fix:** initialise `mux` at both proxy sites, make the `:889` guard match `:903`, and skip `cache.WriteYenc` when `proxy` is set. + +### 2.4 Memory slots never returned when nothing will upload (corrected) + +`MemLockWait` runs for every download item (`Workers.go:298`). `MemReturn` happens only on error paths (`Workers.go:312`, `:394`; `NetConn.go:144`; `Cache.go:410`) and after a re-upload (`Workers.go:400`). The comment at `Workers.go:326` saying memory was "already returned in GoDownsRoutine for cache reads" is false (`Routines.go:154-157`). + +This is by design: the slot is held from download until upload so that RAM between the two stays bounded. The bug is when no upload follows. After `memMax` downloads (default `2*totalMaxConns`, `Session.go:306-311`), every routine blocks at the bare `<-m.memchan` in `MemLimit.go:112`, which no stop signal can wake, and `waitWorker.Wait()` at `Session.go:382` hangs. + +**Trigger (corrected):** a *download* run with no upload providers (`postProviders == 0`, so `allowUp=false`, `Workers.go:711`). The draft's `-checkonly` trigger was wrong: it sets `allowDl=false` (`Workers.go:710`), so nothing is downloaded and nothing is locked. The same leak happens with uploads on for any downloaded item that `pushUP` never queues (e.g. "no provider found", `Workers.go:686`). + +**Fix (corrected):** do **not** add a plain `MemReturn` on every 220/920 success; that removes the RAM limit for upload runs. Instead, release the slot on download success only when `!allowUp`, and release it wherever the divider retires an item without queueing an upload. For the blocking receive, see 3.6: `MemLockWait` needs a stop parameter and a rollback. + +### 2.5 `fatal error: concurrent map read and map write` on `s.segmentChans*` (corrected) + +Written under `globalmux` in each provider's boot goroutine (`Workers.go:65-81`). Read with no lock from `GoWorker` at `Workers.go:191/247/338` and `Routines.go:31`, which can overlap another group's boot write. That is a real, unrecoverable runtime throw. + +**Not racy (dropped from the draft):** `Workers.go:568/670` and `Routines.go:537-538` run after `workerWGconnReady.Wait()`, which comes after every write; `:381` is only reached via divider-pushed items. `len(item.ignoreDlOn)` at `Workers.go:861` is a data race, but `len()` on a map never raises the fatal throw. + +**Fix:** build all three maps once before any `GoWorker` starts, and treat them as read-only. Move the `provider.Group = provider.Name` defaulting (`Workers.go:58-60`) ahead of that, or the keys won't match `s.providerGroups` (`Utils.go:191-193`, which holds the raw, possibly empty, groups). Keep exactly one feeder per group. + +### 2.6 `close of closed channel` on session teardown (corrected; do together with 3.1) + +- `StopRoutines` (`Routines.go:537-538`) closes `segmentChansDowns`/`Reups` via `closeSegmentChannel`, and `closeSession` (`Session.go:484-495`) closes them again. Only reached when `len(sessMap) <= SessThreshold`; `SessThreshold` is JSON-config only and defaults to 0, which takes the `removeSession` branch. +- `segmentChansCheck[group]` is closed by the feeder (`Workers.go:102`) and again in `closeSession` (`Session.go:485`). +- `close(s.stopChan)` (`Routines.go:541`) on a second launch: **latent**, because no current caller relaunches a session. The proxy and web sessions (`main.go:98`, `:122`) have a **nil** `stopChan`, so any `StopRoutines` on them would panic. + +**Fix:** one owner per channel, keyed by **group**, not provider. Guard with `sync.Once`. Create `stopChan` once in `LaunchSession` before any watcher starts. **Do not** recreate it inside `closeSession`: `WatchOpenConnsThread` re-reads the `c.s.stopChan` field every loop (`ConnPool.go:643`), so swapping it races, and a watcher that misses the close spins forever on its `default` branch. + +--- + +## Phase 3: Hangs and resource exhaustion in the engine + +**Before starting 3.2, 3.6 or 3.7, settle the stop-channel design** (per pool or per session, captured once into a local, never swapped). Those three items and 2.6 all depend on it. + +### 3.1 `closeSegmentChannel` leaves the channel open (corrected) + +`Routines.go:547-569`: if the channel holds a buffered item, it is received, logged as "LOST", and the channel is not closed. Impact is smaller than the draft said. `StopRoutines` runs after the divider ends, which needs `dlQ`/`upQ` at 0, so the channel is normally empty. Download/re-upload routines also have 15 s timeouts once `segcheckdone` is set (`Workers.go:270-281`, `361-372`). + +**Fix:** drain, then close **once per group**. `StopRoutines` loops over providers, not groups, so with several providers per group a plain "close unconditionally" panics on the second call, and `closeSession` closes again. This must land in the same change as 2.6. + +### 3.2 Check feeder busy-wait (corrected) + +`Workers.go:84-103` spins until the channel is empty, then closes. If every worker failed to get a connection at boot, it spins forever. A second hang cause: `GoWorkDivider` keeps looping while `checked != todo` (`Workers.go:1029`). Selecting on `s.stopChan` alone does not help, because nothing closes `stopChan` until the divider has ended. + +**Fix:** give the feeder and the divider a shared "give up" signal, for example closed when every check routine of a group has exited, plus a deadline. The feeder's close must follow the 2.6 single-owner rule. + +### 3.3 Shared-connection early returns (dropped as a bug) + +`SharedConnGet` only returns an item with `conn != nil`. On the error path it has already put `nil` back into `sharedCC` (`ConnPool.go:749`). The `conn != nil && srvtp == nil` window doesn't occur: `CloseConn` runs in the holder's own goroutine. The guards at `Routines.go:45-47`, `:170-172`, `:348-350` are defensive dead code. If they are touched, never push a token back on the error path; the cap-1 channel is already refilled. + +### 3.4 `CloseConn` is not nil-safe or idempotent (corrected) + +- **Nil:** `ConnPool.go:520` dereferences `connitem` without a check. `:362` is unreachable (`ParkConn` dereferences first), but `:782`/`:794` (`ReturnSharedConnToPool*`) can pass nil. Add a nil guard. +- **Double close:** `readDotLines` closes the connection itself (`NetConn.go:376`, `:486`, `:576`; `:337` is unreachable, `:476` doesn't close) and the caller closes again (`Routines.go:179`). `openConns--` (`ConnPool.go:540`) runs twice, so the count drifts negative. **The consequence is the reverse of the draft:** `KillConnPool` loops while `openConns > 0`, so negative drift makes it *exit early* and leave real connections open; `GetConn` can dial past `MaxConns`. +- **Fix (corrected):** guard **only the decrement** (`if connitem.conn == nil` → skip `openConns--`), and **always** send the `nil` token to `sharedCC`. In the double-close case the second call is the only one that sends it. An early `return` at the top would leave sibling routines blocked at `ConnPool.go:729`. +- The draft's "blocking send under `provider.mux`" is **dropped**: `checkCapabilities` passes `nil` for `sharedCC` (`NetConn.go:989`). A non-blocking `select` there is harmless. + +### 3.5 `KillConnPool` waits unboundedly while holding `PoolsLock` (corrected) + +`ConnPool.go:144-178` holds the global `PoolsLock` across an unbounded `Sleep(10ms)` loop until `openConns == 0`. It spins only on a *positive* leak (a connection that is never returned), not on the negative drift from 3.4. `LaunchSession` calls it per provider (`Session.go:386-388`). + +**Fix:** drain under a short lock, wait outside `PoolsLock` with a deadline. Force-close only items that are idle in the pool; closing a `connitem` another goroutine is using is a data race. + +### 3.6 Blocking receives with no stop case (corrected) + +`ConnPool.go:435` (`<-c.pool`, "infinite wait"), `ConnPool.go:729` (`<-sharedCC`), `MemLimit.go:112` (`<-m.memchan`). `:436` dereferences `connitem.connid` without a nil check (latent). + +**Not mechanical:** +- `memlim` is global across sessions and `MemLockWait` returns nothing. A stop case needs a signature change (return ok/err), and on abort it must undo `m.waiting` and `delete(m.memdata, item)`. Otherwise the next `MemLockWait` on that item hits `os.Exit(1)` (`MemLimit.go:88-90`). Callers must skip `GoDownsRoutine` after an abort. +- `StopRoutines` closes `stopChan` *before* `waitWorker.Wait()`, so running workers will start getting errors from `GetConn`. That's acceptable, but expect the log noise. +- Read `stopChan` once into a local (see the Phase 3 note). + +### 3.7 Goroutine leaks (corrected) + +| Site | Status | +|---|---| +| `Workers.go:199-207`, `:255-263`, `:346-354`: three ticker goroutines per worker, never stopped | confirmed | +| `ConnPool.go:568-660` `WatchOpenConnsThread`: no exit path at all | confirmed | +| `ConnPool.go:828` `Speedmeter`: never stopped (only runs when `!CheckOnly && PrintStats>0`) | confirmed | +| `NetConn.go:476`, `:487` read errors with `YencTest` 3/4: async decoder blocks on `<-decodeBodyChan`, or `pipeWriter` is never closed | confirmed; **`:578` does not leak** (`CloseWithError` at `:575` unblocks it) | +| `Cache.go:311-324` `WriteYenc` spins if `GoYencWriter` was never started | latent: CLI `-checkonly` never downloads, and the web per-job override that could reach it is commented out | + +**Fix:** stop channel per `ConnPool`, closed by `KillConnPool`; `time.Ticker` with `defer Stop()`; move the `NetConn` cleanup into a `defer` that uses a "done" flag, so the success path (`:669-684`) doesn't close `decodeBodyChan`/`pipeWriter` twice. + +### 3.8 `MemLimiter` check-then-act gap (corrected; fix added) + +`MemLimit.go:74-90`: `memdata[item]` is set before the blocking wait, `memlocked` only after it at `:113`. No concrete path with two concurrent lockers on one item was found. A double `MemReturn` hits `log.Fatalf` (`:148`) only if the channel is full; otherwise it silently adds an extra slot and drives `memlocked` to −1. `MemAvail()` (`:67`) tests `m.waiting < m.mem_max`, which reports "available" with an empty `memchan`; it is used at `Workers.go:523`. + +**Fix:** set `memdata` and `memlocked` together, under one lock, after the slot is acquired. Make a second `MemReturn` a logged no-op instead of a fatal error. Base `MemAvail` on `len(m.memchan)`. + +### 3.9 `s.preBoot`/`s.active` guarded by two mutexes (corrected; fix added) + +`Session.go:170-178`/`:520-524` use `s.mux`; `Workers.go:13-25` uses `globalmux`. `LaunchSession` and `GoBootWorkers` run in the same goroutine (`Session.go:369`), so they don't race each other. The real race is `webserver.go:854` reading `active` under `globalmux` while `closeSession` writes it under `s.mux`. The nil dereference at `Workers.go:15` and the early returns before the `defer` at `:27` are unreachable (`LaunchSession` rejects a double launch at `:171`). + +**Fix:** guard both fields only with `s.mux` (or make them `atomic.Bool`), and read them that way in `webserver.go`. Move `defer waitDivider.Done()` above the early returns anyway; it costs nothing. + +### 3.10 `WaitGroup.Add` after `Wait` has started (corrected) and a boot hang (new) + +`NewConnPool` (via `loadProviderList`, `Session.go:290`) spawns `Speedmeter` and `WatchOpenConnsThread`, which call `workerWGconnReady.Wait()` (`ConnPool.go:572`, `:802`) before the `Add(1)` loop at `Session.go:356-366`. `Wait()` can return early, which is harmless. **Dropped:** the "WaitGroup is reused" panic; it needs reuse after reaching 0, and this WaitGroup is local and never reused. At most `-race` reports it. + +**New:** `Workers.go:49-52` skips providers with `MaxConns <= 0` (`continue`) **without** `workerWGconnReady.Done()`, while `Session.go:358-360` calls `Add(1)` for every provider. `Workers.go:158` then waits forever. `loadProviderList` doesn't filter these providers out. + +**Fix:** `Add()` before any pool is built; call `Done()` (or skip the `Add`) for providers that are skipped. + +### 3.11 `ConnPools` map (corrected) + +Keyed by a per-session index (`Utils.go:200`, `ConnPool.go:134/181`), so sessions overwrite each other. Harmless: nothing ever reads the map. **Fix:** delete it. + +### 3.12 `PoolsLock` ↔ `provider.mux` lock-order inversion (latent) + +`NewConnPool` takes `provider.mux` (`ConnPool.go:88`) then `PoolsLock` (`:133`); `KillConnPool` takes them in the reverse order (`:144`, `:183`). Not live today, because each session copies its providers (`Utils.go:197-198`). Fix while the file is open: never hold `provider.mux` while taking `PoolsLock`. + +### 3.13 The uncommitted band-aid locks (corrected; must land with their replacement) + +- `ConnPool.go:569-571`: `globalmux` around `start := time.Now()`. Protects nothing. **Revert.** +- **(new)** `Session.go:354/368`: `globalmux` around `.Add()` on local WaitGroups. Protects nothing. **Revert.** +- `Utils.go:149-150`: `s.mux` spans all of `loadProviderList`, including `NewConnPool`. No deadlock path was found, but narrow it. The lock exists because `extractSessionStats` (`webserver.go:771-780`) reads `s.providerList`/`s.segmentList` under `s.mux.RLock`, so keep the publish under `s.mux`, and move `s.providerList = nil` (`Session.go:289`, unlocked) under it too. `s.mux` is per session and never protected the shared `cfg.providers` unmarshal against concurrent web jobs; that needs a global lock. +- `Session.go:271-273`: build `segmentList` locally, assign it once under `s.mux`, and use the local slice at `:278-283` (`len()` there is read without the lock). +- `Utils.go:208`: removing `time.Sleep(3s)` was right; nothing depends on it. + +**Ordering:** a plain revert reintroduces the races with `extractSessionStats`. Revert and replace in one change. + +### 3.14 Remaining unsynchronised accesses (corrected) + +Still races: +- `Session.go:467`: `len(s.proc.sessMap)` read without `p.mux`. +- **`ConnPool.go:184` (worse than the draft said).** `provider.ConnPool = nil` runs in `KillConnPool` (`Session.go:387`) before `clearSession` (`:401`). Meanwhile `webserver.go:854-856` calls `prov.ConnPool.GetStats()` while `session.active` is still true. That is a reachable nil-pointer panic from a web poll. +- `NetConn.go:987-989`: `checkCapabilities` holds `provider.mux` across two network reads with no deadline (`UseNoDeadline` makes `ExtendConn` a no-op). Add a deadline; don't hold the lock across I/O. + +**Not races (dropped):** +- `Workers.go:129`, `Routines.go:355-368`, `Workers.go:616-618`: written in the boot goroutine before the workers start, so the writes happen first. +- `writeCsvFile` (`Utils.go:444-451`): runs after the workers are joined. +- `Session.go:295/358/386`: run on the writer's own goroutine. + +--- + +## Phase 4: Web server correctness + +### 4.1 `StartWebServer` can't report a bind failure + +`webserver.go:535-549` runs `router.Run`/`RunTLS` in a goroutine, logs the error at `:545`, and returns `nil`. The only error it can return is a template parse failure (`:497`), which embedded templates make near-impossible. So the `os.Exit(1)` at `main.go:131-134` never fires for a busy port, a bad address or a missing TLS cert (defaults to `fullchain.pem` relative to CWD, `Flags.go:97-98`). The process then sits in `wg.Wait()` with no web server. + +**Fix:** `net.Listen` synchronously and return its error; `tls.LoadX509KeyPair` up front when `-webssl`; then `go srv.Serve(ln)`. + +### 4.2 Shared job state written under a read lock + +`ListJobs` (`webserver.go:178-218`, writes at `:194-200`) and `collectStats` (writes at `:926-933` under `jobQueue.mu.RLock` from `:897`) both modify `*QueuedJob` under `RLock`. `ListJobs` returns live pointers that are marshalled after the lock is dropped (`:652-655`), while the completion goroutine writes `Status`/`Error`/`FinishedAt` (`:293-302`). + +**(new)** `GetJob` also returns a live pointer: `handleJobUpdateConfig` reads `job.Status` at `:698` without the lock, then writes the config (check-then-act). `extractSessionStats` checks `s.segmentList == nil` at `:768` before taking `s.mux`, which races with `Session.go:272`. + +**Fix:** readers must not modify jobs. The job updates its own progress (it currently advances only when someone polls, so it stays 0 with no browser open, even after completion). Return copies (`[]QueuedJob`) under the lock. + +### 4.3 Locks held across long work + +- `AddJob` (`:117-147`) holds `jq.mu` while it opens, gunzips and parses the whole NZB (`Utils.go:67-124`). It swallows the parse error (`err == nil &&`), so a corrupt file is accepted with HTTP 200. The parse is then redone by `LaunchSession` (`Session.go:211`). +- `ListJobs` holds `jq.mu` **and** `webProcessor.mux.RLock` across `extractSessionStats`, which walks every segment taking a per-segment lock (`:767-806`). `newSession`/`newssid` need `p.mux.Lock`, so starting a job waits behind the stats poller. +- `collectStats` holds `globalmux.RLock()` for its whole body (`:847-848`). `globalmux` writers (`Workers.go:13`, `Session.go:301/354`, the `ConnPool.go:830` Speedmeter tick) block behind it, and once a writer is waiting, new readers block too. + +**Fix:** parse outside the lock and reject parse errors with 400. Keep running counters on the session (atomics updated by the workers) and have the web layer read those instead of scanning. + +### 4.4 Login, CSRF, timeouts, uploads (corrected; needs D1) + +- **No authentication** on any route (`:512-525`). The default bind is `127.0.0.1:11911`, but `WEB_JOB_QUEUE.md:115-118` tells users to expose it on `:443`. Delete that example. +- **Login (corrected):** the draft's "reuse `loadPasswdFile`/`verifyPassword` from `ProxyHelper.go`" does not fit as written. Those functions write and read the **global** `passwdMap` behind `proxyMutex` (which also guards `ProxySessions`/`CountConns`). A separate web file would overwrite the proxy users, and reusing `.proxypasswd` would give every proxy user full web control (see D1). They also inherit the 5.2 bugs (map wiped in place, timing oracle). The file format is proxy-specific (username ≥ 10 chars, maxconns/expiry fields, and `verifyPassword` never checks expiry). And bcrypt plus a random sleep of up to 128 ms on every request is too slow for pages that poll every 3–5 s. + **Fix:** refactor into `parsePasswdFile(path) (map[string]*UserData, error)` plus a verify function that takes the map (fixing 5.2 at the same time); give the web server its own credential map; cache successful verifications briefly. +- **CSRF (corrected):** `POST /upload`, `POST /api/jobs/start-all` **and `POST /api/jobs/:id/start`** (empty body, guessable `job-N` IDs) are simple cross-site requests. Basic auth does **not** stop CSRF, because browsers send cached credentials cross-site. Require a custom header (forces a preflight; update the `fetch` calls in `index.html:118` and `jobs.html`) and check `Origin`/`Sec-Fetch-Site`. For DNS rebinding, add a `Host` allowlist. +- **Timeouts:** `router.Run` uses a zero-value `http.Server`. Set `ReadHeaderTimeout: 10s`, `IdleTimeout: 60s`. Limit the body with `http.MaxBytesReader` rather than `WriteTimeout`, so slow uploads still work. +- **Upload size:** only gin's default `MaxMultipartMemory` (32 MiB in memory); the rest streams to an unbounded temp file. Add a size limit. +- **Upload directory (worse than the draft said):** `filepath.Join(os.TempDir(), "nzbrex-uploads")` with `0755` (`:584-596`). Gin's `SaveUploadedFile` also calls `os.Chmod(dir, 0750)` and `os.Create(dst)`, both of which follow a planted symlink, for the directory or the file. Use `os.MkdirTemp` once at startup (mode 0700) and write the upload yourself with `O_EXCL|O_NOFOLLOW`. +- **Filename collisions:** `dst := filepath.Join(uploadDir, file.Filename)` (`:592`). Two uploads of `foo.nzb` overwrite each other, including one that is running. Use the job ID as the on-disk name. +- **Cleanup:** there is no `os.Remove` anywhere in `webserver.go`. Delete uploads on job deletion/completion. +- **TLS:** set `MinVersion: tls.VersionTLS12` explicitly. +- `gin.Default()` trusts all proxies (`0.0.0.0/0`): call `router.SetTrustedProxies(nil)`. +- Bootstrap comes from `cdn.jsdelivr.net` without SRI in all four templates and breaks on an offline host. Embed it locally (the directory is already embedded). + +### 4.5 Job lifecycle (corrected) + +- **Panic recovery** on the job goroutine (`:290`): add it, but it only catches panics on that goroutine. Worker goroutines and the runtime's `fatal error: concurrent map` are not recoverable there; Phase 2 is the real fix. +- **No stop/cancel endpoint**, despite `WEB_JOB_QUEUE.md:11`. If `LaunchSession` hangs, `processingMu` is never released and every later job sits at "running" forever. +- **`StartAllQueued`** marks every job "running" (`:284-291`) before it waits on `processingMu` (`:329`). Add a `waiting` state; set `running`/`StartedAt` in `processJob` after the lock. +- **Session ↔ job matching by path** (`:190`, `:904`) is ambiguous. Corrected mechanism: `SessThreshold` defaults to 0, so finished sessions are removed normally. Stale entries come from early-error returns in `LaunchSession` (`Session.go:213`, `:292`) that skip `clearSession`, and re-uploads with the same name share a path. `QueuedJob.SessionID` exists for this and is never assigned: assign it and match on it. +- **Unbounded queue:** `jq.jobs` has no cap. (Dropped: the `completed` re-slice keeping the backing array alive; `append` reallocates, so retention is bounded.) +- **Mode combinations:** see D3. `-web` is ignored when `-nzbdir` is set (`main.go:86` `select{}` runs before `:114`), and **(new)** also when `-proxytcp`/`-proxytls` is set (else-if order). +- `webProcessor`/`webWg` are assigned twice (`main.go:127-128`, `webserver.go:481-482`). Not a race (the `go` statement orders them), just redundant. Remove the second. +- `webserver.go:349-360`: `wg.Add(1)`/`wg.Wait()` around a synchronous `LaunchSession` that already defers `Done()`. A no-op; remove. +- Status codes: not-found returns 400 at `:663`/`:678`, conflict returns 400 at `:678`/`:699`. Use 404/409. +- No graceful shutdown: add `srv.Shutdown(ctx)` on signal. + +### 4.6 Templates (corrected) + +- `jobs.html:231-235`: the "commented out" Config button is live. `html/template` strips the `` lines as JS comments and keeps the `if (canConfig)` block, so the button renders and the modal PATCHes at `:401-402`. Use `/* … */`, or remove it under D2. +- `base.html` is parsed but never used, and its keys don't match what the handlers pass. (Dropped: "would error at execute time"; tested, it renders with empty values.) Delete it, or make the pages extend it. +- `handleJobsList`/`handleStats` pass `jobs`/`stats` into templates that contain no template actions at all, so every page load pays for `ListJobs()` and throws the result away. Pass nothing. +- `index.html:169-176` `showNotification` inserts `${message}` without `escapeHtml`. Latent (callers at `:80`, `:85` pass static text). Escape it anyway. +- Fine as-is: templates and CSS are embedded (`webserver.go:17-18`, `:495`, `:503`); no `template.HTML` or `text/template`. + +--- + +## Phase 5: NNTP proxy (corrected) + +The proxy runs alone in its own mode (see Context), so its bugs affect only proxy clients, never NZB download workers. The cross-impact is in `NetConn.go`, marked below. + +### 5.1 Hangs and resource exhaustion + +- **No client deadlines.** The only `Set*Deadline` calls are on upstream connections (`ConnPool.go:383`, `:473`). `ProxySession.IsExpired()` (`ProxyHelper.go:322`) has no callers. Before login, unknown commands get `480` and the loop continues (`Server.go:286-289`), so a client can idle forever without logging in, and `MaxConns` only applies after login (`Server.go:214`). Account expiry *is* checked on every command (`:302`); only idle reaping is missing. **Fix:** read deadline per command, idle reaper, a cap on connections that haven't logged in. +- **Unlimited command line.** `textproto.Reader.ReadLine()` calls `readLineSlice(-1)` (go1.27.2 `net/textproto/reader.go:44`); the `len(line) > 128` check at `Server.go:93` runs after the whole line is in memory. **Fix (corrected):** a bounded reader that *replaces* `cliTp.R`, with a limit of **512 bytes** including CRLF (RFC 3977 §3.1). Not 128: message-IDs may be 250 octets (§3.6), and today `ARTICLE ` gets `501` and a disconnect. +- **Slow client holds an upstream connection.** `handleXOverCommand` (`Proxy.go:517`) and `handleXHdrCommand` (`:647`) call `ParkConn` only after writing to the client. Move the park before the write; it's safe because `ReadDotLines` has already buffered the response. (The draft swapped the two references: `handleGroupCommand` parks at `:273/:278`, `handleListCommand` at `:353`.) +- **`handleNextOrLastCommand` returns nothing** (`Proxy.go:745-782`) for any code except 223/421, including 422 and I/O errors with code 0. `LAST` is also answered with "421 No previous". Today GROUP leaves the pointer at `High`, so a normal newsreader doesn't hit 422. **It will once the GROUP → first-article fix below lands, so fix this first.** **Fix:** forward the upstream `code msg` for any code > 0; on code 0 try the next provider. +- **`XOVER 1-2-3` becomes `XOVER 0-`** (`Proxy.go:406-426`, sent at `:478-479`, read with no cap at `:499`). Return `501` unless there are exactly two parts. **(new)** `LIST` drops `args[1:]` (`Proxy.go:325-329`), so `LIST ACTIVE ` fetches the full list. Cap `ReadDotLines` size. +- **Unbounded negative cache** (`ProxyHelper.go:428-439`; entries only cleared on lookup at `:406`). Numeric requests are keyed by bare article number (`Server.go:359-363`), so a miss in one group poisons the same number in every group for 60 s. Every `Set` also logs a line. **Fix:** size cap and janitor; key numeric entries by group + number. +- **Accept loop** (`Proxy.go:140-150`) compares an error string and spins at 100% CPU on `EMFILE`. Use `errors.Is(err, net.ErrClosed)` plus exponential backoff. Nothing ever closes the listener today, so the shutdown branch is unreachable; wire it to shutdown. + +### 5.2 Correctness + +- **Errors disconnect the client** (`Server.go:397-399` returns `502` plus an error; `:115-122` breaks the loop; same for syntax errors at `:332/:338/:403` and the pre-login `501` paths). RFC 3977 §3.2.1: unknown command → `500`, syntax error → `501`, and if the server does close, it must send `400` first. + **Fix (corrected):** `handleRequest` uses its error return to mean "disconnect": `QUIT` (`:395`), login failures (`:304`) and client write failures (`:482`). Introduce a separate fatal/non-fatal result **before** making errors non-fatal, or `QUIT` stops working. + **(dropped)** "There is no MODE case": `MODE READER` is handled before login (`Server.go:256-275`, answers `201`). After login, RFC 4643 §2.2 says the client must not send `MODE READER`. + **(new)** `ARTICLE`/`HEAD`/`BODY`/`STAT` with no argument (allowed by the RFC; it means the current article) gets `501` and a disconnect. +- **Dead connection parked into the pool.** `handleGroupCommand` reaches `Proxy.go:278` (`ParkConn`) even when `ReadCodeLine` failed (`code == 0`). Park only `if code > 0`, as `:702-709`/`:722-728` already do. Proxy pool only. +- **Silent article truncation (corrected root cause; `NetConn.go`, shared).** With `-crc32 -yenctest=4` (4 is the default), a malformed article sets `brokenYenc = true; content = nil` (`NetConn.go:680`). The `isBadCrc || brokenYenc` → `99932` check at `:889` sits *inside* `if cfg.opt.YencCRC && !brokenYenc` (`:687`), so it never fires, `readDotLines` returns `1, nil`, and the client gets an empty `220`. **Dependencies:** download workers also handle `99932` (`Routines.go:267`), so they would start counting these as bad CRC; decide whether that's wanted. The `99932` path does `item.mux.Lock()` (`:890`), so 2.3 must land first. +- **Proxy rewrites relayed articles (corrected; `NetConn.go`, shared).** `Path: not-for-mail` is added at `NetConn.go:500`. Header stripping (`:520-536`) is `-cleanhdr` (not `-cleanheaders`), **on by default** (`Flags.go:42`). The default list (`Config.go:260-270`) strips `X-*`, `Path:`, `Xref:`, `Injection-`, `User-Agent:`, `Organization:`. It does **not** strip `References:` (the draft's threading claim is dropped), but it does break signatures that cover those headers. **Fix:** skip both rewrites when `proxy` is set, as `Date:` handling already does. +- **RFC deviations:** + - GROUP sets the article pointer to `High` (`Proxy.go:289`); RFC 3977 §6.1.1 requires the first article. **Land after the NEXT/LAST fix.** + - `ARTICLE ` resets the selected group (`Server.go:352`), which §6.2.1.2 forbids. **The reset is load-bearing:** `Server.go:413`/`:431` treat `ps.Group != "" && ps.MsgNum > 0` as "request by number". Add a local `byNumber` flag first, or message-ID requests get misrouted. `:357` also moves `MsgNum` before the request has succeeded. + - Article number field: `0` is correct for message-ID requests (§6.2.1.2). Only numeric requests are wrong (the number is hardcoded and also lands in the message-ID slot, `ProxyHelper.go:40`, `Server.go:510`). Use the upstream response line for numeric requests. +- **`passwdMap` wiped in place.** `ProxyHelper.go:132-134` clears the live map before parsing. On `usersLoaded == 0` (`:204`) or `scanner.Err` (`:200`) nothing is restored. Reload runs only after a successful `verifyPassword` (`Server.go:167` before `:173-188`), so once the map is empty nobody can log in again until restart. **Fix:** parse into a new map and swap it in only on success. Policy note: emptying the file on purpose then no longer locks everyone out. +- **(dropped)** "The `.new.*` merge lets anyone add users": this merge is how `-proxyadduser` delivers new users (`Flags.go:176`), and anyone with write access to that directory can already replace `.proxypasswd`. **Keep it.** +- **`ProxySessions` keyed by username** (`Server.go:240`, deleted at `:44`): never read anywhere. Delete the map, or switch it to `map[uint64]` keyed by `ps.id`. +- **Plaintext password kept** in `ps.Password` (`Server.go:232`), never read. Delete the field. +- **Timing oracle for unknown users** (`ProxyHelper.go:219-221` returns before the sleep and bcrypt). Run bcrypt against a dummy hash of the same cost (`DefaultCost`) on the miss path. +- **`AUTHINFO PASS` split on whitespace** (`Server.go:100`, `:165`, plus `TrimSpace` at `:99`). (Corrected: RFC 4643 does not require taking the rest of the line; its grammar excludes spaces, and §2.3.2 makes whitespace a MAY.) The effect is the reverse of the draft: a password with spaces *fails*, and `correctpw ` is *accepted*. Optional: reject extra arguments. +- **Client writes unchecked** almost everywhere (the `LinesWriter` error at `Server.go:481` is checked). Check them, so handlers stop fetching upstream for clients that are gone. +- `ps.Close()` (`ProxyHelper.go:311-319`) logs the username after clearing it, and would skip the deferred `CountConns` decrement (`Server.go:35`). Currently unreachable. +- `log.Fatalf` in the listener (`Proxy.go:117/126/133`) at startup only. It doesn't take an NZB session with it (the modes are separate), but a TLS failure kills the TCP listener too. Return an error instead. +- Two `bufio.Writer`s on one connection (`Server.go:235-236`). Use `ps.CliTp.W` for article data. +- **(new)** `ProxyParent` is published (`main.go:103`) and the proxy listener started (`:106`) before `LaunchSession` fills `providerList` (`Session.go:289`, `Utils.go:203`). Handlers read `providerList` without a lock: a data race, and an early client sees no providers. Start the listener after providers are loaded. + +--- + +## Phase 6: Per-job settings (needs D2) + +### 6.1 The per-job settings do nothing + +The jobs page has a per-job settings dialog and a "default settings" dialog, with six fields: Check Only, Check First, CRC32 verification, yEnc test mode, yEnc CPU threads, cache directory. They're backed by `PATCH /api/jobs/:id/config` and `GET/PUT /api/jobs/default-config`. The code that would apply them is commented out (`webserver.go:332-348`, `/* disabled: data race! */`), so every job runs with the startup flags whatever you set. `WEB_JOB_QUEUE.md:105` and `:179` say otherwise. (The disabled code wouldn't have worked anyway: `cfg.opt` is a `*CFG`, so `oldConfig := cfg.opt` copies the pointer and the "restore" restores the same object.) + +**Size of a real implementation (corrected):** the draft's "convert ~200 `cfg.opt.X` call sites to `s.opt.X`" is the wrong measure, and alone it would not work: +- 280 `cfg.opt.` uses in the five engine files (501 repo-wide), but most are debug flags. The six per-job fields appear on ~35 lines, which also include `Results.go` and `Yenc.go:42`. +- `Cachedir` and `CheckOnly` are built into the **global** `cache` at startup (`Flags.go:213-220`); `YencCpu` resizes the global `core_chan` (`Session.go:322`). Per-session copies of `cfg.opt` don't change those. +- Running jobs at the same time (removing `processingMu`) also needs per-session `memlim`, counters (`GCounter`, e.g. `postProviders` at `Workers.go:711`) and pools (3.11). + +`JobConfig.Cachedir` (`webserver.go:613`) is a free-form filesystem path from the form. It must be restricted to an allowlist before it is ever applied. + +### 6.2 Process-wide state changed by jobs (corrected) + +- `Workers.go:40-42` sets `cfg.opt.ChanSize` from a job's segment count, but only when `-chansize <= 0` (default 1000). +- `memlim` is created by the long-lived web session at startup (`Session.go:302-312`), not by the first job. `Session.go:313-322` sets `YencCpu`/`YencAsyncCpu` and resizes `core_chan` process-wide. +- `Session.go:180-197` redirects the process-wide `log` output into the job's log file, but only with `-log`. (Dropped: "gin's access log included"; gin writes to stdout directly.) **(new)** With `-log`, the web session itself redirects to `logs/.log` (from `filepath.Base("")`) and never restores it. + +### 6.3 `WEB_JOB_QUEUE.md` corrections + +- Stop/cancel doesn't exist (`:11`). +- Config is never validated (`:83`); `YencTest` is clamped only on the `/upload` path. +- `Progress` is a 0..1 fraction, not 0–100 (`:54`). +- The route list (`:29-35`) omits `start-all` and both `default-config` endpoints. +- The `QueuedJob` struct (`:44-55`) is missing seven fields. +- `webserver.go` is new, not modified (`:193`); `base.html` and `style.css` are undocumented. +- Uploads are never cleaned up; `SessionID` is never assigned. +- `collectStats` filters on `session.active` (`webserver.go:854`), but the web session blocks at `Session.go:341` before `GoBootWorkers` sets `active = true`, so `/api/stats` shows no providers when no job is running. + +--- + +## Phase 7: Smaller cleanups (corrected) + +- `main.go:98-102` and `:122-126` build `&SESSION{...}` directly instead of via `newSession`, so `stopChan`, `counter`, `fileStat` and `WorkDividerChan` are nil. `<-s.stopChan` at `Session.go:341` blocks forever only because the channel is nil, and any future `close(s.stopChan)` on these sessions panics. Add a `newProxySession()` that sets the same fields. +- `ConnPool.go:498`: `if always || cfg.opt.BUG && ...` with `always` a `const true` (`Config.go:14`), so `c.mux.RLock()` is taken on every park. Performance only. +- **Queue counter ordering (corrected).** The draft's "Increase/Decrease mismatch → underflow" is wrong: every `Increase` (`Workers.go:572/674`) pairs with a successful push and each receive does one `Decrease` (`:305/:390`). The real bug: `Increase` runs *after* the non-blocking send. A fast worker (e.g. a cache hit) can `Decrease` first, which is clamped at 0 and logs an error. The late `Increase` then leaves `dlQueueCnt = 1` forever, so `GoWorkDivider` (`Workers.go:1029`) never ends. **Fix:** `Increase` before the send; undo it if the send fails. +- **(dropped)** `Cache.go:86-101` request/response correlation: concurrent requests are for the same item and get the same answer, and `pushDL` sends each item to only one group. + +--- + +## Phase 8 (new): Pure-Go yEnc decoder using Go SIMD (prototype) + +go1.27.x ships `simd` (portable, pure-Go fallback where hardware is missing) and `simd/archsimd` (amd64, arm64, wasm intrinsics, including carry-less multiply). Both require `GOEXPERIMENT=simd` and are outside the Go 1 compatibility promise. SIMD in Go wasn't available when the cgo build was chosen. + +What the C library is still needed for: only mode-4 decoding. CRC already runs through Go's `hash/crc32` (already hardware-accelerated). Modes 1–3 already use the pure-Go `yenc` package. + +Plan: +1. Write a pure-Go yEnc decoder behind a build tag (`//go:build goexperiment.simd`) with a scalar fallback, implementing the same interface as the rapidyenc wrapper (`AcquireDecoder`, `Read`, `Meta`). +2. Test it against `rapidyenc/yenc/*.yenc` and against rapidyenc output on generated articles (including escapes at line ends, `=y` lines inside the body, CRLF variations). +3. Benchmark against rapidyenc (MB/s, allocations) on amd64; on arm64 if hardware is available. +4. If close to rapidyenc: make it the default and drop cgo, the `.a`/`.dll` builds and the cross-compile toolchains. Otherwise keep the C build and revisit when `simd` leaves experimental status. + +Independent of Phases 1–7; can run in parallel. + +--- + +## Decisions + +All three answered by the user on 2026-10-17. + +**D1: Web login. DECIDED: separate web passwd file.** +- New flag (e.g. `-webpasswd`) pointing at its own file, same bcrypt format, loaded into its own map via the refactored `parsePasswdFile` (4.4). Proxy users get no web access. +- No web passwd file configured → the web server refuses to start unless it is bound to loopback. Even then, log a warning that it has no login. +- Rejected: reusing `.proxypasswd`, which would let every proxy user drive jobs on your provider accounts. + +**D2: Per-job settings. DECIDED: show them disabled.** +- Keep the per-job and default-settings dialogs and the ⚙ button visible, with every field disabled and a "not yet supported, jobs use the startup settings" note. +- The three endpoints (`PATCH /api/jobs/:id/config`, `GET/PUT /api/jobs/default-config`) return `501 Not Implemented` instead of silently storing values that are never used. `/upload` ignores the config form fields (and says so in its response) instead of accepting them. +- Delete the commented-out block in `processJob` (`webserver.go:332-348`); it couldn't have worked (6.1). +- Fix `WEB_JOB_QUEUE.md:105`/`:179` to say per-job settings are planned, not active. +- Fix `jobs.html:231-235` so the button state is deliberate (disabled), not an accident of the `` comment (4.6). +- The real implementation (6.1 refactor) stays a separate later change, after Phases 2–3. + +**D3: Startup mode combinations. DECIDED: reject with a clear error.** +- At startup, before any session is created, exit non-zero with a message for: + - `-nzbdir` with `-web`; + - `-nzbdir` with `-proxytcp`/`-proxytls`; + - `-proxytcp`/`-proxytls` with `-web`; + - `-nzbdir` alone (directory processing is unfinished: `processorThread` is a stub, see 2.1), until it is implemented. +- Exactly one of `-nzb`, `-web` or a proxy flag must be given. +- Running proxy and web together is not supported: it would need shared pools, or two sessions could exceed each provider account's connection limit. + +--- + +## Execution order (corrected) + +1. **Phase 1:** Go bump + dependency commit. Independent. +2. **Phase 2.1, 2.2, 2.3:** small, independent fixes. +3. **Phase 3.13 together with the replacement locking.** Revert the band-aids and add the local-build + locked-publish in one change. +4. **Settle the stop-channel design** (Phase 3 note), then **2.6 + 3.1 together** (single owner per group), then 2.4, 2.5, 3.6 (with the `MemLockWait` API change), 3.2, 3.4, 3.5, 3.7, then the rest of Phase 3, plus 3.10 (new boot hang). +5. **Phase 4** after D1/D3: 4.1, 4.2, 4.3 and the `ConnPool` nil panic from 3.14, then 4.4 (needs the passwd refactor from 5.2), then 4.5 and 4.6. +6. **Phase 5:** 5.1 first. In 5.2: the fatal/non-fatal split before "don't disconnect"; NEXT/LAST before GROUP → first article; the `byNumber` flag before removing the group reset; the `NetConn.go` changes gated on `proxy`, with 2.3 already in. +7. **Phase 6** per D2. +8. **Phase 7** cleanups. +9. **Phase 8** in parallel at any time. + +## Verification (corrected) + +**Never against live Usenet providers.** All runs that talk to a server use local go-while/nntp-server instances (`/tank0/efs/GO/src/github.com/go-while/nntp-server`), never `provider.json`. `race_linux.sh` hardcodes `-provider="provider.json"`, so don't run it as-is; use a copy that takes the provider file as an argument. + +### Test setup + +- **Storage:** `/tank0/tmp/nzbrex/nntp-server/` (created), one directory per instance: + - `a/` on `127.0.0.1:11119`: holds every test article. + - `b/` on `127.0.0.1:11120`: missing a known subset, so NZBreX must download from A and re-upload to B. + - each with `active/ ov/ storage/ history/ hashdb/ config.json users.json`. +- **nntp-server settings:** run with `-dropbinary=false` (the default rejects yEnc). `AcceptAllGroups: true`. A `Peers` entry for `127.0.0.1` without `ReadOnlyAccess`, so `IHAVE` is allowed. Reader accounts in `users.json`. Avoid port 1119, which is the NZBreX proxy default. +- **Ingest:** nntp-server accepts articles only via `IHAVE` (no `POST`). NZBreX re-uploads via `IHAVE` when the provider sets `PreferIHAVE`. +- **Test data:** the sample NZBs in `nzbs/` point at real Usenet articles and can't be used. Write a small seeding tool that generates yEnc multi-part articles (random payload, known CRC32), writes a matching `.nzb`, and feeds A (all) and B (subset) via `IHAVE`. Include deliberately broken articles (bad CRC, truncated yEnc) for 2.3/5.2. +- **`provider.test.json`:** two providers in different groups (needed for 2.5), A download-only and B with `PreferIHAVE: true`; a third provider with `MaxConns: 0` for 3.10. + +### Checks + +- `go build ./... && go vet ./...` after each phase; `go test -race ./rapidyenc/`. +- **Race detector run** (`go run -race -tags=other . -provider=provider.test.json -nzb=.nzb …`): 2.5, 3.9, 3.14 and 4.2 should report today and be silent afterwards. +- **2.1:** `-nzbdir` with a generated NZB: today the watcher hangs on the first tick. Afterwards the NZB appears in `sessMap`/`seenFiles` on every refresh. (It won't be processed; see D3.) +- **2.2:** `printf 'CAPABILITIES\r\n' | nc 127.0.0.1 ` with the proxy pointed at server A: today the process dies; afterwards a capability list comes back. +- **2.3:** proxy with `-crc32`, then `ARTICLE` a deliberately broken article from server A: today it panics; afterwards an error response. +- **2.4 (corrected):** a **download** run with no upload provider (A only, `NoUpload` on everything else) over more segments than `2*totalMaxConns`. Today it stalls; afterwards it completes and `TOTAL_MemLock - TOTAL_MemReturned` (logged at `Workers.go:1001-1003`) returns to 0. A `-checkonly` run does *not* reproduce this. +- **3.1/3.2:** stop server B mid-run; `LaunchSession` must still return. +- **3.10:** a provider with `MaxConns: 0`: today boot hangs; afterwards it is skipped. +- **Goroutine counts** before/after N sequential sessions, via `SIGUSR1` (`setupSigusr1Dump`, `main.go:66`) or `SIGQUIT`, for 3.7. +- **Web:** `-web -webaddr 127.0.0.1:11911 -provider=provider.test.json`; run the `curl` examples from `WEB_JOB_QUEUE.md`; start two jobs; poll `/api/jobs` and `/api/stats` from two terminals in a loop under `-race`, including while a session shuts down (3.14 nil `ConnPool`). Start a second instance on the same port: it must exit non-zero (4.1). Check the D3 rejections. +- **Proxy:** point a newsreader (or a scripted `nc` session) at the proxy, backed by server A. Check `MODE READER`; `LAST` at the start of a group (after the GROUP fix); a 600-byte line (rejected without disconnect) and a 300-byte message-ID (accepted); `XOVER 1-2-3` (`501`); an unknown command (`500`, no disconnect); `QUIT` still closes; an abrupt disconnect mid-article. Provider connection counts must return to baseline. +- **Phase 8:** decoder tests against rapidyenc on the generated corpus; `go test -bench` comparison. From dcce00e53d08b7ca9b8f65984f3725793e648b6f Mon Sep 17 00:00:00 2001 From: go-while Date: Sat, 17 Oct 2026 12:15:34 +0000 Subject: [PATCH 120/152] Bump Go to 1.27.2 - go.mod: go 1.25.3 -> 1.27.2 (go mod tidy: no changes) - CI build_and_publish_test: setup-go 1.25.3 -> 1.27.2 (codeql and goreleaser workflows read go-version-file: go.mod) - copilot-instructions: required Go version matches go.mod Verified with go1.27.2 on linux/amd64: go build, go vet, gofmt, go test -race ./rapidyenc/, NZBreX -testrapidyenc. Plan: review-the-codebase-for-jolly-petal.md, Phase 1. Co-Authored-By: Claude Opus 5.5 --- .github/copilot-instructions.md | 2 +- .github/workflows/build_and_publish_test.yml | 2 +- go.mod | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index f6b97e2..b392151 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -13,7 +13,7 @@ Always reference these instructions first and fallback to search or bash command # For cross-compilation (optional): apt-get install gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu mingw-w64 gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 g++-aarch64-linux-gnu ``` -- Go 1.24.3 or later is required (as specified in go.mod) +- Go 1.27.2 or later is required (as specified in go.mod) - CMake 3.31+ is required for building rapidyenc ### Bootstrap, Build, and Test the Repository diff --git a/.github/workflows/build_and_publish_test.yml b/.github/workflows/build_and_publish_test.yml index f6c4785..27b02dc 100644 --- a/.github/workflows/build_and_publish_test.yml +++ b/.github/workflows/build_and_publish_test.yml @@ -130,7 +130,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v5 with: - go-version: '1.25.3' + go-version: '1.27.2' cache: false # Install system dependencies based on runner OS diff --git a/go.mod b/go.mod index f53772a..0a424ba 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/go-while/NZBreX -go 1.25.3 +go 1.27.2 require ( github.com/Tensai75/nzbparser v0.1.0 From 4eab79ffffa549155f8dff8903d7f2a09f440ef7 Mon Sep 17 00:00:00 2001 From: go-while Date: Sat, 17 Oct 2026 12:33:47 +0000 Subject: [PATCH 121/152] Add nntpseed test tool and local testing notes tools/nntpseed generates deterministic yEnc test articles (optionally with bad CRC or truncated yEnc), feeds them to a local NNTP server via IHAVE, and writes a matching .nzb plus a JSON manifest. -omit-every leaves gaps, so a second server can act as the re-upload target. nzbs/LOCAL_TESTING.md: tests and uploads only go to local go-while/nntp-server instances; TODO for fetching the Debian NZB articles once (read-only) to test with real data. Co-Authored-By: Claude Opus 5.5 --- nzbs/LOCAL_TESTING.md | 32 +++++ tools/nntpseed/main.go | 293 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 325 insertions(+) create mode 100644 nzbs/LOCAL_TESTING.md create mode 100644 tools/nntpseed/main.go diff --git a/nzbs/LOCAL_TESTING.md b/nzbs/LOCAL_TESTING.md new file mode 100644 index 0000000..5566add --- /dev/null +++ b/nzbs/LOCAL_TESTING.md @@ -0,0 +1,32 @@ +# Testing with these NZBs + +These NZBs point at real Usenet articles. Tests must not run against live providers. All test runs, and **every upload**, go to local go-while/nntp-server instances only. + +## Local test servers + +| Server | Address | Storage | Content | +|---|---|---|---| +| A | `127.0.0.1:11119` | `/tank0/tmp/nzbrex/nntp-server/a/` | full article set | +| B | `127.0.0.1:11120` | `/tank0/tmp/nzbrex/nntp-server/b/` | partial set (re-upload target) | + +Start each with `nntp-server -config /config.json -dropbinary=false` (the default rejects yEnc). A `127.0.0.1` peer entry with no `ReadOnlyAccess` allows `IHAVE` and reading without login. + +Test runs use `/tank0/tmp/nzbrex/run/provider.test.json`, which points only at those two ports. Run from `/tank0/tmp/nzbrex/run`, which has no `provider.json`. `race_linux.sh` defaults to `-provider=provider.json`, so don't run it as-is. + +Synthetic data: `go run ./tools/nntpseed -addr 127.0.0.1:11119 -files 2 -parts 10 -broken-crc 1 -broken-yenc 1 -nzb test.nzb -manifest test.manifest.json`. For server B add `-omit-every 3` (same `-seed`). + +## TODO (later): real data from the Debian NZBs + +Download the articles of the Debian NZBs once, from cubenet (read only: `ARTICLE`), and feed them into servers A and B with `IHAVE` (B with gaps). Then the real NZBs here work unchanged against the local servers. + +| NZB | Segments | Size | +|---|---|---| +| `debian-11.6.0-amd64-netinst.iso.nzb.gz` | 581 | 410 MiB (start here) | +| `debian-12.9.0-amd64-netinst.nzb.gz` | 925 | 653 MiB | +| `debian-12.2.0-amd64-netinst.iso.nzb.gz` | 4020 | 2.8 GiB | +| `debian-12.9.0-amd64-DVD-1.iso.nzb.gz` | 5555 | 3.9 GiB | +| `debian-11.1.0-i386-DVD-1.nzb.gz` | 6560 | 4.1 GiB | + +- Use the **cubenet4** or **cubenet5** account, for downloading and checking only. Credentials are probably in `../nntp-proxy/config*.json`. Never copy them into this repo. +- Never upload anything to cubenet or any other live provider. +- Note which articles cubenet is missing; they are useful "missing article" cases. diff --git a/tools/nntpseed/main.go b/tools/nntpseed/main.go new file mode 100644 index 0000000..54f65c9 --- /dev/null +++ b/tools/nntpseed/main.go @@ -0,0 +1,293 @@ +// Command nntpseed generates deterministic yEnc test articles, feeds them to a +// local NNTP server with IHAVE, and writes a matching .nzb file. +// +// It exists so NZBreX can be tested against a local go-while/nntp-server +// instead of live Usenet providers. The same -seed always produces the same +// articles, so several servers can be filled from one data set, each missing +// a different subset (-omit-every). +// +// Example: +// +// nntpseed -addr 127.0.0.1:11119 -files 2 -parts 20 -broken-crc 1 -broken-yenc 1 -nzb test.nzb +// nntpseed -addr 127.0.0.1:11120 -files 2 -parts 20 -omit-every 3 -nzb /dev/null +package main + +import ( + "bufio" + "encoding/json" + "encoding/xml" + "flag" + "fmt" + "hash/crc32" + "log" + "math/rand" + "net" + "net/textproto" + "os" + "time" +) + +const lineLen = 128 + +type article struct { + MsgID string `json:"msgid"` // without angle brackets + File int `json:"file"` + Part int `json:"part"` + Bytes int `json:"bytes"` + Kind string `json:"kind"` // ok | badcrc | truncated + Sent bool `json:"sent"` + Result string `json:"result"` + body []byte + name string + total int + begin int + size int + fcrc uint32 +} + +func main() { + addr := flag.String("addr", "127.0.0.1:11119", "NNTP server address (must accept IHAVE from this host)") + group := flag.String("group", "alt.binaries.nzbrex.test", "newsgroup for the articles") + seed := flag.Int64("seed", 1, "random seed; same seed = same articles") + files := flag.Int("files", 1, "number of files") + parts := flag.Int("parts", 10, "parts per file") + partSize := flag.Int("partsize", 64*1024, "decoded bytes per part") + brokenCRC := flag.Int("broken-crc", 0, "number of parts (from the end of the last file) with a wrong pcrc32") + brokenYenc := flag.Int("broken-yenc", 0, "number of parts (before the bad-crc ones) with the =yend line cut off") + omitEvery := flag.Int("omit-every", 0, "if > 0: do not send every Nth article (1-based), to leave gaps on this server") + nzbOut := flag.String("nzb", "", "write the .nzb file here (empty: don't write)") + manifest := flag.String("manifest", "", "write a JSON list of all articles and what happened to them") + dry := flag.Bool("dry", false, "generate only, don't connect") + flag.Parse() + + rng := rand.New(rand.NewSource(*seed)) + var arts []*article + for f := 1; f <= *files; f++ { + name := fmt.Sprintf("nzbrex-test-s%d-f%d.bin", *seed, f) + data := make([]byte, *parts**partSize) + rng.Read(data) + fcrc := crc32.ChecksumIEEE(data) + for p := 1; p <= *parts; p++ { + begin := (p - 1) * *partSize + arts = append(arts, &article{ + MsgID: fmt.Sprintf("s%d.f%d.p%d.%s@nzbrex.test", *seed, f, p, randTag(rng)), + File: f, Part: p, Kind: "ok", + body: data[begin : begin+*partSize], name: name, total: *parts, + begin: begin + 1, size: len(data), fcrc: fcrc, + }) + } + } + // Mark broken articles at the end of the list so the first files stay clean. + n := len(arts) + for i := 0; i < *brokenCRC && i < n; i++ { + arts[n-1-i].Kind = "badcrc" + } + for i := 0; i < *brokenYenc && *brokenCRC+i < n; i++ { + arts[n-1-*brokenCRC-i].Kind = "truncated" + } + + if !*dry { + if err := feed(*addr, *group, arts, *omitEvery); err != nil { + log.Fatalf("feed: %v", err) + } + } + if *nzbOut != "" { + if err := writeNZB(*nzbOut, *group, arts); err != nil { + log.Fatalf("nzb: %v", err) + } + } + if *manifest != "" { + b, _ := json.MarshalIndent(arts, "", " ") + if err := os.WriteFile(*manifest, b, 0o644); err != nil { + log.Fatalf("manifest: %v", err) + } + } + counts := map[string]int{} + for _, a := range arts { + counts[a.Kind]++ + if a.Sent { + counts["sent:"+a.Result]++ + } + } + log.Printf("articles=%d %v", len(arts), counts) +} + +func randTag(r *rand.Rand) string { + const abc = "abcdefghijklmnopqrstuvwxyz0123456789" + b := make([]byte, 8) + for i := range b { + b[i] = abc[r.Intn(len(abc))] + } + return string(b) +} + +// encodeArticle returns the article lines (headers, blank line, yEnc body), +// without dot-stuffing; textproto.DotWriter does that. +func encodeArticle(a *article, group string) []string { + date := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).Add(time.Duration(a.File*1000+a.Part) * time.Second) + lines := []string{ + "From: nzbrex-test ", + "Newsgroups: " + group, + fmt.Sprintf("Subject: [%d] \"%s\" yEnc (%d/%d)", a.File, a.name, a.Part, a.total), + "Message-ID: <" + a.MsgID + ">", + "Date: " + date.Format(time.RFC1123Z), + "Path: nzbrex.test!not-for-mail", + "", + fmt.Sprintf("=ybegin part=%d total=%d line=%d size=%d name=%s", a.Part, a.total, lineLen, a.size, a.name), + fmt.Sprintf("=ypart begin=%d end=%d", a.begin, a.begin+len(a.body)-1), + } + body := yencLines(a.body) + lines = append(lines, body...) + pcrc := crc32.ChecksumIEEE(a.body) + if a.Kind == "badcrc" { + pcrc ^= 0xdeadbeef + } + yend := fmt.Sprintf("=yend size=%d part=%d pcrc32=%08x", len(a.body), a.Part, pcrc) + if a.Part == a.total { + yend += fmt.Sprintf(" crc32=%08x", a.fcrc) + } + switch a.Kind { + case "truncated": + // cut the body in half and drop =yend + lines = lines[:len(lines)-len(body)/2] + default: + lines = append(lines, yend) + } + return lines +} + +// yencLines encodes data as yEnc lines of at most lineLen output characters. +func yencLines(data []byte) []string { + var out []string + var line []byte + for i, c := range data { + e := byte(c + 42) + esc := e == 0 || e == '\n' || e == '\r' || e == '=' + if len(line) == 0 && (e == '\t' || e == ' ' || e == '.') { + esc = true + } + if (len(line) >= lineLen-1 || i == len(data)-1) && (e == '\t' || e == ' ') { + esc = true // no trailing whitespace + } + if esc { + line = append(line, '=', e+64) + } else { + line = append(line, e) + } + if len(line) >= lineLen { + out = append(out, string(line)) + line = line[:0] + } + } + if len(line) > 0 { + out = append(out, string(line)) + } + return out +} + +func feed(addr, group string, arts []*article, omitEvery int) error { + conn, err := net.DialTimeout("tcp", addr, 10*time.Second) + if err != nil { + return err + } + defer conn.Close() + tp := textproto.NewConn(conn) + if _, _, err := tp.ReadCodeLine(200); err != nil { + return fmt.Errorf("welcome: %w", err) + } + for i, a := range arts { + if omitEvery > 0 && (i+1)%omitEvery == 0 { + continue + } + code, msg, err := ihave(tp, a, group) + if err != nil { + return fmt.Errorf("IHAVE <%s>: %w", a.MsgID, err) + } + a.Sent = true + a.Result = fmt.Sprintf("%d", code) + if code != 235 && code != 435 { + log.Printf("IHAVE <%s> kind=%s: %d %s", a.MsgID, a.Kind, code, msg) + } + } + tp.PrintfLine("QUIT") + return nil +} + +func ihave(tp *textproto.Conn, a *article, group string) (int, string, error) { + if err := tp.PrintfLine("IHAVE <%s>", a.MsgID); err != nil { + return 0, "", err + } + code, msg, err := tp.ReadCodeLine(335) + if err != nil { + return code, msg, nil // 435/436/437: server doesn't want it + } + dw := tp.DotWriter() + bw := bufio.NewWriter(dw) + for _, l := range encodeArticle(a, group) { + bw.WriteString(l) + bw.WriteString("\n") + } + if err := bw.Flush(); err != nil { + return 0, "", err + } + if err := dw.Close(); err != nil { + return 0, "", err + } + code, msg, err = tp.ReadCodeLine(0) + if err != nil && code == 0 { + return 0, "", err + } + return code, msg, nil +} + +type nzbXML struct { + XMLName xml.Name `xml:"nzb"` + Xmlns string `xml:"xmlns,attr"` + Files []nzbFile `xml:"file"` +} + +type nzbFile struct { + Poster string `xml:"poster,attr"` + Date int64 `xml:"date,attr"` + Subject string `xml:"subject,attr"` + Groups []string `xml:"groups>group"` + Segments []nzbSeg `xml:"segments>segment"` +} + +type nzbSeg struct { + Bytes int `xml:"bytes,attr"` + Number int `xml:"number,attr"` + ID string `xml:",chardata"` +} + +func writeNZB(path, group string, arts []*article) error { + n := nzbXML{Xmlns: "http://www.newzbin.com/DTD/2003/nzb"} + byFile := map[int]int{} // file number -> index in n.Files + for _, a := range arts { + i, ok := byFile[a.File] + if !ok { + n.Files = append(n.Files, nzbFile{ + Poster: "nzbrex-test ", + Date: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).Unix(), + Subject: fmt.Sprintf("[%d] \"%s\" yEnc (1/%d)", a.File, a.name, a.total), + Groups: []string{group}, + }) + i = len(n.Files) - 1 + byFile[a.File] = i + } + raw := 0 + for _, l := range encodeArticle(a, group) { + raw += len(l) + 2 + } + a.Bytes = raw + n.Files[i].Segments = append(n.Files[i].Segments, nzbSeg{Bytes: raw, Number: a.Part, ID: a.MsgID}) + } + b, err := xml.MarshalIndent(n, "", " ") + if err != nil { + return err + } + out := []byte(xml.Header + "\n") + out = append(out, b...) + out = append(out, '\n') + return os.WriteFile(path, out, 0o644) +} From e04cbf4a08488ee64a41d005e4679bb19010e16c Mon Sep 17 00:00:00 2001 From: go-while Date: Sat, 17 Oct 2026 12:33:47 +0000 Subject: [PATCH 122/152] Fix watchDir deadlock, pre-auth CAPABILITIES panic, proxy -crc32 panics Plan items 2.1-2.3 (review-the-codebase-for-jolly-petal.md). 2.1 watchDirThread held p.mux.RLock() while refreshDir() and newSession() take p.mux.Lock(), deadlocking on the first tick. Read p.refresh under a short lock instead. (Latent: DirRefresh can't be set today.) 2.2 CAPABILITIES before AUTHINFO dereferenced a nil ps.CliTp and killed the process. Set CliTp when the session is created, fix the nil check in handleRequest, and add a recover() safety net in handleConnection. 2.3 Proxy articles had a nil item mutex: with -crc32 a bad-CRC article panicked at NetConn.go:890 (yenctest 1-3), and with -yencout every article panicked in Cache.WriteYenc. Give proxy items a mutex, guard the badcrc lock, and skip WriteYenc for items without a session. Session_test.go adds regression tests for 2.1 and 2.2; both fail on the old code. 2.2 and 2.3 were also reproduced and verified against a local nntp-server. go test -race ./... passes. Co-Authored-By: Claude Opus 5.5 --- Cache.go | 5 ++ NetConn.go | 8 +- Server.go | 26 ++++++- Session.go | 9 ++- Session_test.go | 104 +++++++++++++++++++++++++ review-the-codebase-for-jolly-petal.md | 27 ++++++- 6 files changed, 167 insertions(+), 12 deletions(-) create mode 100644 Session_test.go diff --git a/Cache.go b/Cache.go index a9a65e6..747f9a1 100644 --- a/Cache.go +++ b/Cache.go @@ -294,6 +294,11 @@ func (c *Cache) GetYenc(item *segmentChanItem) (filename string, filename_tmp st } // end func GetYenc func (c *Cache) WriteYenc(item *segmentChanItem, yPart *yenc.Part) { + // proxy items have no session, file or nzb name: GetYenc and the writer + // would dereference nil (in another goroutine, so nothing could recover it). + if proxy || item == nil || item.mux == nil || item.s == nil || item.file == nil || item.nzbhashname == nil { + return + } if c.yenc_writer_chan == nil { dlog(always, "ERROR cache.WriteYenc yenc_writer_chan is nil") return diff --git a/NetConn.go b/NetConn.go index e999743..edca7eb 100644 --- a/NetConn.go +++ b/NetConn.go @@ -887,9 +887,11 @@ readlines: } // end switch yencTest dlog(cfg.opt.DebugWorker, "readDotLines: YencCRC yenctest=%d brokenYenc=%t seg.Id='%s' @ '%s' rxb=%d content=(%d lines) Part.Validate:took=(%d µs) readDotLines:took=(%d µs) startReadSignals:took=(%d µs) cfg.opt.YencWrite=%t err='%v'", cfg.opt.YencTest, brokenYenc, item.segment.Id, connitem.c.provider.Name, rxb, len(content), time.Since(startReadLines).Microseconds(), time.Since(yencstart).Microseconds(), time.Since(startReadSignals).Microseconds(), cfg.opt.YencWrite, err) if isBadCrc || brokenYenc { - item.mux.Lock() - item.badcrc++ - item.mux.Unlock() + if item.mux != nil { + item.mux.Lock() + item.badcrc++ + item.mux.Unlock() + } dlog(always, "ERROR readDotLines crc32 failed seg.Id='%s' @ '%s'", item.segment.Id, connitem.c.provider.Name) return 99932, rxb, nil, nil // 99932 is a custom code for bad crc32 } diff --git a/Server.go b/Server.go index db1d104..88e1d27 100644 --- a/Server.go +++ b/Server.go @@ -14,20 +14,33 @@ import ( mrand "math/rand" // Added for random delays in password verification (to mitigate timing attacks) "net" "net/textproto" // Added for textproto + "runtime/debug" "strings" + + "github.com/go-while/go-loggedrwmutex" ) // handleConnection manages a single NNTP proxy client connection. func handleConnection(conn net.Conn) { + // Safety net: one bad client must never take down the whole process. + // This only covers this goroutine; it is not a substitute for fixing the panic. + // Registered first, so it runs after the cleanup defer below. + defer func() { + if r := recover(); r != nil { + dlog(always, "PANIC in proxy handleConnection remote='%s': %v\n%s", conn.RemoteAddr(), r, debug.Stack()) + conn.Close() + } + }() //log.Printf("Handling connection from %s", conn.RemoteAddr()) cliTp := textproto.NewConn(conn) var currentUser string // Stores username after successful AUTHINFO USER authenticated := false now := time.Now() // Get the current time for session initialization var ps = &ProxySession{ - Conn: conn, // Store the connection in the session - LastCmd: now, // Initialize last command time - ConnectedAt: now, // Set the connection time + Conn: conn, // Store the connection in the session + CliTp: cliTp, // set before any command is handled: CAPABILITIES works before auth + LastCmd: now, // Initialize last command time + ConnectedAt: now, // Set the connection time } // ProxySession to hold user session data // Ensure connection is closed and proxy session cleaned up when done @@ -299,7 +312,10 @@ func (ps *ProxySession) handleRequest(command string, args []string) error { // For now, we just log the command received. // returning any error (e.g.: via fmt.Errorf) will disconnect the user var retry error = nil // retry is used to indicate that the command should be retried - if !ps.Authed || (ps.Username == "") || (ps.CliTp == nil) || ps.ExpireAt < time.Now().Unix() { + if ps.CliTp == nil { + return fmt.Errorf("no client connection") + } + if !ps.Authed || (ps.Username == "") || ps.ExpireAt < time.Now().Unix() { ps.CliTp.PrintfLine("480 Authentication required for %s command", command) return fmt.Errorf("authentication required") } @@ -348,6 +364,7 @@ func (ps *ProxySession) handleRequest(command string, args []string) error { segment: &nzbparser.NzbSegment{ Id: args[0], }, + mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "PROXYITEM"}, // readDotLines locks item.mux } ps.MsgNum, ps.Group = 0, "" // Reset message number and group on valid pass = true // we have a valid message ID, so we can pass it to a provider @@ -360,6 +377,7 @@ func (ps *ProxySession) handleRequest(command string, args []string) error { segment: &nzbparser.NzbSegment{ Id: args[0], }, + mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "PROXYITEM"}, // readDotLines locks item.mux } } diff --git a/Session.go b/Session.go index d69841d..f8a286e 100644 --- a/Session.go +++ b/Session.go @@ -639,16 +639,19 @@ forever: case <-tAchan: //dlog( "watchDir checking '%s'", p.nzbDir) + // read p.refresh under a short lock only: + // refreshDir() and newSession() take p.mux themselves (not reentrant). p.mux.RLock() - if p.refresh > 0 { + refresh := p.refresh + p.mux.RUnlock() + if refresh > 0 { p.refreshDir() - tAchan = time.After(p.refresh) + tAchan = time.After(refresh) } else { // if p.refresh set to <= 0 don't run refreshDir // but recheck every 15 seconds if value has been updated tAchan = time.After(15 * time.Second) } - p.mux.RUnlock() } } // end for dlog(always, "Processor watchDir '%s' quit", p.nzbDir) diff --git a/Session_test.go b/Session_test.go new file mode 100644 index 0000000..c838ae1 --- /dev/null +++ b/Session_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "bufio" + "net" + "os" + "path/filepath" + "strings" + gotesting "testing" // Workers.go declares a const named testing + "time" + + "github.com/go-while/go-loggedrwmutex" +) + +// lockWithin reports whether p.mux can be write-locked within d. +// A deadlocked mutex never becomes free, so the helper goroutine leaks +// in that case; that is acceptable in a failing test. +func lockWithin(p *PROCESSOR, d time.Duration, fn func()) bool { + done := make(chan struct{}) + go func() { + p.mux.Lock() + fn() + p.mux.Unlock() + close(done) + }() + select { + case <-done: + return true + case <-time.After(d): + return false + } +} + +// Plan 2.1: watchDirThread used to hold p.mux.RLock() while refreshDir() +// takes p.mux.Lock() (not reentrant), deadlocking on the first tick. +func TestWatchDirThreadPicksUpFilesWithoutDeadlock(t *gotesting.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "a.nzb"), []byte(""), 0o644); err != nil { + t.Fatal(err) + } + p := &PROCESSOR{ + mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "TEST-PROCESSOR"}, + nzbDir: dir, + seenFiles: make(map[string]bool), + sessMap: make(map[uint64]*SESSION), + stopChan: make(chan struct{}, 1), + refresh: 15 * time.Second, + } + go p.watchDirThread() + + // first lookup happens after 5 seconds + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + var seen bool + var sessions int + if !lockWithin(p, 2*time.Second, func() { seen, sessions = p.seenFiles["a.nzb"], len(p.sessMap) }) { + t.Fatal("PROCESSOR mutex not available: watchDirThread deadlocked") + } + if seen && sessions == 1 { + p.stopChan <- struct{}{} + return + } + time.Sleep(200 * time.Millisecond) + } + t.Fatal("a.nzb was not picked up within 10s") +} + +// Plan 2.2: CAPABILITIES before AUTHINFO used to dereference a nil +// ps.CliTp and panic, killing the whole process. +func TestProxyCapabilitiesBeforeAuth(t *gotesting.T) { + srv, cli := net.Pipe() + defer cli.Close() + go handleConnection(srv) + + cli.SetDeadline(time.Now().Add(5 * time.Second)) + r := bufio.NewReader(cli) + welcome, err := r.ReadString('\n') + if err != nil { + t.Fatalf("welcome: %v", err) + } + if !strings.HasPrefix(welcome, "20") { + t.Fatalf("unexpected welcome %q", welcome) + } + if _, err := cli.Write([]byte("CAPABILITIES\r\n")); err != nil { + t.Fatal(err) + } + first, err := r.ReadString('\n') + if err != nil { + t.Fatalf("CAPABILITIES response: %v", err) + } + if !strings.HasPrefix(first, "101 ") { + t.Fatalf("want 101, got %q", first) + } + for { + line, err := r.ReadString('\n') + if err != nil { + t.Fatalf("capability list not terminated: %v", err) + } + if line == ".\r\n" { + break + } + } + cli.Write([]byte("QUIT\r\n")) +} diff --git a/review-the-codebase-for-jolly-petal.md b/review-the-codebase-for-jolly-petal.md index 765fa78..63df44d 100644 --- a/review-the-codebase-for-jolly-petal.md +++ b/review-the-codebase-for-jolly-petal.md @@ -4,7 +4,11 @@ The first draft was checked claim by claim against the working tree by four independent reviews. Almost every bug it lists is real and most line numbers are accurate to ±3. But several proposed fixes would have caused new hangs or panics, some cause-and-effect claims were wrong, and some bugs were missing. This revision fixes all of that. Items marked **(corrected)** changed in substance; **(new)** were missing; **(dropped)** turned out to be wrong. -Done since the first draft: **Phase 0** (rapidyenc C library from upstream), commit `c136df9`. +Done so far (branch `wip/web-job-queue-2025-10`): +- **Phase 0:** rapidyenc C library from upstream, `c136df9`. +- **Phase 1:** Go 1.27.2 bump, `dcce00e`. +- **2.1, 2.2, 2.3:** fixed, with regression tests for 2.1/2.2 (`Session_test.go`); reproduced and verified against local nntp-server (see each item's *Result*). +- **Test setup:** local nntp-server A/B and `tools/nntpseed` are working (see Verification). Decisions (answered 2026-10-17, see [Decisions](#decisions)): **D1** separate web passwd file; **D2** show per-job settings disabled; **D3** reject broken mode combinations at startup. @@ -57,12 +61,16 @@ To bump later: `git -C rapidyenc/rapidyenc fetch && git -C rapidyenc/rapidyenc c **Fix:** read `p.refresh` into a local under a short RLock, release, then call `refreshDir()` with no lock held. +**Result (2026-10-17):** fixed. **Correction:** not reachable from the command line today. `DirRefresh` has no flag, and the JSON config loader (`loadConfigFile`, `Utils.go:127-139`) is commented out, so `refresh` is always 0 and `refreshDir` never runs. The deadlock fires on the first tick as soon as the value can be set. `TestWatchDirThreadPicksUpFilesWithoutDeadlock` fails on the old code ("mutex not available") and passes now. + **Note:** even after the fix, `refreshDir` only *registers* sessions. `processorThread` is a TODO stub, so directory NZBs are never processed. See D3. ### 2.2 `CAPABILITIES` before login kills the process `ps` is built at `Server.go:27-31` without `CliTp`. `CliTp` is only assigned at `:235`, after `AUTHINFO PASS`. The pre-auth switch calls `ps.printCapabilities()` at `:129`, and its `PrintfLine` at `ProxyHelper.go:69` dereferences nil. With no `recover()` in the repo, `printf 'CAPABILITIES\r\n' | nc host 1119` kills the process. `Server.go:302-303` also dereferences `ps.CliTp` inside its own nil guard. +**Result (2026-10-17):** fixed. Reproduced first: `printf 'CAPABILITIES\r\n' | nc` against the unfixed proxy crashed it (panic at `ProxyHelper.go:69` via `Server.go:129`, exit 2). Now it returns the capability list and the proxy stays up. `TestProxyCapabilitiesBeforeAuth` panics on the old code and passes now. + **Fix:** assign `ps.CliTp = cliTp` at construction (the cleanup defer at `:54-58` already handles a non-nil `CliTp`), and fix `:302-303`. Add a `recover()` in `handleConnection` that logs and closes the connection as a safety net, not as the fix. It only covers that goroutine, and a panic after `GetConn` or while `proxyMutex` is held (`:178-247`) would still leak the connection or leave the mutex locked. ### 2.3 Nil mutex panic on a proxied article with `-crc32` (corrected) @@ -73,6 +81,15 @@ A third path also crashes: with `-crc32 -yencwrite` and a cache, `cache.WriteYen **Fix:** initialise `mux` at both proxy sites, make the `:889` guard match `:903`, and skip `cache.WriteYenc` when `proxy` is set. +**Result (2026-10-17):** fixed (`WriteYenc` also returns early for items with no session, file or nzb name). Reproduced against server A with the unfixed proxy: +- `-crc32 -yenctest=3`, bad-CRC article: panic at `NetConn.go:890`. +- `-crc32 -yenctest=4 -yencout -cd=…`, good article: panic at `Cache.go:307` via `NetConn.go:881`. + +Both now leave the proxy running, and a good article still arrives intact. Findings for Phase 5.2: +- **Mode 4 (default):** a CRC mismatch comes back from rapidyenc as a read error, so it takes the `brokenYenc` path. The client gets `220` with an **empty** article (43 bytes), confirming the silent-truncation bug. +- **(new) Mode 2:** a bad-CRC article is passed to the client as a normal `220`. +- **(new)** After the fix, the bad-CRC case answers `502 Unknown Response: 99932`, leaking an internal code. It should map to a proper NNTP error (e.g. `430`, or try the next provider). + ### 2.4 Memory slots never returned when nothing will upload (corrected) `MemLockWait` runs for every download item (`Workers.go:298`). `MemReturn` happens only on error paths (`Workers.go:312`, `:394`; `NetConn.go:144`; `Cache.go:410`) and after a re-upload (`Workers.go:400`). The comment at `Workers.go:326` saying memory was "already returned in GoDownsRoutine for cache reads" is false (`Routines.go:154-157`). @@ -411,7 +428,13 @@ All three answered by the user on 2026-10-17. **Never against live Usenet providers.** All runs that talk to a server use local go-while/nntp-server instances (`/tank0/efs/GO/src/github.com/go-while/nntp-server`), never `provider.json`. `race_linux.sh` hardcodes `-provider="provider.json"`, so don't run it as-is; use a copy that takes the provider file as an argument. -### Test setup +### Test setup (working, 2026-10-17) + +- Binaries in `/tank0/tmp/nzbrex/bin`: `nntp-server` (built from the sibling checkout), `nntpseed`, and NZBreX before/after builds. Runs happen in `/tank0/tmp/nzbrex/run`, which has `provider.test.json` (A download-only, B upload target with `PreferIHAVE`) and a test proxy user in `.proxypasswd.new.*`. +- Checked so far: a download run from A → B with `-crc32 -yenctest=4` copies every article missing on B, and `STAT` on B confirms them. +- `Workers.go:695` declares `const testing = false` (unused), so tests in package `main` must import `testing` under an alias. +- Later: real data from the Debian NZBs, fetched once (read-only) via the cubenet4/cubenet5 account. See `nzbs/LOCAL_TESTING.md`. Uploads only ever go to the local servers. + - **Storage:** `/tank0/tmp/nzbrex/nntp-server/` (created), one directory per instance: - `a/` on `127.0.0.1:11119`: holds every test article. From 9299fd83457e371efaf5e54a274befea0933fa7d Mon Sep 17 00:00:00 2001 From: go-while Date: Sat, 17 Oct 2026 13:21:26 +0000 Subject: [PATCH 123/152] Replace band-aid locks with proper publishing; fix WaitGroup race and web wedge Plan items 3.13 and 3.10, plus the web-server wedge found while testing (review-the-codebase-for-jolly-petal.md). 3.13 Revert the no-op globalmux locks (ConnPool.go WatchOpenConnsThread, LaunchSession around WaitGroup.Add). loadProviderList now parses into a local slice (the shared cfg.providers field is gone), builds the provider list and groups locally and publishes them once under s.mux. LaunchSession builds segmentList locally and publishes it once. The web stats readers take the same lock. 3.10 Speedmeter/WatchOpenConnsThread called workerWGconnReady.Wait() before LaunchSession's Add(). Add(1) now happens in loadProviderList right before NewConnPool; proxy/web sessions release it, GoBootWorkers releases it on its skip path. Providers with MaxConns <= 0 are skipped at load time: they used to hang the boot forever. Web wedge: collectStats hit a nil provider.ConnPool (closed by KillConnPool while the session was still active) while holding webProcessor.mux.RLock() without defer. Gin recovered the panic, the read lock leaked, removeSession blocked forever and the whole web server hung. The loop now unlocks via defer and copies ConnPool, counters and speeds under prov.mux, skipping closed pools. Verified against local nntp-servers with -race: CLI A->B clean; web mode with two jobs and 50 ms polling no longer wedges and drops from 29 races to only webserver.go job-field races (plan 4.2); MaxConns=0 provider no longer hangs; proxy mode still starts its pool watchers. Co-Authored-By: Claude Opus 5.5 --- Config.go | 3 +- ConnPool.go | 2 - Session.go | 27 +++-- Utils.go | 133 ++++++++++++++----------- Workers.go | 2 + review-the-codebase-for-jolly-petal.md | 14 +++ webserver.go | 64 +++++++----- 7 files changed, 150 insertions(+), 95 deletions(-) diff --git a/Config.go b/Config.go index 3aa6d4b..2162150 100644 --- a/Config.go +++ b/Config.go @@ -45,8 +45,7 @@ const ( ) type Config struct { - providers []Provider - opt *CFG + opt *CFG } type CFG struct { diff --git a/ConnPool.go b/ConnPool.go index fecbc8d..1c9e11c 100644 --- a/ConnPool.go +++ b/ConnPool.go @@ -566,9 +566,7 @@ func (c *ConnPool) GetStats() (openconns int, idle int) { // It will run until the global stopChan is closed or the provider is killed. // It will also print the status of the mutexes for debugging purposes. func (c *ConnPool) WatchOpenConnsThread(workerWGconnReady *sync.WaitGroup) { - globalmux.Lock() start := time.Now() - globalmux.Unlock() workerWGconnReady.Wait() dlog(cfg.opt.Debug, "WatchOpenConnsThread started for '%s' waited=%v", c.provider.Name, time.Since(start)) defer dlog(cfg.opt.Debug, "WatchOpenConnsThread for '%s' defer returned", c.provider.Name) diff --git a/Session.go b/Session.go index f8a286e..070777f 100644 --- a/Session.go +++ b/Session.go @@ -216,6 +216,8 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s dlog(cfg.opt.Debug, "nzbfile='%#v'", s.nzbFile) + // build the segment list locally and publish it once under s.mux below + segmentList := make([]*segmentChanItem, 0, nzbfile.TotalSegments) // loop through all file tags within the NZB file for _, file := range nzbfile.Files { if cfg.opt.Debug { @@ -268,25 +270,26 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s item.segment.Id = "<" + segment.Id + ">" // ensure segment.Id is always in the format "" for consistency } item.hashedId = SHA256str(item.segment.Id) - s.mux.Lock() - s.segmentList = append(s.segmentList, item) - s.mux.Unlock() + segmentList = append(segmentList, item) } } + s.mux.Lock() + s.segmentList = segmentList + s.mux.Unlock() mibsize := float64(nzbfile.Bytes) / 1024 / 1024 - artsize := mibsize / float64(len(s.segmentList)) * 1024 + artsize := mibsize / float64(len(segmentList)) * 1024 s.nzbSize = nzbfile.Bytes // store the size of the nzb file in bytes - dlog(always, "%s [%s] loaded NZB: '%s' [%d/%d] ( %.02f MiB | ~%.0f KiB/segment )", appName, appVersion, s.nzbName, len(s.segmentList), nzbfile.TotalSegments, mibsize, artsize) + dlog(always, "%s [%s] loaded NZB: '%s' [%d/%d] ( %.02f MiB | ~%.0f KiB/segment )", appName, appVersion, s.nzbName, len(segmentList), nzbfile.TotalSegments, mibsize, artsize) // left-padding for log output - s.digStr = fmt.Sprintf("%d", len(s.segmentList)) + s.digStr = fmt.Sprintf("%d", len(segmentList)) s.D = fmt.Sprintf("%d", len(s.digStr)) } // end if s.nzbFile } // end if !s.proxy // re-load the provider list var workerWGconnReady sync.WaitGroup // workerWGconnReady is used to wait for all connections to be established before starting the work - s.providerList = nil + // loadProviderList replaces s.providerList and s.providerGroups under s.mux if err := cfg.loadProviderList(s, &workerWGconnReady); err != nil { dlog(always, "ERROR unable to load providerfile '%s' err='%v'", cfg.opt.ProvFile, err) return err @@ -337,6 +340,11 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s globalmux.Unlock() if s.proxy { + // proxy/web sessions never boot workers: release the pool goroutines + // waiting on workerWGconnReady (one Add per provider in loadProviderList) + for range s.providerList { + workerWGconnReady.Done() + } // infinite block here for proxy server. if a wg is supplied it will never be released _, ok := <-s.stopChan if !ok { @@ -351,13 +359,13 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s var waitDivider sync.WaitGroup // waitDivider is used to block GoWorkDivider() until all workers are ready var waitDividerDone sync.WaitGroup // waitDividerDone is used to signal that GoWorkDivider() has quit var waitPool sync.WaitGroup // waitPool waits until all workers closed their connections and routines - globalmux.Lock() + // prepare wait groups waitDivider.Add(1) waitDividerDone.Add(1) for _, provider := range s.providerList { //waitWorker.Add(1) // this 1 releases in GoWorker when routines have booted - workerWGconnReady.Add(1) // #1 for every provider + // workerWGconnReady: #1 for every provider was already added in loadProviderList for i := 1; i <= provider.MaxConns; i++ { //workerWGconnReady.Add(1) // #2 for every connection per provider waitWorker.Add(3) // for the 3 routines per worker in go func will release when routines stop @@ -365,7 +373,6 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s } } // run the go routines - globalmux.Unlock() s.GoBootWorkers(&waitDivider, &workerWGconnReady, &waitWorker, &waitPool, s.nzbFile.Bytes) workerWGconnReady.Wait() // waits for all connections to be established before starting the work diff --git a/Utils.go b/Utils.go index 02136c8..60d454c 100644 --- a/Utils.go +++ b/Utils.go @@ -140,71 +140,90 @@ func loadConfigFile(path string) (*CFG, error) { */ // loadProviderList loads the provider list from the configuration file. -// It reads the JSON file specified in cfg.opt.ProvFile, unmarshals it into the cfg.providers slice, -// and initializes each provider with a connection pool. +// It reads the JSON file specified in cfg.opt.ProvFile into a local slice, +// initializes each enabled provider with a connection pool, and then publishes +// s.providerList and s.providerGroups under s.mux (the web stats reader locks it). +// Nothing shared is written while parsing, so concurrent sessions don't race. func (cfg *Config) loadProviderList(s *SESSION, workerWGconnEstablish *sync.WaitGroup) error { - if file, err := os.ReadFile(cfg.opt.ProvFile); err != nil { + file, err := os.ReadFile(cfg.opt.ProvFile) + if err != nil { return err - } else { - s.mux.Lock() - defer s.mux.Unlock() - if err := json.Unmarshal(file, &cfg.providers); err != nil { - return err + } + var providers []Provider + if err := json.Unmarshal(file, &providers); err != nil { + return err + } + var providerList []*Provider + var providerGroups []string + id := 0 + providernames := make(map[string]bool) // tmp map to check unique/duped provider names + providerACL := make(map[string]map[string]bool) // map to check for bad config combinations prevents bypass in pushDL/pushUP + for n := range providers { + if !providers[n].Enabled { + dlog(cfg.opt.Debug, "CFG Skipped Provider (n=%d) '%s' because not enabled", n, providers[n].Name) + continue } - id := 0 - providernames := make(map[string]bool) // tmp map to check unique/duped provider names - providerACL := make(map[string]map[string]bool) // map to check for bad config combinations prevents bypass in pushDL/pushUP - for n := range cfg.providers { - if !cfg.providers[n].Enabled { - dlog(cfg.opt.Debug, "CFG Skipped Provider (n=%d) '%s' because not enabled", n, cfg.providers[n].Name) - continue - } - Name := cfg.providers[n].Name - if providernames[Name] { - // duplicate provider Name - return fmt.Errorf("error in loadProviderList duplicate provider.Name='%s' (n=%d) in config", Name, n) - } - providernames[Name] = true - Group := cfg.providers[n].Group - if providerACL[Group] == nil { - providerACL[Group] = make(map[string]bool) - } - - // check for inconsistent configuration - if !cfg.providers[n].NoDownload && providerACL[Group]["NoDownload"] { - dlog(always, "ERROR loadProviderList provider '%s'. set 'NoDownload' to same value in group '%s'!", Name, Group) - os.Exit(1) - } - if !cfg.providers[n].NoUpload && providerACL[Group]["NoUpload"] { - dlog(always, "ERROR loadProviderList provider '%s'. set 'NoUpload' to same value in group '%s'!", Name, Group) - os.Exit(1) + if providers[n].MaxConns <= 0 { + // GoBootWorkers would never boot workers for it + dlog(always, "CFG Skipped Provider (n=%d) '%s' because MaxConns=%d", n, providers[n].Name, providers[n].MaxConns) + continue + } + Name := providers[n].Name + if providernames[Name] { + // duplicate provider Name + // release the workerWGconnEstablish counts taken for the providers loaded so far + for range providerList { + workerWGconnEstablish.Done() } + return fmt.Errorf("error in loadProviderList duplicate provider.Name='%s' (n=%d) in config", Name, n) + } + providernames[Name] = true + Group := providers[n].Group + if providerACL[Group] == nil { + providerACL[Group] = make(map[string]bool) + } - // read NoDownload / NoUpload values from provider and set once per group for all - if cfg.providers[n].NoDownload { - providerACL[Group]["NoDownload"] = true - } - if cfg.providers[n].NoUpload { - providerACL[Group]["NoUpload"] = true - } + // check for inconsistent configuration + if !providers[n].NoDownload && providerACL[Group]["NoDownload"] { + dlog(always, "ERROR loadProviderList provider '%s'. set 'NoDownload' to same value in group '%s'!", Name, Group) + os.Exit(1) + } + if !providers[n].NoUpload && providerACL[Group]["NoUpload"] { + dlog(always, "ERROR loadProviderList provider '%s'. set 'NoUpload' to same value in group '%s'!", Name, Group) + os.Exit(1) + } - if !slices.Contains(s.providerGroups, Group) { - // add group to providerGroups if not already present - s.providerGroups = append(s.providerGroups, Group) - } + // read NoDownload / NoUpload values from provider and set once per group for all + if providers[n].NoDownload { + providerACL[Group]["NoDownload"] = true + } + if providers[n].NoUpload { + providerACL[Group]["NoUpload"] = true + } - // link to this provider - provider := cfg.providers[n] - p := &provider - // provider is ready to connect - p.id = id - p.mux = &loggedrwmutex.LoggedSyncRWMutex{Name: p.Name} - NewConnPool(s, p, workerWGconnEstablish) - s.providerList = append(s.providerList, p) - dlog(cfg.opt.Debug, "CFG Loaded Provider (id=%d) '%s'", id, p.Name) - id++ + if !slices.Contains(providerGroups, Group) { + // add group to providerGroups if not already present + providerGroups = append(providerGroups, Group) } - } + + // every session gets its own copy of each provider + p := &providers[n] + // provider is ready to connect + p.id = id + p.mux = &loggedrwmutex.LoggedSyncRWMutex{Name: p.Name} + // Add before NewConnPool: its Speedmeter and WatchOpenConnsThread goroutines + // call workerWGconnEstablish.Wait() right away. + // Released once per provider by GoBootWorkers, or by LaunchSession for proxy sessions. + workerWGconnEstablish.Add(1) + NewConnPool(s, p, workerWGconnEstablish) + providerList = append(providerList, p) + dlog(cfg.opt.Debug, "CFG Loaded Provider (id=%d) '%s'", id, p.Name) + id++ + } + s.mux.Lock() + s.providerList = providerList + s.providerGroups = providerGroups + s.mux.Unlock() return nil } // end func loadProviderList diff --git a/Workers.go b/Workers.go index 36dc22f..12473b1 100644 --- a/Workers.go +++ b/Workers.go @@ -47,7 +47,9 @@ func (s *SESSION) GoBootWorkers(waitDivider *sync.WaitGroup, workerWGconnReady * dlog(cfg.opt.DebugWorker, "GoBootWorkers list=%d check provider='%#v' ", len(s.providerList), provider.Name) if !provider.Enabled || provider.MaxConns <= 0 { + // loadProviderList already filters these; release its Add(1) to keep the count balanced dlog(cfg.opt.Verbose, "ignored provider: '%s'", provider.Name) + workerWGconnReady.Done() continue } diff --git a/review-the-codebase-for-jolly-petal.md b/review-the-codebase-for-jolly-petal.md index 63df44d..3f6b312 100644 --- a/review-the-codebase-for-jolly-petal.md +++ b/review-the-codebase-for-jolly-petal.md @@ -8,6 +8,7 @@ Done so far (branch `wip/web-job-queue-2025-10`): - **Phase 0:** rapidyenc C library from upstream, `c136df9`. - **Phase 1:** Go 1.27.2 bump, `dcce00e`. - **2.1, 2.2, 2.3:** fixed, with regression tests for 2.1/2.2 (`Session_test.go`); reproduced and verified against local nntp-server (see each item's *Result*). +- **3.13 + 3.10:** band-aid locks replaced in one change, plus the web-server wedge fix from 3.14 (see each item's *Result*). - **Test setup:** local nntp-server A/B and `tools/nntpseed` are working (see Verification). Decisions (answered 2026-10-17, see [Decisions](#decisions)): **D1** separate web passwd file; **D2** show per-job settings disabled; **D3** reject broken mode combinations at startup. @@ -192,6 +193,11 @@ Written under `globalmux` in each provider's boot goroutine (`Workers.go:65-81`) **Fix:** `Add()` before any pool is built; call `Done()` (or skip the `Add`) for providers that are skipped. +**Result (2026-10-17):** fixed together with 3.13 (reverting the `globalmux` band-aids made the race detector report this `Add`/`Wait` race, `ConnPool.go:137` vs `LaunchSession`). +- `loadProviderList` now calls `Add(1)` right before each `NewConnPool`, and skips providers with `MaxConns <= 0`. +- Proxy/web sessions, which never boot workers, release it in `LaunchSession`. `GoBootWorkers` calls `Done()` on its skip path. The error path releases what it added. +- Verified: with a `MaxConns: 0` provider the old binary hangs at boot (killed by timeout), the new one skips it and completes. Proxy mode still starts its pool watchers. The race report is gone. + ### 3.11 `ConnPools` map (corrected) Keyed by a per-session index (`Utils.go:200`, `ConnPool.go:134/181`), so sessions overwrite each other. Harmless: nothing ever reads the map. **Fix:** delete it. @@ -210,10 +216,18 @@ Keyed by a per-session index (`Utils.go:200`, `ConnPool.go:134/181`), so session **Ordering:** a plain revert reintroduces the races with `extractSessionStats`. Revert and replace in one change. +**Result (2026-10-17):** done, together with 3.10. +- Both no-op `globalmux` band-aids reverted. +- `loadProviderList` parses into a local slice (the shared `cfg.providers` field is removed; nothing else used it), builds the provider list and groups locally, and publishes both once under `s.mux`. `LaunchSession` builds `segmentList` locally and publishes it once under `s.mux`. +- `extractSessionStats` checks `segmentList` under the lock; `collectStats` copies `providerList` under `session.mux`. +- Verified with `-race`, CLI download A → B: clean before and after. +- Verified with `-race`, web mode (2 jobs, `/api/jobs` + `/api/stats` polled every 50 ms): the old build logged 29 races including `Session.go:272` vs `webserver.go:768`, then wedged (below). The new build completes both jobs. The remaining 20–26 races are all inside `webserver.go` job-field handling (4.2). + ### 3.14 Remaining unsynchronised accesses (corrected) Still races: - `Session.go:467`: `len(s.proc.sessMap)` read without `p.mux`. +- **(new, fixed 2026-10-17) Web server wedge.** Reproduced in the web race test. The nil-`ConnPool` panic below fired in `collectStats` while it held `webProcessor.mux.RLock()` without `defer`. Gin's recovery caught the panic, so the read lock leaked: `removeSession` then blocked forever on `p.mux.Lock()`, the next job never started, and every `/api/jobs` and `/api/stats` request hung. **Fix:** the session loop runs in a closure with a deferred unlock. Each provider's `ConnPool`, article counters and speeds are copied under one `prov.mux.RLock()`, and a nil pool is skipped. This also removed the `GoCheckRoutine` (`Routines.go:78-79`) vs `collectStats` counter races. - **`ConnPool.go:184` (worse than the draft said).** `provider.ConnPool = nil` runs in `KillConnPool` (`Session.go:387`) before `clearSession` (`:401`). Meanwhile `webserver.go:854-856` calls `prov.ConnPool.GetStats()` while `session.active` is still true. That is a reachable nil-pointer panic from a web poll. - `NetConn.go:987-989`: `checkCapabilities` holds `provider.mux` across two network reads with no deadline (`UseNoDeadline` makes `ExtendConn` a no-op). Add a deadline; don't hold the lock across I/O. diff --git a/webserver.go b/webserver.go index 7ca8c0f..b86107f 100644 --- a/webserver.go +++ b/webserver.go @@ -765,11 +765,14 @@ func handleAPIStats(c *gin.Context) { // extractSessionStats extracts real-time statistics from a SESSION // This function counts the various states of segments in the session func extractSessionStats(s *SESSION) *SessionStats { - if s == nil || s.segmentList == nil { + if s == nil { return nil } s.mux.RLock() defer s.mux.RUnlock() + if s.segmentList == nil { + return nil + } stats := &SessionStats{ TotalArticles: uint64(len(s.segmentList)), @@ -848,36 +851,50 @@ func collectStats() WebStats { defer globalmux.RUnlock() if webProcessor != nil { - webProcessor.mux.RLock() - // Iterate through all active sessions - for _, session := range webProcessor.sessMap { - if session != nil && session.active && session.providerList != nil { - for _, prov := range session.providerList { - openConns, idleConns := prov.ConnPool.GetStats() - - // Get bandwidth from provider counter - provRX := prov.ConnPool.Counter.GetValue("TOTAL_RXbytes") - provTX := prov.ConnPool.Counter.GetValue("TOTAL_TXbytes") - - // Get speed values stored by Speedmeter function + // closure + defer: a panic in here must never leave webProcessor.mux read-locked, + // or removeSession() blocks forever and the whole web server wedges + func() { + webProcessor.mux.RLock() + defer webProcessor.mux.RUnlock() + // Iterate through all active sessions + for _, session := range webProcessor.sessMap { + if session == nil || !session.active { + continue + } + session.mux.RLock() + providerList := session.providerList // published under session.mux by loadProviderList + session.mux.RUnlock() + for _, prov := range providerList { + // KillConnPool sets prov.ConnPool = nil and the workers update the + // counters under prov.mux: copy everything under one read lock prov.mux.RLock() - downloadSpeed := prov.speed.downloadSpeed + cp := prov.ConnPool + articles := prov.articles + downloadSpeed := prov.speed.downloadSpeed // stored by Speedmeter uploadSpeed := prov.speed.uploadSpeed prov.mux.RUnlock() + if cp == nil { + continue // pool already closed: session is shutting down + } + openConns, idleConns := cp.GetStats() + + // Get bandwidth from provider counter + provRX := cp.Counter.GetValue("TOTAL_RXbytes") + provTX := cp.Counter.GetValue("TOTAL_TXbytes") provStats := ProviderStats{ Name: prov.Name, Group: prov.Group, OpenConns: openConns, IdleConns: idleConns, - Checks: prov.articles.checked, - Downloads: prov.articles.downloaded, - Uploads: prov.articles.refreshed, - Checked: prov.articles.checked, - Available: prov.articles.available, - Missing: prov.articles.missing, - Downloaded: prov.articles.downloaded, - Uploaded: prov.articles.refreshed, + Checks: articles.checked, + Downloads: articles.downloaded, + Uploads: articles.refreshed, + Checked: articles.checked, + Available: articles.available, + Missing: articles.missing, + Downloaded: articles.downloaded, + Uploaded: articles.refreshed, TotalRXbyte: provRX, TotalTXbyte: provTX, TotalDownload: provRX, // For JSON compatibility @@ -888,8 +905,7 @@ func collectStats() WebStats { stats.Providers = append(stats.Providers, provStats) } } - } - webProcessor.mux.RUnlock() + }() } // Collect active job information From 1661308d693962873ac3f3bb3247154e4495755d Mon Sep 17 00:00:00 2001 From: go-while Date: Sat, 17 Oct 2026 18:10:29 +0000 Subject: [PATCH 124/152] Single close owner per channel; build segment chans once per group - s.stop() closes stopChan once (sync.Once); proxy/web sessions get a stopChan; LaunchSession rejects an already-stopped session. - closeWorkChans closes downs/reups once per group (was per provider); closeSession no longer closes channels again (close of closed channel). - makeSegmentChans builds all group channels and one feeder per group before any worker starts (fatal concurrent map read/write). - Default provider.Group to Name in loadProviderList: fixes ungrouped providers sharing group "" (startup exit on mixed NoUpload, proxy skipping other ungrouped providers after a 430). - Plan: stop-channel design settled; results for 2.5, 2.6, 3.1. Co-Authored-By: Claude Opus 5.5 --- Routines.go | 84 ++++++++++++-------- Session.go | 63 +++++++-------- Teardown_test.go | 106 +++++++++++++++++++++++++ Utils.go | 6 ++ Workers.go | 105 +++++++++++++----------- main.go | 14 ++-- review-the-codebase-for-jolly-petal.md | 19 ++++- 7 files changed, 275 insertions(+), 122 deletions(-) create mode 100644 Teardown_test.go diff --git a/Routines.go b/Routines.go index 5acd747..d0718f4 100644 --- a/Routines.go +++ b/Routines.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "log" "os" "time" ) @@ -527,43 +526,64 @@ func (s *SESSION) GoReupsRoutine(wid int, provider *Provider, item *segmentChanI return code, err } // end func GoReupsRoutine +// StopRoutines ends the session's work: it closes the download and re-upload +// channels (the workers drain what is left and exit) and then the stopChan. +// Called by LaunchSession after GoWorkDivider, the only sender on those channels, has quit. +// Safe to call more than once. func (s *SESSION) StopRoutines() { - if cfg.opt.Debug { - log.Print("StopRoutines: pushing") - } - // pushing nil into the segment chans will stop the routines - for _, provider := range s.providerList { - //closeSegmentChannel(s.segmentChansCheck[provider.Group]) - closeSegmentChannel(s.segmentChansDowns[provider.Group]) - closeSegmentChannel(s.segmentChansReups[provider.Group]) - - } - close(s.stopChan) - if cfg.opt.Debug { - log.Print("StopRoutines: released") - } + dlog(cfg.opt.Debug, "StopRoutines: closing work channels sessId=%d", s.sessId) + s.closeWorkChans() + s.stop() + dlog(cfg.opt.Debug, "StopRoutines: released sessId=%d", s.sessId) } // end func StopRoutines -func closeSegmentChannel(input chan *segmentChanItem) { - if input == nil { - log.Printf("closeChannel: input channel is nil") - return +// stop closes s.stopChan exactly once. A nil stopChan is ignored. +func (s *SESSION) stop() { + s.stopOnce.Do(func() { + if s.stopChan != nil { + close(s.stopChan) + } + }) +} // end func stop + +// stopped reports whether stop() has run. +func (s *SESSION) stopped() bool { + if s.stopChan == nil { + return false } select { - case item, ok := <-input: - if item != nil { - log.Printf("ERROR in closeSegmentChannel: LOST item seg.Id='%s'", item.segment.Id) - } - if !ok { - if cfg.opt.Debug { - log.Printf("closeSegmentChannel: channel %p already closed", input) - } - return - } + case <-s.stopChan: + return true default: - close(input) // close the channel if it is not already closed - if cfg.opt.Debug { - log.Printf("closeSegmentChannel: closed channel %p", input) + return false + } +} // end func stopped + +// closeWorkChans closes the download and re-upload channel of every provider +// group exactly once. The maps are keyed by group, so groups with several +// providers are closed only once (looping over providers closed them twice). +// The check channels belong to their feeder and are not touched here. +func (s *SESSION) closeWorkChans() { + s.closeChansOnce.Do(func() { + for group, ch := range s.segmentChansDowns { + closeSegmentChannel("downs", group, ch) } + for group, ch := range s.segmentChansReups { + closeSegmentChannel("reups", group, ch) + } + }) +} // end func closeWorkChans + +// closeSegmentChannel closes a segment channel. Must only be called by the +// channel's owner, once. Items still buffered are not lost: receivers get them +// before they see the close. +func closeSegmentChannel(kind string, group string, ch chan *segmentChanItem) { + if ch == nil { + return + } + if n := len(ch); n > 0 { + dlog(always, "closeSegmentChannel: %s channel of group '%s' still holds %d items, workers will drain them", kind, group, n) } + close(ch) + dlog(cfg.opt.Debug, "closeSegmentChannel: closed %s channel of group '%s'", kind, group) } // end func closeSegmentChannel diff --git a/Session.go b/Session.go index 070777f..ce802f2 100644 --- a/Session.go +++ b/Session.go @@ -46,20 +46,27 @@ type PROCESSOR struct { // It is created by the PROCESSOR when a new NZB file is detected and processed. type SESSION struct { //mux sync.RWMutex - stopChan chan struct{} // stopChan is used to stop the session/routines - mux *loggedrwmutex.LoggedSyncRWMutex - counter *Counter_uint64 // private counter for every session // TODO check all GCounter if they should be private - proc *PROCESSOR // the parent where we belong to - sessId uint64 // session ID - nzbName string // test.nzb - nzbHashedname string // the hashed name of the full filename with .extension - nzbPath string // /path/to/nzbDir/test.nzb(.gz) - nzbFile *nzbparser.Nzb // the parsed NZB file structure - nzbSize int64 // the byte size of all the files in the nzb. will be > 0 if nzbFile is loaded or has been loaded before - nzbGroups []string // informative + // stopChan is closed exactly once, by stop(), when the session's work is over. + // It is created with the session and never replaced: readers may copy it into a local. + stopChan chan struct{} + stopOnce sync.Once // guards close(stopChan) + closeChansOnce sync.Once // guards closing segmentChansDowns/Reups (see closeWorkChans) + mux *loggedrwmutex.LoggedSyncRWMutex + counter *Counter_uint64 // private counter for every session // TODO check all GCounter if they should be private + proc *PROCESSOR // the parent where we belong to + sessId uint64 // session ID + nzbName string // test.nzb + nzbHashedname string // the hashed name of the full filename with .extension + nzbPath string // /path/to/nzbDir/test.nzb(.gz) + nzbFile *nzbparser.Nzb // the parsed NZB file structure + nzbSize int64 // the byte size of all the files in the nzb. will be > 0 if nzbFile is loaded or has been loaded before + nzbGroups []string // informative //lastRun time.Time // lastRun is the time when the session was last run //nextRun time.Time // nextRun is the time when the session is scheduled to run next - segmentList []*segmentChanItem // list holds the segments from parsed nzbFile + segmentList []*segmentChanItem // list holds the segments from parsed nzbFile + // One channel per provider group, built once by GoBootWorkers before any worker starts, + // read-only while the session runs. Close owners: the group's feeder closes its check + // channel, closeWorkChans closes the downs/reups channels after the divider has quit. segmentChansCheck map[string]chan *segmentChanItem // map[provider.Group]chan segmentChansDowns map[string]chan *segmentChanItem // map[provider.Group]chan segmentChansReups map[string]chan *segmentChanItem // map[provider.Group]chan @@ -167,6 +174,10 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s if s == nil { return fmt.Errorf("error LaunchSession: session is nil, can't start") } + if s.stopped() { + // sessions are single-use: their stopChan and channels are closed for good + return fmt.Errorf("error LaunchSession: session %d has already run, create a new session", s.sessId) + } s.mux.Lock() if s.preBoot || s.active { // we are already booting or active @@ -486,28 +497,10 @@ func (s *SESSION) closeSession() { dlog(cfg.opt.Debug, "closeSession: sessId=%d nzbName='%s' nzbPath='%s'", s.sessId, s.nzbName, s.nzbPath) - // close all channels in segmentChansCheck, segmentChansDowns, segmentChansReup - // app will panic if anybody still sends which should not be possible - for n, ch := range s.segmentChansCheck { - close(ch) - s.segmentChansCheck[n] = nil // remove from map - } - for n, ch := range s.segmentChansDowns { - close(ch) - s.segmentChansDowns[n] = nil // remove from map - } - for n, ch := range s.segmentChansReups { - close(ch) - s.segmentChansReups[n] = nil // remove from map - } - - s.segmentChansCheck = nil - s.segmentChansDowns = nil - s.segmentChansReups = nil - - s.segmentChansCheck = make(map[string]chan *segmentChanItem) - s.segmentChansDowns = make(map[string]chan *segmentChanItem) - s.segmentChansReups = make(map[string]chan *segmentChanItem) + // The segment channels are not closed here: each has a single close owner + // (the group's feeder, or closeWorkChans via StopRoutines), see the SESSION struct. + // Closing them again here panicked with "close of closed channel". + // The maps are dropped below under s.mux. // todo cleanup memDL and memUP /* @@ -522,7 +515,7 @@ func (s *SESSION) closeSession() { } */ - dlog(cfg.opt.Debug, "cleanupSession: closed all segment channels for sessId=%d nzbName='%s' nzbPath='%s'", s.sessId, s.nzbName, s.nzbPath) + dlog(cfg.opt.Debug, "closeSession: cleaning up sessId=%d nzbName='%s' nzbPath='%s'", s.sessId, s.nzbName, s.nzbPath) s.mux.Lock() // lock the session for cleanup s.active = false // set active to false, we are no longer processing diff --git a/Teardown_test.go b/Teardown_test.go new file mode 100644 index 0000000..4f595a1 --- /dev/null +++ b/Teardown_test.go @@ -0,0 +1,106 @@ +package main + +import ( + gotesting "testing" // Workers.go declares a const named testing + "time" + + "github.com/go-while/go-loggedrwmutex" +) + +func testCfg() { + if cfg == nil { + cfg = &Config{opt: &CFG{}} + } +} + +func closedWithin(ch chan *segmentChanItem, d time.Duration) bool { + deadline := time.After(d) + for { + select { + case _, ok := <-ch: + if !ok { + return true + } + case <-deadline: + return false + } + } +} + +// Plan 2.6 + 3.1 + 2.5: every segment channel has one close owner and the +// maps are built once per group, so teardown never closes a channel twice. +// StopRoutines used to close the downs/reups channels per provider and +// closeSession closed them (and the feeder-closed check channels) again: +// "close of closed channel". +func TestSessionTeardownClosesEachChannelOnce(t *gotesting.T) { + testCfg() + cfg.opt.ChanSize = 4 + prov := func(name, group string) *Provider { + return &Provider{Name: name, Group: group, Enabled: true, MaxConns: 1, + mux: &loggedrwmutex.LoggedSyncRWMutex{Name: name}} + } + s := &SESSION{ + mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "TEST-SESSION"}, + stopChan: make(chan struct{}), + // two providers share group g: the old per-provider loop closed g twice + providerList: []*Provider{prov("a", "g"), prov("b", "g"), prov("c", "h")}, + } + defer func() { + if r := recover(); r != nil { + t.Fatalf("teardown panicked: %v", r) + } + }() + + s.makeSegmentChans() + if len(s.segmentChansCheck) != 2 || len(s.segmentChansDowns) != 2 || len(s.segmentChansReups) != 2 { + t.Fatalf("want one channel per group (2), got check=%d downs=%d reups=%d", + len(s.segmentChansCheck), len(s.segmentChansDowns), len(s.segmentChansReups)) + } + check, downs, reups := s.segmentChansCheck, s.segmentChansDowns, s.segmentChansReups + + // the feeders own the check channels: with no segments they close them after ~1s + for g, ch := range check { + if !closedWithin(ch, 5*time.Second) { + t.Fatalf("feeder did not close check channel of group %q", g) + } + } + + s.StopRoutines() + s.StopRoutines() // second call must be a no-op + for g := range downs { + if !closedWithin(downs[g], time.Second) || !closedWithin(reups[g], time.Second) { + t.Fatalf("downs/reups channel of group %q not closed", g) + } + } + if !s.stopped() { + t.Fatal("stopChan not closed") + } + + s.closeSession() // must not close anything again + if s.segmentChansCheck != nil || s.segmentChansDowns != nil || s.segmentChansReups != nil { + t.Fatal("closeSession did not drop the channel maps") + } + + // sessions are single-use + p := &PROCESSOR{mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "TEST-PROCESSOR"}} + if err := p.LaunchSession(s, "", nil); err == nil { + t.Fatal("LaunchSession accepted a session that already ran") + } +} + +// Plan 2.6: proxy/web sessions were built with a nil stopChan, so any +// StopRoutines on them panicked on close(nil). +func TestStopRoutinesWithoutChannels(t *gotesting.T) { + testCfg() + defer func() { + if r := recover(); r != nil { + t.Fatalf("StopRoutines panicked: %v", r) + } + }() + s := &SESSION{mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "TEST-PROXY"}, proxy: true} + s.StopRoutines() + s.StopRoutines() + if s.stopped() { + t.Fatal("nil stopChan reported as stopped") + } +} diff --git a/Utils.go b/Utils.go index 60d454c..f8a1387 100644 --- a/Utils.go +++ b/Utils.go @@ -178,6 +178,12 @@ func (cfg *Config) loadProviderList(s *SESSION, workerWGconnEstablish *sync.Wait return fmt.Errorf("error in loadProviderList duplicate provider.Name='%s' (n=%d) in config", Name, n) } providernames[Name] = true + if providers[n].Group == "" { + // a provider without a group is its own group. Set here, before anything reads it: + // GoBootWorkers used to do this per provider while other goroutines read Group, + // and providerGroups/providerACL here held "" for all of them. + providers[n].Group = Name + } Group := providers[n].Group if providerACL[Group] == nil { providerACL[Group] = make(map[string]bool) diff --git a/Workers.go b/Workers.go index 12473b1..d539adc 100644 --- a/Workers.go +++ b/Workers.go @@ -41,6 +41,8 @@ func (s *SESSION) GoBootWorkers(waitDivider *sync.WaitGroup, workerWGconnReady * cfg.opt.ChanSize = len(s.segmentList) } + s.makeSegmentChans() + // loop over the providerList and boot up anonymous workers for each provider for _, provider := range s.providerList { @@ -57,55 +59,8 @@ func (s *SESSION) GoBootWorkers(waitDivider *sync.WaitGroup, workerWGconnReady * go func(provider *Provider, workerWGconnReady *sync.WaitGroup, waitWorker *sync.WaitGroup, waitPool *sync.WaitGroup) { // releases at end of func when provider capabilities have been checked and all GoWorker() booted defer workerWGconnReady.Done() // release #1 for every provider in go func - if provider.Group == "" { - provider.Group = provider.Name - } - dlog(cfg.opt.Debug, "GoBootWorkers: Booting Provider '%s' group '%s'", provider.Name, provider.Group) - // create once if not exists - globalmux.Lock() - if s.segmentChansCheck == nil { - dlog(cfg.opt.Debug, "Creating chans for group '%s' cfg.opt.ChanSize=%d", provider.Group, cfg.opt.ChanSize) - if cfg.opt.Debug { - time.Sleep(2 * time.Second) // sleeps 2 sec to print a debug line about creating channels for group - } - // create maps which holds the channels for every provider group - s.segmentChansCheck = make(map[string]chan *segmentChanItem) - s.segmentChansDowns = make(map[string]chan *segmentChanItem) - s.segmentChansReups = make(map[string]chan *segmentChanItem) - } - if s.segmentChansCheck[provider.Group] == nil { - // create channels once if not exists - // since we run concurrently only the first worker hitting the lock will create the channels - s.segmentChansCheck[provider.Group] = make(chan *segmentChanItem, cfg.opt.ChanSize) - s.segmentChansDowns[provider.Group] = make(chan *segmentChanItem, cfg.opt.ChanSize) - s.segmentChansReups[provider.Group] = make(chan *segmentChanItem, cfg.opt.ChanSize) - - // launch anonymous go routine to feed the segmentChanCheck with pointers to segmentChanItem - go func(segmentChanCheck chan *segmentChanItem) { - - start := time.Now() - for _, item := range s.segmentList { - segmentChanCheck <- item // is allowed to block inside feeder routine - } - for { - time.Sleep(time.Second) // wait for check routine to empty out the chan - if len(segmentChanCheck) == 0 { - break - } - } - dlog(cfg.opt.Verbose, " | Done feeding items=%d -> segmentChanCheck Group '%s' took='%.0f sec'", len(s.segmentList), provider.Group, time.Since(start).Seconds()) - s.mux.Lock() - // set checkFeedDone only signals that feeding to the channel is done - // the check routine may still be activly checking! - s.checkFeedDone = true - s.mux.Unlock() - close(segmentChanCheck) // close the chan to signal no more items will come - }(s.segmentChansCheck[provider.Group]) - } - globalmux.Unlock() - dlog(cfg.opt.Verbose, "Connecting to Provider '%s' MaxConns=%d SSL=%t", provider.Name, provider.MaxConns, provider.SSL) if !cfg.opt.CheckOnly && !provider.NoUpload { @@ -163,6 +118,60 @@ func (s *SESSION) GoBootWorkers(waitDivider *sync.WaitGroup, workerWGconnReady * dlog(cfg.opt.DebugWorker, "GoBootWorkers: all workers connected (or died if no conn could be established)") } // end func GoBootWorkers +// makeSegmentChans builds the check/downs/reups channel of every provider group +// and starts one feeder per group. It runs before any GoWorker or the divider +// starts, so the maps are complete before anyone reads them and stay read-only +// afterwards (they used to be written by each provider's boot goroutine while +// other groups' workers already read them: fatal concurrent map read and write). +// provider.Group is defaulted to provider.Name in loadProviderList. +func (s *SESSION) makeSegmentChans() { + check := make(map[string]chan *segmentChanItem) + downs := make(map[string]chan *segmentChanItem) + reups := make(map[string]chan *segmentChanItem) + for _, provider := range s.providerList { + if !provider.Enabled || provider.MaxConns <= 0 { + continue + } + if check[provider.Group] != nil { + continue + } + dlog(cfg.opt.Debug, "Creating chans for group '%s' cfg.opt.ChanSize=%d", provider.Group, cfg.opt.ChanSize) + check[provider.Group] = make(chan *segmentChanItem, cfg.opt.ChanSize) + downs[provider.Group] = make(chan *segmentChanItem, cfg.opt.ChanSize) + reups[provider.Group] = make(chan *segmentChanItem, cfg.opt.ChanSize) + } + s.mux.Lock() + s.segmentChansCheck = check + s.segmentChansDowns = downs + s.segmentChansReups = reups + s.mux.Unlock() + for group, ch := range check { + go s.feedCheckChan(group, ch) // the feeder owns ch and is the only one closing it + } +} // end func makeSegmentChans + +// feedCheckChan pushes every segment of the session into the check channel of one group, +// waits for the check routines to empty it, then closes it. +func (s *SESSION) feedCheckChan(group string, segmentChanCheck chan *segmentChanItem) { + start := time.Now() + for _, item := range s.segmentList { + segmentChanCheck <- item // is allowed to block inside feeder routine + } + for { + time.Sleep(time.Second) // wait for check routine to empty out the chan + if len(segmentChanCheck) == 0 { + break + } + } + dlog(cfg.opt.Verbose, " | Done feeding items=%d -> segmentChanCheck Group '%s' took='%.0f sec'", len(s.segmentList), group, time.Since(start).Seconds()) + s.mux.Lock() + // set checkFeedDone only signals that feeding to the channel is done + // the check routine may still be activly checking! + s.checkFeedDone = true + s.mux.Unlock() + close(segmentChanCheck) // close the chan to signal no more items will come +} // end func feedCheckChan + func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGroup, waitPool *sync.WaitGroup) { dlog(cfg.opt.DebugWorker, "GoWorker (%d) launching routines '%s'", wid, provider.Name) var thisWorkerWG sync.WaitGroup @@ -380,7 +389,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro } if item == nil { dlog(cfg.opt.DebugWorker, "ReupsRoutine received a nil pointer to quit") - s.segmentChansReups[provider.Group] <- nil // refill the nil so others will die too + segCR <- nil // refill the nil so others will die too break forGoReupsRoutine } timeOut = time.Now().Add(15 * time.Second) diff --git a/main.go b/main.go index 9b11ab3..a3afffd 100644 --- a/main.go +++ b/main.go @@ -96,9 +96,10 @@ func main() { // to establish connections to the providers and have pools running dlog(cfg.opt.Debug, "pre:thisProcessor.LaunchSession: proxy server") s := &SESSION{ - proc: thisProcessor, - mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "PROXYSESSION"}, - proxy: true, + proc: thisProcessor, + mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "PROXYSESSION"}, + stopChan: make(chan struct{}), // closed only by s.stop() + proxy: true, } ProxyParent = s // set global proxy session so proxy has access to provider list and pools go GoCliRxTxCounter() @@ -120,9 +121,10 @@ func main() { // similar to proxy server setup dlog(cfg.opt.Debug, "pre:thisProcessor.LaunchSession: web server") s := &SESSION{ - proc: thisProcessor, - mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "WEBSESSION"}, - proxy: true, // Set proxy flag to initialize providers without processing NZB + proc: thisProcessor, + mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "WEBSESSION"}, + stopChan: make(chan struct{}), // closed only by s.stop() + proxy: true, // Set proxy flag to initialize providers without processing NZB } webProcessor = thisProcessor webWg = wg diff --git a/review-the-codebase-for-jolly-petal.md b/review-the-codebase-for-jolly-petal.md index 3f6b312..8ab809f 100644 --- a/review-the-codebase-for-jolly-petal.md +++ b/review-the-codebase-for-jolly-petal.md @@ -109,6 +109,10 @@ Written under `globalmux` in each provider's boot goroutine (`Workers.go:65-81`) **Fix:** build all three maps once before any `GoWorker` starts, and treat them as read-only. Move the `provider.Group = provider.Name` defaulting (`Workers.go:58-60`) ahead of that, or the keys won't match `s.providerGroups` (`Utils.go:191-193`, which holds the raw, possibly empty, groups). Keep exactly one feeder per group. +**Result (2026-10-17):** fixed in the same change as 2.6/3.1. `makeSegmentChans()` builds all three maps and starts one `feedCheckChan` per group before the provider boot loop, then publishes them under `s.mux`. The group default (`Group = Name`) moved into `loadProviderList`. +- **Side effect, a bug fixed (new):** before, all ungrouped providers shared the group `""` in `loadProviderList`'s ACL check. One ungrouped download-only provider plus one ungrouped upload provider made the old build exit at start ("set 'NoUpload' to same value in group ''"). Verified with the 3.13 build, which exits 1; the new build runs the session. +- **Side effect in the proxy (new):** the proxy never ran `GoBootWorkers`, so its providers kept `Group == ""`. After the first provider answered 430, `checkedProviderGroups[""]` made the proxy skip every other ungrouped provider. Verified: with providers [B, A] and an article missing only on B, the 3.13 proxy returns `430 NOPG` to `STAT`, and the new one returns `223`. + ### 2.6 `close of closed channel` on session teardown (corrected; do together with 3.1) - `StopRoutines` (`Routines.go:537-538`) closes `segmentChansDowns`/`Reups` via `closeSegmentChannel`, and `closeSession` (`Session.go:484-495`) closes them again. Only reached when `len(sessMap) <= SessThreshold`; `SessThreshold` is JSON-config only and defaults to 0, which takes the `removeSession` branch. @@ -117,11 +121,20 @@ Written under `globalmux` in each provider's boot goroutine (`Workers.go:65-81`) **Fix:** one owner per channel, keyed by **group**, not provider. Guard with `sync.Once`. Create `stopChan` once in `LaunchSession` before any watcher starts. **Do not** recreate it inside `closeSession`: `WatchOpenConnsThread` re-reads the `c.s.stopChan` field every loop (`ConnPool.go:643`), so swapping it races, and a watcher that misses the close spins forever on its `default` branch. +**Result (2026-10-17):** fixed together with 3.1 and 2.5. +- `s.stop()` closes `stopChan` once (`sync.Once`), and ignores a nil channel. The proxy and web sessions in `main.go` now get a `stopChan`. Sessions are single-use: `LaunchSession` rejects one that has already been stopped (every caller creates a new session anyway). +- `closeWorkChans()` closes the downs/reups channels once, iterating the per-group maps, not the providers. The check channel belongs to its feeder. `closeSession` no longer closes anything; it only drops the maps. +- Tests: `TestSessionTeardownClosesEachChannelOnce` (two providers in one group, `StopRoutines` twice, then `closeSession`, then a relaunch is rejected) and `TestStopRoutinesWithoutChannels` (proxy-style session, nil channels). + --- ## Phase 3: Hangs and resource exhaustion in the engine -**Before starting 3.2, 3.6 or 3.7, settle the stop-channel design** (per pool or per session, captured once into a local, never swapped). Those three items and 2.6 all depend on it. +**Stop-channel design (settled 2026-10-17).** 3.2, 3.6, 3.7 and 2.6 depend on it. +- **Session lifetime** (workers, divider, feeders): `s.stopChan`. It is created with the session (`newSession`, and in `main.go` for proxy/web) and closed only by `s.stop()` (`sync.Once`). Nobody sends on it, and it is never replaced, so readers may copy it into a local once. Sessions are single-use. +- **Pool lifetime** (`WatchOpenConnsThread`, `Speedmeter`): a stop channel per `ConnPool`, closed once by `KillConnPool`. Pools are built per session but the proxy's pools are never killed, so they must not hang off the session's channel (to do in 3.7). +- **Global `memlim`**: `MemLockWait` takes the caller's session stop channel as a parameter and reports an abort (3.6). The limiter itself has no stop. +- **Segment channels**: one close owner each. The feeder owns the check channel; `closeWorkChans` (via `StopRoutines`, after the divider has quit) owns downs/reups. Nobody else closes them. ### 3.1 `closeSegmentChannel` leaves the channel open (corrected) @@ -129,6 +142,10 @@ Written under `globalmux` in each provider's boot goroutine (`Workers.go:65-81`) **Fix:** drain, then close **once per group**. `StopRoutines` loops over providers, not groups, so with several providers per group a plain "close unconditionally" panics on the second call, and `closeSession` closes again. This must land in the same change as 2.6. +**Result (2026-10-17):** fixed with 2.6. The owner closes the channel without draining it first: buffered items stay receivable, so the workers process them before they see the close, instead of the old code taking one item off and logging it as "LOST". A non-empty channel at close is logged. +- Verified under `-race` against the local servers: A→B, two providers in one group (`local-b1`/`local-b2`, which split the 9 uploads 5/4), and ungrouped providers. In every run each group's channels closed once, with no races and no panics. +- Web mode with two queued jobs: both complete. The 27 race reports are all in `webserver.go` job fields (4.2). + ### 3.2 Check feeder busy-wait (corrected) `Workers.go:84-103` spins until the channel is empty, then closes. If every worker failed to get a connection at boot, it spins forever. A second hang cause: `GoWorkDivider` keeps looping while `checked != todo` (`Workers.go:1029`). Selecting on `s.stopChan` alone does not help, because nothing closes `stopChan` until the divider has ended. From d42d56301b30c522baa8d16712488c85972a6134 Mon Sep 17 00:00:00 2001 From: go-while Date: Sat, 17 Oct 2026 21:40:07 +0000 Subject: [PATCH 125/152] Fix mem slot leak, make blocking waits stoppable, fix IHAVE conn wedge - 2.4: the divider retires checked items no provider can take an upload for (returns their mem slot, skips the download) and counts them in a new close case; normal A->B runs no longer end via the force-quit path. Slots still held at session end are returned (memlim is global). - 3.6: MemLockWait takes a stop channel, prefers a free slot, rolls back on abort and reports it. GetConn and SharedConnGet give up when the owning session stops. SharedConnGet extended the wrong (nil) conn. - 3.15 (new): CMD_IHAVE returned before EndResponse on any non-335 reply (435/436), so the next command on that conn blocked forever in textproto's StartResponse. Found by stopping server B mid-run. - Tests: MemLockWait abort/rollback, IHAVE 436-then-435 on one conn. Co-Authored-By: Claude Opus 5.5 --- ConnPool.go | 31 +++++++++- MemLimit.go | 53 +++++++++------- MemLimit_test.go | 64 ++++++++++++++++++++ NetConn.go | 9 ++- NetConn_test.go | 61 +++++++++++++++++++ Routines.go | 4 +- Session.go | 1 + Workers.go | 83 +++++++++++++++++++++++++- review-the-codebase-for-jolly-petal.md | 30 ++++++++++ 9 files changed, 308 insertions(+), 28 deletions(-) create mode 100644 MemLimit_test.go create mode 100644 NetConn_test.go diff --git a/ConnPool.go b/ConnPool.go index 1c9e11c..2eb5d9f 100644 --- a/ConnPool.go +++ b/ConnPool.go @@ -82,8 +82,14 @@ type ConnPool struct { // we cann call ConnPools[provider.id].CloseConn(connitem) provider *Provider // pointer to the provider which created this ConnPool and holds the config s *SESSION // session pointer to the session which created this ConnPool (access via c.s.%%SESSIONVARIABLES%% OR provider.ConnPool.s.%%SESSIONVARIABLES%%). + // sessStop is the owning session's stopChan (nil: never stops). Blocking waits + // for a connection give up when it is closed. + sessStop <-chan struct{} } +// errSessionStopped is returned by waits that gave up because the session stopped. +var errSessionStopped = errors.New("session stopped") + func NewConnPool(s *SESSION, provider *Provider, workerWGconnReady *sync.WaitGroup) { provider.mux.Lock() // NewConnPool mutex #d028 defer provider.mux.Unlock() // NewConnPool mutex #d028 @@ -129,6 +135,7 @@ func NewConnPool(s *SESSION, provider *Provider, workerWGconnReady *sync.WaitGro provider: provider, mux: &loggedrwmutex.LoggedSyncRWMutex{Name: fmt.Sprintf("ConnPool-%s", provider.Name)}, s: s, // session pointer to the session which created this ConnPool + sessStop: s.stopChan, } PoolsLock.Lock() ConnPools[provider.id] = provider.ConnPool @@ -432,7 +439,15 @@ getConnFromPool: waiting = time.Now() dlog(cfg.opt.DebugConnPool, "ConnPool GetConn: infinite waiting for a conn '%s'", c.provider.Name) - connitem = <-c.pool + select { + case connitem = <-c.pool: + case <-c.sessStop: + return nil, fmt.Errorf("error in ConnPool GetConn '%s': %w", c.provider.Name, errSessionStopped) + } + if connitem == nil { + // ParkConn never parks nil, but don't hand out a nil item + continue getConnFromPool + } GCounter.Incr("TOTAL_GetConns") dlog(cfg.opt.DebugConnPool, "ConnPool GetConn: released, got connid=%d age=(%v) %s: waited='%v'", connitem.connid, time.Since(connitem.parktime), c.provider.Name, time.Since(waiting)) return @@ -724,7 +739,17 @@ func SharedConnGet(sharedCC chan *ConnItem, provider *Provider) (connitem *ConnI // if any routine in this worker gets a nil connitem from sharedCC: // - it will get a new connection from the provider // - and put the conn back into the sharedCC channel later - aconnitem := <-sharedCC + var aconnitem *ConnItem + select { + case aconnitem = <-sharedCC: // a waiting token wins over a closed stop + default: + select { + case aconnitem = <-sharedCC: + case <-provider.ConnPool.sessStop: + // we took no token, so there is nothing to put back + return nil, fmt.Errorf("error in SharedConnGet '%s': %w", provider.Name, errSessionStopped) + } + } switch aconnitem { case nil: // no connection available @@ -740,7 +765,7 @@ func SharedConnGet(sharedCC chan *ConnItem, provider *Provider) (connitem *ConnI if !needNewConn && aconnitem != nil && aconnitem.conn != nil { // we have a valid connection, use it dlog(cfg.opt.DebugConnPool, "SharedConnGet: got shared connection '%s' aconnitem='%#v'", provider.Name, aconnitem) - provider.ConnPool.ExtendConn(connitem) + provider.ConnPool.ExtendConn(aconnitem) // was ExtendConn(connitem), the still-nil named return return aconnitem, nil } //provider.ConnPool.CloseConn(aconnitem, sharedCC) // close the nil connection item to reduce counter if needed and put nil back into sharedCC diff --git a/MemLimit.go b/MemLimit.go index 696ae8e..3d5b360 100644 --- a/MemLimit.go +++ b/MemLimit.go @@ -71,53 +71,66 @@ func (m *MemLimiter) MemAvail() (retbool bool) { return } -func (m *MemLimiter) MemLockWait(item *segmentChanItem, who string) { +// MemLockWait takes a memory slot for item. It waits until a slot is free or +// until stop is closed (a nil stop waits forever). It returns false if it gave +// up because of stop: the item then holds no slot and must not be downloaded. +// It returns true if the item holds a slot (also when it already held one). +func (m *MemLimiter) MemLockWait(item *segmentChanItem, who string, stop <-chan struct{}) bool { item.mux.Lock() if item.memlocked > 0 { item.mux.Unlock() dlog(always, "MemLockWait called on already memlocked item seg.Id='%s' who='%s'", item.segment.Id, who) - return + return true } item.mux.Unlock() GCounter.Incr("MemLockWait") defer GCounter.Decr("MemLockWait") - defer GCounter.Incr("TOTAL_MemLock") m.mux.Lock() if m.memdata[item] { dlog(always, "ERROR ! MemLimit tried to lock an item already in mem! seg.Id='%s' who='%s'", item.segment.Id, who) os.Exit(1) // this is a bug! we should never try to lock an item already in mem! - return } m.memdata[item] = true // flag as in mem - //if cfg.opt.Debug && m.waiting > 0 { dlog(cfg.opt.DebugMemlim, "MemLockWait avail=%d/%d m.waiting=%d who='%s'", len(m.memchan), m.mem_max, m.waiting, who) - //} m.waiting++ m.mux.Unlock() - /* - for { - m.mux.Lock() - if !m.memdata[item] { - m.memdata[item] = true // flag as in mem - m.mux.Unlock() - break - } - m.mux.Unlock() - time.Sleep(time.Second) // infinite wait for memlim. if this trigger something is wrong! - - } // end for waithere - */ - <-m.memchan // infinite wait to get a slot from chan + + // take a free slot right away if there is one, so a closed stop never wins over a free slot + got := false + select { + case <-m.memchan: + got = true + default: + select { + case <-m.memchan: + got = true + case <-stop: + } + } + + if !got { + // roll back, or the next MemLockWait on this item hits the os.Exit above + m.mux.Lock() + m.waiting-- + delete(m.memdata, item) + m.mux.Unlock() + GCounter.Incr("TOTAL_MemLockAborted") + dlog(cfg.opt.DebugMemlim, "MemLockWait aborted by stop seg.Id='%s' who='%s'", item.segment.Id, who) + return false + } + item.mux.Lock() item.memlocked++ item.mux.Unlock() + GCounter.Incr("TOTAL_MemLock") m.mux.Lock() m.waiting-- dlog(cfg.opt.DebugMemlim, "NewMemLock gotSLOT seg.Id='%s' m.waiting=%d who='%s'", item.segment.Id, m.waiting, who) m.mux.Unlock() + return true } // end func memlim.MemLockWait // MemReturn removes the item from mem and returns the slot to the chan diff --git a/MemLimit_test.go b/MemLimit_test.go new file mode 100644 index 0000000..0d1226a --- /dev/null +++ b/MemLimit_test.go @@ -0,0 +1,64 @@ +package main + +import ( + gotesting "testing" // Workers.go declares a const named testing + "time" + + "github.com/Tensai75/nzbparser" + "github.com/go-while/go-loggedrwmutex" +) + +func testItem(id string) *segmentChanItem { + return &segmentChanItem{ + mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "TEST-ITEM-" + id}, + segment: &nzbparser.NzbSegment{Id: id}, + } +} + +// Plan 3.6: MemLockWait used to block on <-memchan with no way out. A closed +// stop now aborts the wait and rolls back, so the item can be locked again later +// (the old code left memdata set and the next MemLockWait called os.Exit(1)). +func TestMemLockWaitAbortsOnStopAndRollsBack(t *gotesting.T) { + testCfg() + if GCounter == nil { + GCounter = NewCounter(10) + } + m := NewMemLimiter(1) + holder, waiter := testItem("holder"), testItem("waiter") + if !m.MemLockWait(holder, "test", nil) { + t.Fatal("first lock failed with a free slot") + } + + stop := make(chan struct{}) + res := make(chan bool, 1) + go func() { res <- m.MemLockWait(waiter, "test", stop) }() + select { + case <-res: + t.Fatal("MemLockWait returned while no slot was free") + case <-time.After(200 * time.Millisecond): + } + close(stop) + select { + case ok := <-res: + if ok { + t.Fatal("aborted wait reported a slot") + } + case <-time.After(2 * time.Second): + t.Fatal("MemLockWait did not return after stop was closed") + } + m.mux.RLock() + waiting, inData := m.waiting, m.memdata[waiter] + m.mux.RUnlock() + if waiting != 0 || inData || waiter.memlocked != 0 { + t.Fatalf("no rollback: waiting=%d memdata=%t memlocked=%d", waiting, inData, waiter.memlocked) + } + + // a free slot wins over a closed stop + m.MemReturn("test", holder) + if !m.MemLockWait(waiter, "test", stop) { + t.Fatal("closed stop beat a free slot") + } + if used, _ := m.Usage(); used != 1 { + t.Fatalf("used slots = %d, want 1", used) + } +} diff --git a/NetConn.go b/NetConn.go index edca7eb..4aad11c 100644 --- a/NetConn.go +++ b/NetConn.go @@ -167,11 +167,16 @@ func CMD_IHAVE(connitem *ConnItem, item *segmentChanItem) (int, string, uint64, connitem.srvtp.StartResponse(id) code, msg, err := connitem.srvtp.ReadCodeLine(335) - if err != nil { + // EndResponse must run on every path: textproto serves responses in order, + // so a skipped EndResponse makes the next command on this conn block forever + // in StartResponse (seen when a server answers 436 while shutting down). + connitem.srvtp.EndResponse(id) + if err != nil && code == 0 { + // read error, no reply line dlog(always, "ERROR CMD_IHAVE srvtp.ReadCodeLine @ '%s' err='%v'", connitem.c.provider.Name, err) return code, msg, 0, err } - connitem.srvtp.EndResponse(id) + // a reply other than 335 (435, 436, ...) also sets err; the switch below handles it switch code { case 335: diff --git a/NetConn_test.go b/NetConn_test.go new file mode 100644 index 0000000..1aefac7 --- /dev/null +++ b/NetConn_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "bufio" + "net" + "net/textproto" + "strings" + gotesting "testing" // Workers.go declares a const named testing + "time" + + "github.com/go-while/go-loggedrwmutex" +) + +// CMD_IHAVE used to return on a non-335 reply without srvtp.EndResponse, so +// the next command on the same connection blocked forever in StartResponse. +// Seen live: a server shutting down answers 436, the upload is retried on the +// same conn, and the session never ends. +func TestIHAVEReplyNot335KeepsConnUsable(t *gotesting.T) { + testCfg() + cli, srv := net.Pipe() + defer cli.Close() + defer srv.Close() + + go func() { + r := bufio.NewReader(srv) + for _, reply := range []string{"436 Retry later", "435 Not wanted"} { + line, err := r.ReadString('\n') + if err != nil || !strings.HasPrefix(line, "IHAVE ") { + return + } + if _, err := srv.Write([]byte(reply + "\r\n")); err != nil { + return + } + } + }() + + prov := &Provider{Name: "fake", mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "fake"}} + connitem := &ConnItem{ + c: &ConnPool{provider: prov}, + conn: cli, + srvtp: textproto.NewConn(cli), + writer: bufio.NewWriter(cli), + } + item := testItem("") + + for _, want := range []int{436, 435} { + done := make(chan int, 1) + go func() { + code, _, _, _ := CMD_IHAVE(connitem, item) + done <- code + }() + select { + case code := <-done: + if code != want { + t.Fatalf("CMD_IHAVE code=%d, want %d", code, want) + } + case <-time.After(3 * time.Second): + t.Fatalf("CMD_IHAVE expecting %d blocked: previous response was never ended", want) + } + } +} diff --git a/Routines.go b/Routines.go index d0718f4..e5eb175 100644 --- a/Routines.go +++ b/Routines.go @@ -331,7 +331,9 @@ func (s *SESSION) GoReupsRoutine(wid int, provider *Provider, item *segmentChanI if !memlocked { dlog(cfg.opt.DebugWorker, "GoReupsRoutine: item not memlocked seg.Id='%s' @ '%s'#'%s'", item.segment.Id, provider.Name, provider.Group) - memlim.MemLockWait(item, "GoUR") // gets memlock here + if !memlim.MemLockWait(item, "GoUR", s.stopChan) { // gets memlock here + return 0, fmt.Errorf("error in GoReupsRoutine: waiting for a mem slot seg.Id='%s' @ '%s': %w", item.segment.Id, provider.Name, errSessionStopped) + } } var err error diff --git a/Session.go b/Session.go index ce802f2..294ab89 100644 --- a/Session.go +++ b/Session.go @@ -400,6 +400,7 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s waitWorker.Wait() dlog(always, "sess: released waitWorker.Wait(), waiting on waitPool.Wait()") waitPool.Wait() + s.releaseMemSlots() // no worker is left; don't let leaked slots shrink memlim for the next session dlog(always, "sess: released waitPool.Wait() closing all provider connections") for _, provider := range s.providerList { KillConnPool(provider) // close the connection pool for this provider diff --git a/Workers.go b/Workers.go index d539adc..a8e25d1 100644 --- a/Workers.go +++ b/Workers.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "log" "math/rand" @@ -306,7 +307,15 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro start := time.Now() who := fmt.Sprintf("DR=%d@'%s'#'%s' seg.Id='%s'", wid, provider.Name, provider.Group, item.segment.Id) - memlim.MemLockWait(item, who) // gets memlock here + if !memlim.MemLockWait(item, who, s.stopChan) { // gets memlock here + // session is stopping: give the item back without downloading it + item.mux.Lock() + item.flaginDL = false + item.mux.Unlock() + DecreaseDLQueueCnt() + dlog(cfg.opt.DebugWorker, "GoDownsRoutine: (%d@'%s') stop while waiting for a mem slot, skip seg.Id='%s'", wid, provider.Name, item.segment.Id) + continue forGoDownsRoutine + } dlog(cfg.opt.DebugWorker && cfg.opt.DebugMemlim, "GoDownsRoutine got MemCheckWait who='%s' waited=(%d ms)", who, time.Since(start).Milliseconds()) errStr := "" StartDowns := time.Now() @@ -402,7 +411,9 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro if err != nil || code == 0 { errStr := fmt.Sprintf("ERROR in GoReupsRoutine code='%d' err='%v' no retry", code, err) dlog(always, "%s", errStr) - memlim.MemReturn("MemRetOnERR: "+errStr, item) // memfree GoReupsRoutine on error + if !errors.Is(err, errSessionStopped) { // a stopped wait took no slot + memlim.MemReturn("MemRetOnERR: "+errStr, item) // memfree GoReupsRoutine on error + } continue forGoReupsRoutine } speedInKBytes := (float64(item.size) / 1024) / float64(time.Since(StartReUps).Seconds()) @@ -494,6 +505,58 @@ func (item *segmentChanItem) FlagDLFailed(providerList []*Provider, providerGrou } } +// hasUploadTarget reports whether some provider the item is missing on could +// still take an upload for it (same rules as pushUP; a pending retry counts). +// The caller holds item.mux (read or write). +func (s *SESSION) hasUploadTarget(item *segmentChanItem) bool { + for pid, miss := range item.missingOn { + if !miss { + continue + } + p := s.providerList[pid] + if p.NoUpload || (!p.capabilities.ihave && !p.capabilities.post) { + continue + } + if item.ignoreUlOn[pid] || item.uploadedTo[pid] || item.availableOn[pid] || + item.unwantedOn[pid] || item.dmcaOn[pid] || item.errorOn[pid] { + continue + } + return true + } + return false +} // end func hasUploadTarget + +// retireNoUploadTarget returns the mem slot of an item that will never be uploaded, +// and drops its article when nothing else needs it (cache written or off, no yenc output). +func (s *SESSION) retireNoUploadTarget(item *segmentChanItem) { + item.mux.Lock() + locked := item.memlocked > 0 + if locked && !cfg.opt.YencWrite && !item.flaginYenc && (!cacheON || item.flagCache) { + item.article = nil // free memory + } + item.mux.Unlock() + if locked { + memlim.MemReturn("DV:noUploadTarget", item) + } +} // end func retireNoUploadTarget + +// releaseMemSlots returns the mem slots still held by this session's items. +// memlim is global: a slot leaked by one session would be lost for every later one. +// Call only after all workers have exited. +func (s *SESSION) releaseMemSlots() { + released := 0 + for _, item := range s.segmentList { + item.mux.RLock() + locked := item.memlocked > 0 + item.mux.RUnlock() + if locked { + memlim.MemReturn("session end", item) + released++ + } + } + dlog(released > 0, "releaseMemSlots: returned %d mem slots still held at session end", released) +} // end func releaseMemSlots + // matchThisUP checks if the item is a candidate for upload // it returns true if the item is a candidate for upload func matchThisUP(item *segmentChanItem) bool { @@ -766,6 +829,7 @@ forever: // uint64 var segm, allOk, done, fails, dead, isdl, indl, inup, isup, checked, dmca, nodl, noup, cached, inretry, inyenc, isyenc, dlQ, upQ, yeQ uint64 + var notarget uint64 // checked items no provider can take an upload for (retired) // Tnodl, Tnoup counts segments that not have been downloaded // or uploaded because provider has config flat NoDownload or NoUpload set @@ -848,6 +912,15 @@ forever: item.mux.RUnlock() // RUNLOCKS HERE #824d continue forsegmentList } + if allowUp && !s.hasUploadTarget(item) && (item.flagisDL || item.flagCache || !(cacheON || cfg.opt.YencWrite)) { + // missing only where nothing can upload it (NoUpload, no IHAVE/POST, errors, ...): + // no upload will ever return its mem slot. Retire it: give the slot back, + // don't download it, and count it as finished. + notarget++ + item.mux.RUnlock() // RUNLOCKS HERE #824d + s.retireNoUploadTarget(item) + continue forsegmentList + } item.mux.RUnlock() // RUNLOCKS HERE #824d pushedUp, nNoUp, nInRetry, a1err := s.pushUP(allowUp, item) @@ -1061,6 +1134,8 @@ forever: closeCase5 := (dead+isup == todo || dead+isup+fails == todo) closeCase6 := (dead+done == todo || dead+done+fails == todo) closeCase7 := false // placeholder + // items that can't be uploaded anywhere are finished too + closeCase8 := notarget > 0 && (dead+done+notarget == todo || dead+done+fails+notarget == todo) globalmux.RUnlock() if closeCase0 { @@ -1095,6 +1170,10 @@ forever: closeWait-- closeCase = closeCase + "|Debug#7" + } else if closeCase8 { + closeWait-- + closeCase = closeCase + "|Debug#8" + } else { if closeWait >= 10 { errmsg := fmt.Sprintf("WARN [DV] hit impossible closeCase ... kill in %d more loops", 15-closeWait) diff --git a/review-the-codebase-for-jolly-petal.md b/review-the-codebase-for-jolly-petal.md index 8ab809f..9592dcc 100644 --- a/review-the-codebase-for-jolly-petal.md +++ b/review-the-codebase-for-jolly-petal.md @@ -101,6 +101,15 @@ This is by design: the slot is held from download until upload so that RAM betwe **Fix (corrected):** do **not** add a plain `MemReturn` on every 220/920 success; that removes the RAM limit for upload runs. Instead, release the slot on download success only when `!allowUp`, and release it wherever the divider retires an item without queueing an upload. For the blocking receive, see 3.6: `MemLockWait` needs a stop parameter and a rollback. +**Result (2026-10-17):** fixed. **Trigger corrected again:** a run with no upload providers never downloads anything. When `!allowUp`, `pushUP` returns an error and the divider skips `pushDL`, so the `!allowUp` release isn't needed. The real trigger is an item the divider downloads although nothing can upload it: it is missing only on a `NoUpload` provider (or one with errors, no IHAVE/POST, ...). +- Reproduced: B has all 90 articles, A (`NoUpload`) has every second one. The old build downloaded 16 of 45, hit `MEM:16/16`, and hung in `waitWorker.Wait()` until killed. +- **Fix:** + - The divider now checks `hasUploadTarget(item)` (same rules as `pushUP`) for each checked item that isn't in flight. Items with no target are retired: their slot is returned, and the article is dropped when neither the cache nor `-yencout` needs it. They are not downloaded, and they count toward a new close case `#8` (`dead+done+notarget == todo`). + - `releaseMemSlots()` returns any slot still held at session end, because `memlim` is global and a leak would shrink it for every later web job. +- **Result:** the run above finishes in 6 s with nothing downloaded. +- **Side effect, a bug fixed (new):** every earlier "normal" A→B run ended through "hit impossible closeCase ... force quit" after about 20 s of waiting. Articles that only A (`NoUpload`) lacked never counted as done. With `#8` the same A→B run takes 10 s instead of 28 s, with identical results (9 downloaded, 9 uploaded). +- **Not fixed (new, noted):** download-only runs (`-cd`/`-yencout` without any upload provider) never download, because `pushUP`'s error skips `pushDL`. The `MemLimit.go` header says that case was intended to work. + ### 2.5 `fatal error: concurrent map read and map write` on `s.segmentChans*` (corrected) Written under `globalmux` in each provider's boot goroutine (`Workers.go:65-81`). Read with no lock from `GoWorker` at `Workers.go:191/247/338` and `Routines.go:31`, which can overlap another group's boot write. That is a real, unrecoverable runtime throw. @@ -178,6 +187,15 @@ Written under `globalmux` in each provider's boot goroutine (`Workers.go:65-81`) - `StopRoutines` closes `stopChan` *before* `waitWorker.Wait()`, so running workers will start getting errors from `GetConn`. That's acceptable, but expect the log noise. - Read `stopChan` once into a local (see the Phase 3 note). +**Result (2026-10-17):** fixed. +- `MemLockWait(item, who, stop) bool` takes a free slot first, otherwise it waits for a slot or `stop`. On abort it rolls back `waiting`/`memdata` and returns false. `TOTAL_MemLock` now only counts successful locks; aborts go to `TOTAL_MemLockAborted`. + - The download routine gives the item back (clears `flaginDL`, decreases the queue count) and skips `GoDownsRoutine`. + - `GoReupsRoutine` returns an error wrapping `errSessionStopped`, and its caller then skips `MemReturn`. +- `GetConn`'s infinite wait and `SharedConnGet` select on the pool's `sessStop` (the owning session's `stopChan`, captured in `NewConnPool`). `SharedConnGet` prefers a waiting token over a closed stop. A nil item from the pool is no longer handed out. +- **Bug fixed (new):** `SharedConnGet` called `ExtendConn(connitem)` on the still-nil named return, so reused shared connections never had their deadline extended. It now extends `aconnitem`. +- Test: `TestMemLockWaitAbortsOnStopAndRollsBack`. +- Verified by stopping server B mid-upload. The session returns after 63 s with no races: 4 re-upload waits aborted, 2 leaked slots were returned at session end. That needed 3.15 as well. + ### 3.7 Goroutine leaks (corrected) | Site | Status | @@ -240,6 +258,18 @@ Keyed by a per-session index (`Utils.go:200`, `ConnPool.go:134/181`), so session - Verified with `-race`, CLI download A → B: clean before and after. - Verified with `-race`, web mode (2 jobs, `/api/jobs` + `/api/stats` polled every 50 ms): the old build logged 29 races including `Session.go:272` vs `webserver.go:768`, then wedged (below). The new build completes both jobs. The remaining 20–26 races are all inside `webserver.go` job-field handling (4.2). +### 3.15 (new) `CMD_IHAVE` leaves the textproto pipeline stuck: the connection hangs forever + +`NetConn.go` `CMD_IHAVE` returned on any `ReadCodeLine(335)` error **before** `EndResponse(id)`. `ReadCodeLine` returns an error for any reply other than 335, including normal `435`/`436`. The caller keeps the connection for `436` (retry) and `435`/unknown (unwanted). The next command on it then blocks forever in `StartResponse`, before any I/O, so no read deadline helps. + +**Found** by the 3.1/3.2 check "stop server B mid-run". nntp-server answers `436 Retry later` while shutting down, and the session hung in `waitWorker.Wait()`. The goroutine dump showed 4 re-upload routines in `textproto.(*sequencer).Start` via `NetConn.go:168`. A live provider answering 435/436 during a normal run wedges that connection the same way. + +**Fix:** `EndResponse` right after `ReadCodeLine`. A read error with no reply line still returns early; 435/436 now reach the existing switch. Every other `StartResponse` in the code already ends on all paths. **Note:** `textproto.Cmd` returns an `id` even when the write fails, so a conn whose `Cmd` failed must never be reused. The callers close it on `code == 0`. + +**Result (2026-10-17):** fixed. `TestIHAVEReplyNot335KeepsConnUsable` (fake server answers 436, then 435) blocks on the old code and passes now. + +**Also noted (not fixed):** `FlagError` (`Workers.go`) sets `retryIn = int64(time.Now().Add(5 * time.Second).Second())`. That is the seconds field of the current minute (0–59), not a Unix time, so `retryIn > now` is never true and the 5 s retry delay never applies. It should be `.Unix()`. + ### 3.14 Remaining unsynchronised accesses (corrected) Still races: From 68ba5d7fb5c2b2bb79f98669de31424668344057 Mon Sep 17 00:00:00 2001 From: go-while Date: Sun, 18 Oct 2026 00:46:11 +0000 Subject: [PATCH 126/152] Give up on dead check groups and down providers instead of hanging - Count live check routines per group; the feeder closes its channel and the divider quits ("checkersGone") once a group has none left. - A failed STAT is retried once, then recorded as an error on the group so the item still counts as checked; a check routine quits after 3 items without a connection. - Replace the -verbose-only watchdog (which killed progressing runs that had dead segments) with an always-on 3 min no-progress check that also counts failed attempts. - GetConn failures in the downs/reups routines now flag the item, so it no longer stays "in flight" forever; uploads give up a provider after 3 fails, and the FlagError retry delay works (.Unix(), retryOn set). - Failed uploads keep their mem slot: returning it deadlocked downloads and re-queued uploads on MemLockWait. - After the stop, queued items are dropped and GetConn no longer dials. - A provider whose pool gave up on dial errors is skipped for the rest of the session (never in proxy mode). Co-Authored-By: Claude Opus 5.5 --- Config.go | 6 +- ConnPool.go | 36 +++- Routines.go | 32 ++- Session.go | 5 +- Teardown_test.go | 51 +++++ Workers.go | 262 +++++++++++++++++++------ review-the-codebase-for-jolly-petal.md | 38 +++- 7 files changed, 361 insertions(+), 69 deletions(-) diff --git a/Config.go b/Config.go index 2162150..4b6d8e5 100644 --- a/Config.go +++ b/Config.go @@ -1,6 +1,7 @@ package main import ( + "sync/atomic" "time" "github.com/Tensai75/nzbparser" @@ -138,7 +139,10 @@ type Provider struct { ConnPool *ConnPool // the pool of connections for this provider //mux sync.RWMutex // mutex to protect the provider struct mux *loggedrwmutex.LoggedSyncRWMutex // debug mutex - id int // will be set in loadProviderList + // down is set when the conn pool gave up dialing (see markDown). Each session has + // its own provider copies, so it only lasts for the session. Never set in proxy mode. + down atomic.Bool + id int // will be set in loadProviderList // flags capabilities struct { check bool // if true, provider supports checking articles diff --git a/ConnPool.go b/ConnPool.go index 2eb5d9f..dd55abf 100644 --- a/ConnPool.go +++ b/ConnPool.go @@ -90,6 +90,22 @@ type ConnPool struct { // errSessionStopped is returned by waits that gave up because the session stopped. var errSessionStopped = errors.New("session stopped") +// errNoConn: a routine could not get a connection at all (GetConn already retried). +var errNoConn = errors.New("no connection") + +// errDial: the TCP/TLS dial itself failed (refused, timeout, unreachable), not the banner or auth. +var errDial = errors.New("dial failed") + +// markDown takes the provider out of this session: every routine and the divider skip it +// from now on, instead of paying GetConn's dial retries again for every item. +func (p *Provider) markDown(err error) { + if p.down.CompareAndSwap(false, true) { + dlog(always, "WARN provider '%s' is down for this session, skipping it from now on: %v", p.Name, err) + } +} + +func (p *Provider) isDown() bool { return p.down.Load() } + func NewConnPool(s *SESSION, provider *Provider, workerWGconnReady *sync.WaitGroup) { provider.mux.Lock() // NewConnPool mutex #d028 defer provider.mux.Unlock() // NewConnPool mutex #d028 @@ -213,10 +229,10 @@ func (c *ConnPool) connect() (connitem *ConnItem, err error) { dlog(cfg.opt.DebugConnPool, "ConnPool raw tcp.Dial took='%v' '%s'", time.Since(start), c.provider.Name) if err != nil { if isNetworkUnreachable(err) { - return nil, fmt.Errorf("error connect Unreachable network! '%s' @ '%s' err='%v'", c.provider.Host, c.provider.Name, err) + return nil, fmt.Errorf("error connect Unreachable network! '%s' @ '%s': %w: %v", c.provider.Host, c.provider.Name, errDial, err) } dlog(cfg.opt.DebugConnPool, "ERROR connect Dial '%s' no retry err='%v'", c.provider.Name, err) - return nil, err + return nil, fmt.Errorf("%w: %w", errDial, err) } case true: conf := &tls.Config{ @@ -229,10 +245,10 @@ func (c *ConnPool) connect() (connitem *ConnItem, err error) { dlog(cfg.opt.DebugConnPool, "ConnPool raw tls.Dial took='%v' '%s'", time.Since(start), c.provider.Name) if err != nil { if isNetworkUnreachable(err) { - return nil, fmt.Errorf("error connect Unreachable network! '%s' @ '%s' err='%v'", c.provider.Host, c.provider.Name, err) + return nil, fmt.Errorf("error connect Unreachable network! '%s' @ '%s': %w: %v", c.provider.Host, c.provider.Name, errDial, err) } dlog(cfg.opt.DebugConnPool, "ERROR connect Dial '%s' no-retry err='%v'", c.provider.Name, err) - return nil, err + return nil, fmt.Errorf("%w: %w", errDial, err) } } // end switch dlog(cfg.opt.DebugConnPool, "ConPool connect raw conn established '%s' took=(%d ms)", c.provider.Name, time.Since(start).Milliseconds()) @@ -335,7 +351,7 @@ func (c *ConnPool) NewConn() (connitem *ConnItem, err error) { c.openConns-- dlog(cfg.opt.DebugConnPool, "ERROR in ConnPool NewConn: connect failed '%s' openConns=%d connitem='%v' err='%v'", c.provider.Name, c.openConns, connitem, err) c.mux.Unlock() // mutex c460 - return nil, fmt.Errorf("error in ConnPool NewConn: connect failed '%s' err='%v'", c.provider.Name, err) + return nil, fmt.Errorf("error in ConnPool NewConn: connect failed '%s' err='%w'", c.provider.Name, err) } c.mux.RLock() // mutex c461 dlog(cfg.opt.DebugConnPool, "ConnPool NewConn: got new connid=%d '%s' openConns after=%d/%d before=%d connectTook=(%d ms) err='%v", connitem.connid, c.provider.Name, c.openConns, c.provider.MaxConns, openConnsBefore, time.Since(start).Milliseconds(), err) @@ -455,6 +471,12 @@ getConnFromPool: dlog(cfg.opt.DebugConnPool, "ConnPool GetConn: try to open a new connection '%s' openConns=%d provider.MaxConns=%d", c.provider.Name, c.openConns, c.provider.MaxConns) c.mux.RUnlock() // mutex c458 + select { + case <-c.sessStop: + // don't dial (and retry for seconds) for a session that is over + return nil, fmt.Errorf("error in ConnPool GetConn '%s': %w", c.provider.Name, errSessionStopped) + default: + } // try to open a new connection newconnitem, aerr := c.NewConn() if aerr != nil || newconnitem == nil || newconnitem.conn == nil { @@ -465,6 +487,10 @@ getConnFromPool: // we have retried too often dlog(cfg.opt.DebugConnPool, "ERROR in ConnPool GetConn: retried %d times to get a new conn '%s' giving up! aerr='%v'", retried, c.provider.Name, aerr) err = aerr + if errors.Is(aerr, errDial) && c.s != nil && !c.s.proxy { + // the proxy's pools live as long as the process: never mark them down + c.provider.markDown(aerr) + } break getConnFromPool // break out of the for loop } continue getConnFromPool // until chan rans empty diff --git a/Routines.go b/Routines.go index e5eb175..043e662 100644 --- a/Routines.go +++ b/Routines.go @@ -26,6 +26,9 @@ func (s *SESSION) GoCheckRoutine(wid int, provider *Provider, item *segmentChanI } GCounter.Incr("GoCheckRoutines") defer GCounter.Decr("GoCheckRoutines") + if provider.isDown() { + return 0, fmt.Errorf("GoCheckRoutine '%s' is down: %w", provider.Name, errNoConn) // checkItem marks it failed here + } dlog(cfg.opt.DebugCR, "GoWorker (%d) checking seg.Id='%s' @ '%s' remainingInChan=%d/%d", wid, item.segment.Id, provider.Name, len(s.segmentChansCheck[provider.Group]), cap(s.segmentChansCheck[provider.Group])) @@ -39,7 +42,7 @@ func (s *SESSION) GoCheckRoutine(wid int, provider *Provider, item *segmentChanI connitem, err = provider.ConnPool.GetConn() } if err != nil { - return 0, fmt.Errorf("error in GoCheckRoutine: ConnGet '%s' connitem='%v' sharedCC='%v' err='%v'", provider.Name, connitem, sharedCC, err) + return 0, fmt.Errorf("error in GoCheckRoutine: ConnGet '%s' connitem='%v' sharedCC='%v': %w: %v", provider.Name, connitem, sharedCC, errNoConn, err) } if connitem == nil || connitem.conn == nil { return 0, fmt.Errorf("error in GoCheckRoutine: ConnGet got nil item or conn '%s' connitem='%v' sharedCC='%v' err='%v'", provider.Name, connitem, sharedCC, err) @@ -148,6 +151,13 @@ func (s *SESSION) GoDownsRoutine(wid int, provider *Provider, item *segmentChanI } GCounter.Incr("GoDownsRoutines") defer GCounter.Decr("GoDownsRoutines") + if provider.isDown() { + item.FlagError(provider.id) // clears flaginDL, drops availableOn there + item.mux.Lock() + item.ignoreDlOn[provider.id] = true + item.mux.Unlock() + return 0, fmt.Errorf("GoDownsRoutine '%s' is down: %w", provider.Name, errNoConn) + } // check cache before download if cacheON && cache.ReadCache(item) > 0 { @@ -164,10 +174,12 @@ func (s *SESSION) GoDownsRoutine(wid int, provider *Provider, item *segmentChanI connitem, err = provider.ConnPool.GetConn() } if err != nil { - return 0, fmt.Errorf("error in GoDownsRoutine: ConnGet '%s' connitem='%v' sharedCC='%v' err='%v'", provider.Name, connitem, sharedCC, err) + item.FlagError(provider.id) // clears flaginDL: the divider would count it as in download forever + return 0, fmt.Errorf("error in GoDownsRoutine: ConnGet '%s' connitem='%v' sharedCC='%v': %w: %v", provider.Name, connitem, sharedCC, errNoConn, err) } if connitem == nil || connitem.conn == nil { - return 0, fmt.Errorf("error in GoDownsRoutine: ConnGet got nil item or conn '%s' connitem='%v' sharedCC='%v' err='%v'", provider.Name, connitem, sharedCC, err) + item.FlagError(provider.id) + return 0, fmt.Errorf("error in GoDownsRoutine: ConnGet got nil item or conn '%s' connitem='%v' sharedCC='%v': %w", provider.Name, connitem, sharedCC, errNoConn) } dlog(cfg.opt.DebugWorker, "GoDownsRoutine got connitem='%v' sharedCC='%v' --> CMD_ARTICLE seg.Id='%s'", connitem, sharedCC, item.segment.Id) @@ -323,6 +335,14 @@ func (s *SESSION) GoReupsRoutine(wid int, provider *Provider, item *segmentChanI } GCounter.Incr("GoReupsRoutines") defer GCounter.Decr("GoReupsRoutines") + if provider.isDown() { + // no upload there this session; the divider retires the item if it has no other target + item.mux.Lock() + item.errorOn[provider.id] = true + item.flaginUP = false + item.mux.Unlock() + return 0, fmt.Errorf("GoReupsRoutine '%s' is down: %w", provider.Name, errNoConn) + } //who := fmt.Sprintf("UR=%d@'%s' seg.Id='%s'", wid, provider.Name, item.segment.Id) // DISABLED MEMRETURN item.mux.Lock() @@ -344,10 +364,12 @@ func (s *SESSION) GoReupsRoutine(wid int, provider *Provider, item *segmentChanI connitem, err = provider.ConnPool.GetConn() } if err != nil { - return 0, fmt.Errorf("error in GoReupsRoutine: ConnGet '%s' connitem='%v' sharedCC='%v' err='%v'", provider.Name, connitem, sharedCC, err) + item.FlagError(provider.id) // clears flaginUP: the divider would count it as in upload forever + return 0, fmt.Errorf("error in GoReupsRoutine: ConnGet '%s' connitem='%v' sharedCC='%v': %w: %v", provider.Name, connitem, sharedCC, errNoConn, err) } if connitem == nil || connitem.conn == nil || connitem.srvtp == nil { - return 0, fmt.Errorf("error in GoReupsRoutine: ConnGet got nil item or conn '%s' connitem='%v' sharedCC='%v' err='%v'", provider.Name, connitem, sharedCC, err) + item.FlagError(provider.id) + return 0, fmt.Errorf("error in GoReupsRoutine: ConnGet got nil item or conn '%s' connitem='%v' sharedCC='%v': %w", provider.Name, connitem, sharedCC, errNoConn) } var uploaded, unwanted, retry bool diff --git a/Session.go b/Session.go index 294ab89..f2d755a 100644 --- a/Session.go +++ b/Session.go @@ -9,6 +9,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "time" "github.com/Tensai75/nzbparser" @@ -70,7 +71,9 @@ type SESSION struct { segmentChansCheck map[string]chan *segmentChanItem // map[provider.Group]chan segmentChansDowns map[string]chan *segmentChanItem // map[provider.Group]chan segmentChansReups map[string]chan *segmentChanItem // map[provider.Group]chan - //memDL map[string]map[*string]*segmentChanItem // (map[provider.group]map[ptr to segment.Id]ptr to Item) --- with -checkfirst queues items here. map will be create in GoWorkDivider() + checkers map[string]*checkerGroup // live check routines per provider.Group, built with the channels + workDone atomic.Uint64 // items finished by any check/downs/reups routine, ok or not (divider watchdog) + //memDL map[string]map[*string]*segmentChanItem // (map[provider.group]map[ptr to segment.Id]ptr to Item) --- with -checkfirst queues items here. map will be create in GoWorkDivider() //memUP map[string]map[*string]*segmentChanItem // TODO: (map[provider.group]map[ptr to segment.Id]ptr to Item) --- with -uploadlater process uploads after downloads (possible only with cacheON) map will be create in GoWorkDivider() preBoot bool // will be set to false before workers start active bool // will be set to true when processing starts // REVIEW!TODO!FIXME: set to false is missing!! diff --git a/Teardown_test.go b/Teardown_test.go index 4f595a1..857fdd6 100644 --- a/Teardown_test.go +++ b/Teardown_test.go @@ -104,3 +104,54 @@ func TestStopRoutinesWithoutChannels(t *gotesting.T) { t.Fatal("nil stopChan reported as stopped") } } + +// Plan 3.2: when every check routine of a group is gone (here: none ever +// started, like a provider that can't connect at boot) the feeder used to block +// on a full channel, or spin on len(ch) > 0, forever. It now closes the channel. +func TestFeederClosesWhenCheckersGone(t *gotesting.T) { + testCfg() + cfg.opt.ChanSize = 2 + s := &SESSION{ + mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "TEST-SESSION"}, + stopChan: make(chan struct{}), + providerList: []*Provider{ + {Name: "a", Group: "a", Enabled: true, MaxConns: 2, mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "a"}}, + {Name: "b", Group: "b", Enabled: true, MaxConns: 3, mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "b"}}, + }, + } + for i := 0; i < 10; i++ { // more items than the channel holds: the feeder blocks on send + s.segmentList = append(s.segmentList, testItem(string(rune('a'+i)))) + } + s.makeSegmentChans() + if got := s.checkers["b"].left.Load(); got != 3 { + t.Fatalf("group b: want 3 check routines, got %d", got) + } + chB := s.segmentChansCheck["b"] + time.Sleep(100 * time.Millisecond) + if s.allCheckersGone() { + t.Fatal("allCheckersGone before any routine exited") + } + + s.checkersExit(s.providerList[1], 3) // b: boot failed, none of its 3 check routines start + deadline := time.After(3 * time.Second) + for closed := false; !closed; { + select { + case _, ok := <-chB: + closed = !ok + case <-deadline: + t.Fatal("feeder of group b did not close its channel after its checkers were gone") + } + } + if s.allCheckersGone() { + t.Fatal("group a still has check routines") + } + s.checkersExit(s.providerList[0], 1) + s.checkersExit(s.providerList[0], 1) + s.checkersExit(s.providerList[0], 1) // over-counting must not panic (close once) + if !s.allCheckersGone() { + t.Fatal("allCheckersGone false after every group exited") + } + if !closedWithin(s.segmentChansCheck["a"], 3*time.Second) { + t.Fatal("feeder of group a did not close its channel") + } +} diff --git a/Workers.go b/Workers.go index a8e25d1..5b8a3ff 100644 --- a/Workers.go +++ b/Workers.go @@ -6,6 +6,7 @@ import ( "log" "math/rand" "sync" + "sync/atomic" "time" ) @@ -72,6 +73,7 @@ func (s *SESSION) GoBootWorkers(waitDivider *sync.WaitGroup, workerWGconnReady * if err != nil { dlog(always, "ERROR Boot Provider '%s' err='%v'", provider.Name, err) // Release waitWorker and waitPool counters for all workers that won't be started + s.checkersExit(provider, provider.MaxConns) // their check routines never start for i := 0; i < provider.MaxConns; i++ { waitWorker.Done() // 3 times per worker waitWorker.Done() @@ -129,10 +131,15 @@ func (s *SESSION) makeSegmentChans() { check := make(map[string]chan *segmentChanItem) downs := make(map[string]chan *segmentChanItem) reups := make(map[string]chan *segmentChanItem) + checkers := make(map[string]*checkerGroup) for _, provider := range s.providerList { if !provider.Enabled || provider.MaxConns <= 0 { continue } + if checkers[provider.Group] == nil { + checkers[provider.Group] = &checkerGroup{gone: make(chan struct{})} + } + checkers[provider.Group].left.Add(int64(provider.MaxConns)) // one check routine per conn if check[provider.Group] != nil { continue } @@ -145,26 +152,87 @@ func (s *SESSION) makeSegmentChans() { s.segmentChansCheck = check s.segmentChansDowns = downs s.segmentChansReups = reups + s.checkers = checkers s.mux.Unlock() for group, ch := range check { - go s.feedCheckChan(group, ch) // the feeder owns ch and is the only one closing it + go s.feedCheckChan(group, ch, checkers[group].gone) // the feeder owns ch and is the only one closing it } } // end func makeSegmentChans +// checkerGroup counts the check routines of one provider group that have not exited yet, +// including the ones that will never start because their worker failed to connect. +// gone is closed when the count reaches 0: nothing will read the group's check channel again. +type checkerGroup struct { + left atomic.Int64 + gone chan struct{} + goneOnce sync.Once +} + +// exit marks n check routines of the group as finished (or never started). +func (g *checkerGroup) exit(n int) { + if g == nil || n <= 0 { + return + } + if g.left.Add(-int64(n)) <= 0 { + g.goneOnce.Do(func() { close(g.gone) }) + } +} + +// checkersExit is checkerGroup.exit for a provider's group. The map is read-only once published. +func (s *SESSION) checkersExit(provider *Provider, n int) { + s.checkers[provider.Group].exit(n) +} + +// allCheckersGone reports whether every check routine of every group has exited: +// from then on no item's checkedOn changes anymore. +func (s *SESSION) allCheckersGone() bool { + for _, g := range s.checkers { + select { + case <-g.gone: + default: + return false + } + } + return len(s.checkers) > 0 +} + // feedCheckChan pushes every segment of the session into the check channel of one group, // waits for the check routines to empty it, then closes it. -func (s *SESSION) feedCheckChan(group string, segmentChanCheck chan *segmentChanItem) { +// If all check routines of the group are gone (none could connect, or all died) it stops +// feeding and closes the channel right away: nobody will read it anymore. +func (s *SESSION) feedCheckChan(group string, segmentChanCheck chan *segmentChanItem, checkersGone <-chan struct{}) { start := time.Now() + fed := 0 + abandoned := func() { + dlog(always, "WARN feeder group '%s': all check routines are gone, %d/%d items fed, %d left in chan never checked here", + group, fed, len(s.segmentList), len(segmentChanCheck)) + } +feed: for _, item := range s.segmentList { - segmentChanCheck <- item // is allowed to block inside feeder routine + select { + case segmentChanCheck <- item: // is allowed to block inside feeder routine + fed++ + case <-checkersGone: + abandoned() + break feed + } } - for { - time.Sleep(time.Second) // wait for check routine to empty out the chan - if len(segmentChanCheck) == 0 { - break + if fed == len(s.segmentList) { + tick := time.NewTicker(time.Second) + waitEmpty: + for len(segmentChanCheck) > 0 { // wait for the check routines to empty out the chan + select { + case <-tick.C: + case <-checkersGone: + if len(segmentChanCheck) > 0 { + abandoned() + } + break waitEmpty + } } + tick.Stop() } - dlog(cfg.opt.Verbose, " | Done feeding items=%d -> segmentChanCheck Group '%s' took='%.0f sec'", len(s.segmentList), group, time.Since(start).Seconds()) + dlog(cfg.opt.Verbose, " | Done feeding items=%d/%d -> segmentChanCheck Group '%s' took='%.0f sec'", fed, len(s.segmentList), group, time.Since(start).Seconds()) s.mux.Lock() // set checkFeedDone only signals that feeding to the channel is done // the check routine may still be activly checking! @@ -192,6 +260,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro waitWorker.Done() // release this GoWorker from the pool waitPool.Done() + s.checkersExit(provider, 1) // its check routine never starts dlog(always, "ERROR GoWorker (%d) failed to get shared conn for '%s' err='%v' close worker", wid, provider.Name, err) return } @@ -203,8 +272,10 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro segCC := s.segmentChansCheck[provider.Group] go func(wid int, provider *Provider, waitWorker *sync.WaitGroup, thisWorkerWG *sync.WaitGroup, sharedCC chan *ConnItem, segCC chan *segmentChanItem) { var processed uint64 + noConn := 0 // items in a row that got no connection defer waitWorker.Done() defer thisWorkerWG.Done() + defer s.checkersExit(provider, 1) timeOut := time.Now().Add(15 * time.Second) lastGood := time.Now() toChan := make(chan struct{}, 1) @@ -240,15 +311,27 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro lastGood = time.Now() switch cfg.opt.ByPassSTAT { case false: - code, err := s.GoCheckRoutine(wid, provider, item, sharedCC) + code, err := s.checkItem(wid, provider, item, sharedCC) item.PrintItemFlags(cfg.opt.DebugFlags, true, fmt.Sprintf("post-GoCheckRoutine: code=%d", code)) - if err != nil { // re-queue? + if err != nil && !provider.isDown() { // recorded as error on this group by checkItem dlog(always, "ERROR in GoCheckRoutine err='%v'", err) } + if errors.Is(err, errNoConn) && !provider.isDown() { // down: checkItem fails items at once, keep draining + if noConn++; noConn >= maxCheckNoConn { + dlog(always, "WARN CheckRoutine (%d@'%s'): no connection for %d items in a row, quitting", wid, provider.Name, noConn) + break forGoCheckRoutine + } + } else { + noConn = 0 + } case true: log.Fatal("you should not be here! Quitting...") // FIXME TODO: remove this fatal error } processed++ + s.workDone.Add(1) + // the idle timeout counts from now: one slow item (connect retries) must + // not make the routine quit while items are waiting in the channel + timeOut = time.Now().Add(15 * time.Second) continue forGoCheckRoutine } } // end forGoCheckRoutine @@ -301,6 +384,14 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro segCD <- nil // refill the nil so others will die too break forGoDownsRoutine } + if s.stopped() { + // the divider quit: drop what is still queued instead of working it + item.mux.Lock() + item.flaginDL = false + item.mux.Unlock() + DecreaseDLQueueCnt() + continue forGoDownsRoutine + } timeOut = time.Now().Add(15 * time.Second) lastGood = time.Now() dlog(cfg.opt.DebugWorker, "GoDownsRoutine: (%d@'%s') start seg.Id='%s'", wid, provider.Name, item.segment.Id) @@ -322,6 +413,7 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro code, err := s.GoDownsRoutine(wid, provider, item, sharedCC) item.PrintItemFlags(cfg.opt.DebugFlags, true, fmt.Sprintf("post-GoDownsRoutine: code=%d", code)) processed++ + s.workDone.Add(1) DecreaseDLQueueCnt() if err != nil || (code != 220 && code != 920) { if code != 430 { @@ -401,19 +493,30 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro segCR <- nil // refill the nil so others will die too break forGoReupsRoutine } + if s.stopped() { + // the divider quit: drop what is still queued (its mem slot goes back in releaseMemSlots) + item.mux.Lock() + item.flaginUP = false + item.mux.Unlock() + DecreaseUPQueueCnt() + continue forGoReupsRoutine + } timeOut = time.Now().Add(15 * time.Second) lastGood = time.Now() StartReUps := lastGood code, err := s.GoReupsRoutine(wid, provider, item, sharedCC) item.PrintItemFlags(cfg.opt.DebugFlags, true, fmt.Sprintf("post-GoReupsRoutine: code=%d", code)) processed++ + s.workDone.Add(1) DecreaseUPQueueCnt() if err != nil || code == 0 { errStr := fmt.Sprintf("ERROR in GoReupsRoutine code='%d' err='%v' no retry", code, err) - dlog(always, "%s", errStr) - if !errors.Is(err, errSessionStopped) { // a stopped wait took no slot - memlim.MemReturn("MemRetOnERR: "+errStr, item) // memfree GoReupsRoutine on error - } + dlog(!provider.isDown() || cfg.opt.DebugUR, "%s", errStr) + // Keep the mem slot: the item was flagged and is either re-pushed (retry, another + // group) or retired by the divider (retireNoUploadTarget returns the slot). + // Returning it here let downloads take every slot while the item, queued again + // without one, waited in MemLockWait behind items that held them: deadlock. + // Whatever is still held at session end goes back in releaseMemSlots. continue forGoReupsRoutine } speedInKBytes := (float64(item.size) / 1024) / float64(time.Since(StartReUps).Seconds()) @@ -454,6 +557,51 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro // matchThisDL checks if the item is a candidate for download // it returns true if the item is a candidate for download +const ( + // checkAttempts: a STAT whose connection broke is tried this often (each on a fresh conn). + // No retry if no connection could be had at all: GetConn already retried connecting. + checkAttempts = 2 + // maxCheckNoConn: a check routine quits after this many items in a row got no connection. + // When all of a group's check routines are gone, the feeder and the divider give up on it. + maxCheckNoConn = 3 +) + +// checkItem runs GoCheckRoutine and retries a check whose connection broke. A check that +// still fails is recorded as an error on the provider's group, so the item still +// counts as checked there; otherwise the divider waits forever for checked == todo. +func (s *SESSION) checkItem(wid int, provider *Provider, item *segmentChanItem, sharedCC chan *ConnItem) (code int, err error) { + for attempt := 1; ; attempt++ { + code, err = s.GoCheckRoutine(wid, provider, item, sharedCC) + if code != 0 || err == nil { + return code, err + } + if errors.Is(err, errNoConn) || attempt >= checkAttempts || s.stopped() { + break + } + dlog(always, "WARN check seg.Id='%s' @ '%s' attempt %d/%d failed, retry: %v", item.segment.Id, provider.Name, attempt, checkAttempts, err) + } + s.markCheckFailed(item, provider) + return code, err +} // end func checkItem + +// markCheckFailed records a check that could not be done on the provider's group: +// the item counts as checked there, but its state there is unknown, so it is neither +// downloaded from nor uploaded to that group (errorOn, ignoreDlOn; not missingOn). +func (s *SESSION) markCheckFailed(item *segmentChanItem, provider *Provider) { + item.mux.Lock() + for id, prov := range s.providerList { + if prov.Group != provider.Group { + continue + } + item.checkedOn++ + item.errorOn[id] = true + item.ignoreDlOn[id] = true + } + item.mux.Unlock() + // a down provider was logged once by markDown + dlog(!provider.isDown() || cfg.opt.DebugCR, "WARN check seg.Id='%s' failed on group '%s', marked as error there", item.segment.Id, provider.Group) +} // end func markCheckFailed + func matchThisDL(item *segmentChanItem) bool { // item is a candidate for download if it has no size, is not in DL, not in UP, and not in DLMEM switch cacheON { @@ -471,6 +619,9 @@ func matchThisDL(item *segmentChanItem) bool { return (!item.flaginDL && !item.flagisDL && !item.flaginUP && !item.flagisUP && !item.flaginDLMEM) } // end func matchThisDL +// maxItemFails: after this many errors an upload target is given up for the item. +const maxItemFails = 3 + func (item *segmentChanItem) FlagError(providerId int) { log.Printf("FlagError called for providerId=%d on segment.Id='%s' waiting to lock item mutex", providerId, item.segment.Id) item.mux.Lock() @@ -478,15 +629,24 @@ func (item *segmentChanItem) FlagError(providerId int) { //item.ignoreDlOn[providerId] = true //item.ignoreUlOn[providerId] = true //item.errorOn[providerId] = true + wasUP := item.flaginUP if item.flaginUP { - if item.fails < 3 { - item.retryIn = int64(time.Now().Add(5 * time.Second).Second()) + if item.fails < maxItemFails-1 { + // retry this provider in 5s (was .Second(), the 0-59 seconds field, and retryOn was never set: no delay) + item.retryIn = time.Now().Add(5 * time.Second).Unix() + item.retryOn[providerId] = true } item.flaginUP = false } item.flaginDL = false item.fails++ + if wasUP && item.fails >= maxItemFails { + // stop re-pushing it there: matchThisUP/pushUP don't look at fails. + // With no other target the divider retires it (notarget). + item.errorOn[providerId] = true + dlog(always, "WARN upload of '%s' failed %d times, giving up on providerId=%d", item.segment.Id, item.fails, providerId) + } delete(item.availableOn, providerId) dlog(always, "Flagged item error, will not retry '%s' on providerId=%d", item.segment.Id, providerId) } @@ -514,7 +674,7 @@ func (s *SESSION) hasUploadTarget(item *segmentChanItem) bool { continue } p := s.providerList[pid] - if p.NoUpload || (!p.capabilities.ihave && !p.capabilities.post) { + if p.NoUpload || (!p.capabilities.ihave && !p.capabilities.post) || p.isDown() { continue } if item.ignoreUlOn[pid] || item.uploadedTo[pid] || item.availableOn[pid] || @@ -617,7 +777,7 @@ providerDl: dlog(cfg.opt.DebugWorker, " | [DV-pushDL] (nodl) item errorOn seg.Id='%s' @ #'%s'", item.segment.Id, s.providerList[pid].Group) continue providerDl } - if s.providerList[pid].NoDownload { + if s.providerList[pid].NoDownload || s.providerList[pid].isDown() { nodl++ item.ignoreDlOn[pid] = true continue providerDl @@ -688,7 +848,7 @@ providerUp: continue providerUp } //s.providerList[pid].mux.RLock() // FIXME TODO #b8bd287b: dynamic capas - flagNoUp := (s.providerList[pid].NoUpload || (!s.providerList[pid].capabilities.ihave && !s.providerList[pid].capabilities.post)) + flagNoUp := (s.providerList[pid].NoUpload || (!s.providerList[pid].capabilities.ihave && !s.providerList[pid].capabilities.post) || s.providerList[pid].isDown()) //s.providerList[pid].mux.RUnlock() if flagNoUp { noup++ @@ -768,6 +928,11 @@ type WrappedItem struct { const testing = false // set to true to test the worker loop // GoWorkDivider is the main worker loop that processes segments +// noProgressTimeout ends a session whose counters (and finished attempts, workDone) +// did not move for this long. Note: with UseNoDeadline conns have no read deadline, +// so a routine blocked on a silent server still holds up waitWorker after this fires. +const noProgressTimeout = 3 * time.Minute + func (s *SESSION) GoWorkDivider(waitDivider *sync.WaitGroup, waitDividerDone *sync.WaitGroup) { dlog(cfg.opt.DebugWorker, "go GoWorkDivider() waiting on waitDivider.Wait()") waitDivider.Wait() // waits for all conns to establish @@ -796,8 +961,9 @@ func (s *SESSION) GoWorkDivider(waitDivider *sync.WaitGroup, waitDividerDone *sy fetchedtoDL, fetchedtoUP, backlogDL, refillUP, pushedDL, pushedUP := uint64(0), uint64(0), uint64(0), uint64(0), uint64(0), uint64(0) startLoop := time.Now() deadWorkersDeadline := 9 - deadcounter := 10 - var capture_done, capture_segm, capture_isdl, capture_isup float64 + // watchdog: quit if no counter moved for noProgressTimeout (any -verbose setting) + var lastProgress uint64 + lastProgressAt := time.Now() forever: for { @@ -836,17 +1002,12 @@ forever: var Tnodl, Tnoup uint64 startLoop = time.Now() + checkersGone := s.allCheckersGone() // before counting: then the counts below are final for checks forsegmentList: for _, item := range s.segmentList { item.mux.PrintStatus(false) - if deadcounter <= 0 { - dlog(always, "GoWorkDivider: counters not increasing for too long, exiting") - closeCase = "noProgress" - break forever - } - item.mux.RLock() // RLOCKS HERE #824d if item.flagCache { cached++ @@ -979,35 +1140,6 @@ forever: yenc_perc := float64(isyenc) / float64(todo) * 100 used_slots, max_slots := memlim.Usage() - if segm_perc > capture_segm { - capture_segm = segm_perc - deadcounter = 10 // reset deadcounter - } else if segm_perc < 100 && segm_perc == capture_segm { - deadcounter-- - dlog(always, "GoWorkDivider: segm_perc not increasing, deadcounter=%d", deadcounter) - - } else if segcheckdone && done_perc > capture_done { - capture_done = done_perc - deadcounter = 10 // reset deadcounter - } else if segcheckdone && done_perc < 100 && done_perc == capture_done { - deadcounter-- - dlog(always, "GoWorkDivider: done_perc not increasing, deadcounter=%d", deadcounter) - - } else if segcheckdone && isdl_perc > capture_isdl { - capture_isdl = isdl_perc - deadcounter = 10 // reset deadcounter - } else if segcheckdone && isdl_perc < 100 && isdl_perc == capture_isdl { - deadcounter-- - dlog(always, "GoWorkDivider: isdl_perc not increasing, deadcounter=%d", deadcounter) - - } else if segcheckdone && isup_perc > capture_isup { - capture_isup = isup_perc - deadcounter = 10 // reset deadcounter - } else if segcheckdone && isup_perc < 100 && isup_perc == capture_isup { - deadcounter-- - dlog(always, "GoWorkDivider: isup_perc not increasing, deadcounter=%d", deadcounter) - } - //log00 = fmt.Sprintf(" TODO: %d ", todo) if (cfg.opt.CheckFirst || cfg.opt.CheckOnly) || (checked > 0 && checked != segm && checked != done) { @@ -1109,6 +1241,24 @@ forever: dlog(always, " | Segment Check Done: took='%.1f sec'", took.Seconds()) } + // workDone also moves on failed attempts: slow failing work (a dead provider) is not stuck + if progress := checked + isdl + isup + isyenc + cached + done + dead + fails + notarget + s.workDone.Load(); progress != lastProgress { + lastProgress, lastProgressAt = progress, time.Now() + } else if time.Since(lastProgressAt) > noProgressTimeout { + dlog(always, "WARN [DV] no progress for %v, giving up: checked=%d/%d done=%d isdl=%d isup=%d indl=%d inup=%d dlQ=%d upQ=%d", + noProgressTimeout, checked, todo, done, isdl, isup, indl, inup, dlQ, upQ) + closeCase = "noProgress" + break forever + } + + // checkedOn only changes in check routines: once all are gone, unchecked items stay unchecked. + // Finish what is in flight, then quit instead of waiting for checked == todo forever. + if checked != todo && checkersGone && inup == 0 && indl == 0 && inretry == 0 && inyenc == 0 && dlQ == 0 && upQ == 0 && yeQ == 0 { + dlog(always, "WARN [DV] all check routines are gone, %d/%d items were not checked on every provider: giving up on them", todo-checked, todo) + closeCase = "checkersGone" + break forever + } + // continue as long as any of this triggers because stuff is still in queues and processing if checked != todo || inup > 0 || indl > 0 || inretry > 0 || inyenc > 0 || dlQ > 0 || upQ > 0 || yeQ > 0 { dlog(cfg.opt.DebugWorker, "\n[DV] continue [ allOk=%d | checked=%d !=? todo=%d | done=%d | TupQ=%d !=? isup=%d || TdlQ=%d !=? isdl=%d || inup=%d > 0? || indl=%d > 0? || inretry=%d > 0? || dlQ=%d > 0? || upQ=%d > 0? || yeQ=%d > 0? ]\n", diff --git a/review-the-codebase-for-jolly-petal.md b/review-the-codebase-for-jolly-petal.md index 9592dcc..af30712 100644 --- a/review-the-codebase-for-jolly-petal.md +++ b/review-the-codebase-for-jolly-petal.md @@ -161,6 +161,42 @@ Written under `globalmux` in each provider's boot goroutine (`Workers.go:65-81`) **Fix:** give the feeder and the divider a shared "give up" signal, for example closed when every check routine of a group has exited, plus a deadline. The feeder's close must follow the 2.6 single-owner rule. +**Result (2026-10-18):** fixed. Testing found four more hangs on the same path; all are fixed here. +- **Give-up signal.** + - `checkerGroup` counts each group's check routines, built with the channels in `makeSegmentChans`. + - The count drops when a check routine exits and for every routine that never starts (boot `GetConn` failure, shared-conn failure). At 0 it closes `gone`. + - The feeder selects on `gone` and closes its channel (still the only owner). + - When every group is gone, `checked != todo` and nothing is in flight, the divider quits with `checkersGone`. +- **(new) A failed check was dropped.** A `STAT` with no reply, or no connection, never reached `checkedOn`, so `checked == todo` never held. + - `checkItem` retries once on a broken connection. + - Otherwise `markCheckFailed` sets `errorOn`/`ignoreDlOn` for the group (not `missingOn`), and the divider retires the item ("no upload target"). + - A check routine quits after 3 items in a row with no connection. +- **(new) The watchdog only ran with `-verbose`** (it lived in the stats block; `-verbose` is on by default) and was wrong. + - It counted down whenever `segm_perc` stayed below 100, so any NZB with segments missing everywhere was killed about 20 s into the transfer: 32 of 40 uploads in the test. Without `-verbose` there was no watchdog, so a dead group hung forever. + - Replaced by an always-on 3-minute no-progress check over the counters plus `s.workDone` (every finished attempt, so slow failing work doesn't count as stuck). +- **(new) Conn failures left items "in flight" forever.** `GoReupsRoutine`/`GoDownsRoutine` returned on a `GetConn` failure without `FlagError`, so `flaginUP`/`flaginDL` stayed set and the divider waited forever. +- **(new) Failed uploads were retried forever.** `matchThisUP`/`pushUP` ignore `fails`. After `maxItemFails` (3), `FlagError` now sets `errorOn` for that provider. The retry delay works now: `.Unix()` instead of `.Second()`, and `retryOn` is set. +- **(new) Mem-slot deadlock on upload errors.** The re-upload loop returned the slot on error. Downloads took every slot, and the re-queued item waited in `MemLockWait` behind items that held them; goroutine dump: all 8 routines in `MemLimit.go:106`. A failed upload now keeps its slot until it succeeds or is retired; `releaseMemSlots` returns the rest at session end. +- **(new) Work after the stop.** The downs/reups routines drop items still queued once the session has stopped, and `GetConn` no longer dials after the stop. +- **(new) Provider down for the session.** When `GetConn` gives up on a dial error (`errDial`, now wrapped through `connect`/`NewConn`), `Provider.markDown` takes the provider out for the rest of the session. + - The routines and `pushUP`/`pushDL`/`hasUploadTarget` then skip it without touching the network. + - Never in proxy sessions, whose pools live as long as the process. Checked: B dialled again on every request, never marked down. + - Without this, each item cost 3 × ~9 s of dial retries: 90 uploads ran past 400 s. +- Tests: `TestFeederClosesWhenCheckersGone`. +- **Verified under `-race` against the local servers, no races in any run:** + +| Scenario | Before | After | +|---|---|---| +| B down at boot, no `-verbose` | hangs (killed at 120 s) | 13 s | +| Segments missing everywhere, slow uploads, `-verbose` | killed at ~20 s, 32/40 uploads | 50 s, 40/40 | +| B killed during checks | — | 31 s (first fix round: 385 s via watchdog, then 188 s) | +| B killed during uploads | — | 32 s, close case #8, no slots leaked | + + - A→B, same-group and the 2.4 leak setup: unchanged results. + - Web mode: both jobs complete; the 25 races are all in `webserver.go` (4.2). +- **Note:** `UseNoDeadline = true` (`Config.go`): conns have no read deadline, so a server that accepts and then goes silent can still block a routine, and with it `waitWorker`, after the watchdog has fired. That belongs with 3.14 (`checkCapabilities`). +- **Limitation:** items not checked on a dead group are given up for the whole session, even when other groups could still serve them. + ### 3.3 Shared-connection early returns (dropped as a bug) `SharedConnGet` only returns an item with `conn != nil`. On the error path it has already put `nil` back into `sharedCC` (`ConnPool.go:749`). The `conn != nil && srvtp == nil` window doesn't occur: `CloseConn` runs in the holder's own goroutine. The guards at `Routines.go:45-47`, `:170-172`, `:348-350` are defensive dead code. If they are touched, never push a token back on the error path; the cap-1 channel is already refilled. @@ -268,7 +304,7 @@ Keyed by a per-session index (`Utils.go:200`, `ConnPool.go:134/181`), so session **Result (2026-10-17):** fixed. `TestIHAVEReplyNot335KeepsConnUsable` (fake server answers 436, then 435) blocks on the old code and passes now. -**Also noted (not fixed):** `FlagError` (`Workers.go`) sets `retryIn = int64(time.Now().Add(5 * time.Second).Second())`. That is the seconds field of the current minute (0–59), not a Unix time, so `retryIn > now` is never true and the 5 s retry delay never applies. It should be `.Unix()`. +**Also noted (fixed with 3.2):** `FlagError` (`Workers.go`) set `retryIn = int64(time.Now().Add(5 * time.Second).Second())`. That is the seconds field of the current minute (0–59), not a Unix time, so `retryIn > now` was never true and the 5 s retry delay never applied (it also never set `retryOn`). ### 3.14 Remaining unsynchronised accesses (corrected) From ce726f9dc9a68306cbd3876d2e457a972e1a144b Mon Sep 17 00:00:00 2001 From: go-while Date: Sun, 18 Oct 2026 00:57:24 +0000 Subject: [PATCH 127/152] Fix CloseConn double decrement, bound KillConnPool, stop leaked goroutines - CloseConn: nil-safe; the open counters drop once per conn (a second close drifted openConns negative), the shared-conn token is always sent. - KillConnPool: no global lock, bounded wait (5 s), only idle conns closed. The unused ConnPools map and PoolsLock are gone (and the lock inversion). - Worker tick goroutines are time.Tickers; WatchOpenConnsThread and the per-pool Speedmeter stop when KillConnPool closes the pool's stop channel. - readDotLines: early returns close the rapidyenc pipe / decodeBodyChan and wait for the decoder goroutine (with -yenctest=4 the decoder was released while still in use); the async decoder keeps draining after an error. 4 web jobs in a row: 122 goroutines left before, 10 after. Co-Authored-By: Claude Opus 5.5 --- ConnPool.go | 123 +++++++++++++++---------- ConnPool_test.go | 54 +++++++++++ NetConn.go | 33 ++++++- NetConn_test.go | 56 +++++++++++ Workers.go | 39 ++------ review-the-codebase-for-jolly-petal.md | 21 +++++ 6 files changed, 246 insertions(+), 80 deletions(-) create mode 100644 ConnPool_test.go diff --git a/ConnPool.go b/ConnPool.go index dd55abf..66be5ae 100644 --- a/ConnPool.go +++ b/ConnPool.go @@ -49,9 +49,7 @@ const ( var ( //noDeadLine time.Time // used as zero value! - readDeadConn = time.Unix(1, 0) // no syscall needed. value is 1 second after "0" (1970-01-01 00:00:01 UTC) - ConnPools = make(map[int]*ConnPool) // used in KillConnPool - PoolsLock sync.RWMutex + readDeadConn = time.Unix(1, 0) // no syscall needed. value is 1 second after "0" (1970-01-01 00:00:01 UTC) ) // holds an active connection to share around @@ -78,13 +76,15 @@ type ConnPool struct { wants_auth bool // we have created an endless loop // provider.ConnPool.provider.ConnPool.provider.ConnPool.provider.ConnPool.provider.ConnPool.provider.ConnPool.CloseConn(provider, connitem) xD - // if we want to call closeconn from outside the routine which keep a provider ptr too - // we cann call ConnPools[provider.id].CloseConn(connitem) provider *Provider // pointer to the provider which created this ConnPool and holds the config s *SESSION // session pointer to the session which created this ConnPool (access via c.s.%%SESSIONVARIABLES%% OR provider.ConnPool.s.%%SESSIONVARIABLES%%). // sessStop is the owning session's stopChan (nil: never stops). Blocking waits // for a connection give up when it is closed. sessStop <-chan struct{} + // poolStop ends the pool's own goroutines (WatchOpenConnsThread, Speedmeter). + // Closed once by KillConnPool; the proxy's pools are never killed, so it is not the session's. + poolStop chan struct{} + poolStopOnce sync.Once } // errSessionStopped is returned by waits that gave up because the session stopped. @@ -152,10 +152,8 @@ func NewConnPool(s *SESSION, provider *Provider, workerWGconnReady *sync.WaitGro mux: &loggedrwmutex.LoggedSyncRWMutex{Name: fmt.Sprintf("ConnPool-%s", provider.Name)}, s: s, // session pointer to the session which created this ConnPool sessStop: s.stopChan, + poolStop: make(chan struct{}), } - PoolsLock.Lock() - ConnPools[provider.id] = provider.ConnPool - PoolsLock.Unlock() if !cfg.opt.CheckOnly { go Speedmeter(s.nzbSize, provider.ConnPool, nil, workerWGconnReady) // start speed meter for this ConnPool / Provider } @@ -163,45 +161,49 @@ func NewConnPool(s *SESSION, provider *Provider, workerWGconnReady *sync.WaitGro // no return value as we mutate the provider pointer! } // end func NewConnPool -func KillConnPool(provider *Provider) { - PoolsLock.Lock() - defer PoolsLock.Unlock() +// killConnPoolWait is how long KillConnPool waits for conns still in use to come back. +// It runs after all workers have exited, so normally nothing is in use anymore. +const killConnPoolWait = 5 * time.Second - if provider == nil || provider.ConnPool == nil { +// KillConnPool closes the idle conns of the provider's pool, waits (bounded) for the +// ones still in use, then drops the pool. Conns in use belong to their goroutine and +// are not closed here (that would be a data race on the ConnItem). +// No global lock is held while waiting (it used to hold PoolsLock for as long as it took). +func KillConnPool(provider *Provider) { + if provider == nil { return } - - provider.ConnPool.mux.RLock() - openConns := provider.ConnPool.openConns - provider.ConnPool.mux.RUnlock() - - if openConns > 0 { - dlog(cfg.opt.DebugConnPool, "KillConnPool: '%s'", provider.Name) - killed := 0 + provider.mux.RLock() + c := provider.ConnPool + provider.mux.RUnlock() + if c == nil { + return + } + deadline := time.Now().Add(killConnPoolWait) + killed := 0 + for { + drain: for { select { - case connitem := <-provider.ConnPool.pool: - provider.ConnPool.CloseConn(connitem, nil) - - dlog(cfg.opt.DebugConnPool, "KillConnPool: '%s' closed a conn", provider.Name) - + case connitem := <-c.pool: + c.CloseConn(connitem, nil) killed++ default: - // chan ran empty - } - provider.ConnPool.mux.RLock() - openConns := provider.ConnPool.openConns - provider.ConnPool.mux.RUnlock() - dlog(cfg.opt.DebugConnPool, "KillConnPool: '%s' openConns=%d killed=%d", provider.Name, openConns, killed) - if openConns > 0 { - time.Sleep(10 * time.Millisecond) - continue + break drain // chan ran empty } + } + openConns, _ := c.GetStats() + if openConns <= 0 { break - } // end for + } + if time.Now().After(deadline) { + dlog(always, "WARN KillConnPool '%s': %d conns still in use after %v, not waiting for them", provider.Name, openConns, killConnPoolWait) + break + } + time.Sleep(10 * time.Millisecond) } - - delete(ConnPools, provider.id) + dlog(cfg.opt.DebugConnPool, "KillConnPool: '%s' closed %d idle conns", provider.Name, killed) + c.poolStopOnce.Do(func() { close(c.poolStop) }) // ends WatchOpenConnsThread and Speedmeter provider.mux.Lock() provider.ConnPool = nil @@ -553,8 +555,27 @@ func (c *ConnPool) ParkConn(wid int, connitem *ConnItem, src string) { // This is used to signal the routines to get a new connection from the pool. // If the connection is nil, it will not close it and just remove it from the pool. // If the connection is already closed, it will not close it again. +// CloseConn closes connitem and gives back its slot in the pool count, once: +// a second call on the same item (readDotLines closes on read errors, the caller +// closes again) or a nil item only passes the token on. If sharedConnChan is given +// a nil token is always sent, so sibling routines waiting in SharedConnGet go on. func (c *ConnPool) CloseConn(connitem *ConnItem, sharedConnChan chan *ConnItem) { + // only the decrement is guarded: returning early would skip the token below + // and leave the worker's other routines blocked in SharedConnGet + if connitem != nil && connitem.conn != nil { + c.closeConnOnce(connitem) + } + // if a sharedConnChan is supplied we send a nil to the channel + // a nil as connitem signals the routines to get a new conn + // mostly because conn was closed by network, protocol error or timeout + if sharedConnChan != nil { + sharedConnChan <- nil + } +} // end func CloseConn +// closeConnOnce closes an open connitem and decrements the open counters. +// The caller checked connitem.conn != nil; conn is set to nil here, so it runs once per conn. +func (c *ConnPool) closeConnOnce(connitem *ConnItem) { if cfg.opt.DebugConnPool { GCounter.Incr("TOTAL_DisConns") } @@ -582,13 +603,7 @@ func (c *ConnPool) CloseConn(connitem *ConnItem, sharedConnChan chan *ConnItem) dlog(cfg.opt.DebugConnPool, "DisConn '%s' inPool=%d open=%d", c.provider.Name, len(c.pool), c.openConns) c.mux.Unlock() go c.provider.DecrementOpenConns() - // if a sharedConnChan is supplied we send a nil to the channel - // a nil as connitem signals the routines to get a new conn - // mostly because conn was closed by network, protocol error or timeout - if sharedConnChan != nil { - sharedConnChan <- nil - } -} // end func CloseConn +} // end func closeConnOnce // GetStats returns the current stats of the connection pool. // It returns the number of open connections and the number of idle connections in the pool. @@ -612,9 +627,15 @@ func (c *ConnPool) WatchOpenConnsThread(workerWGconnReady *sync.WaitGroup) { dlog(cfg.opt.Debug, "WatchOpenConnsThread started for '%s' waited=%v", c.provider.Name, time.Since(start)) defer dlog(cfg.opt.Debug, "WatchOpenConnsThread for '%s' defer returned", c.provider.Name) //tempItems := []*ConnItem{} // temporary slice to hold ConnItems + tick := time.NewTicker(15 * time.Second) // check every N + defer tick.Stop() forever: for { - time.Sleep(time.Millisecond * 15000) // check every N milliseconds + select { + case <-tick.C: + case <-c.poolStop: + return // KillConnPool: it used to run (and log) forever after the session ended + } if !loggedrwmutex.DisableLogging { c.provider.mux.Lock() if c.provider.ConnPool != nil { @@ -871,10 +892,19 @@ func Speedmeter(byteSize int64, cp *ConnPool, cnt *Counter_uint64, workerWGconnR return fmt.Errorf("error in Speedmeter: supply cp or cnt or nothing to use global counter, but not both") } + var stop <-chan struct{} // nil for the global meter: runs as long as the process + if cp != nil { + stop = cp.poolStop + } var totalTX, totalRX uint64 ticker := time.NewTicker(time.Second * time.Duration(PrintStats)) defer ticker.Stop() - for range ticker.C { + for { + select { + case <-ticker.C: + case <-stop: + return nil // KillConnPool + } var tmpRX, tmpTX uint64 globalmux.Lock() tmpRX, tmpTX = counter.GetReset("TMP_RXbytes"), counter.GetReset("TMP_TXbytes") @@ -900,7 +930,6 @@ func Speedmeter(byteSize int64, cp *ConnPool, cnt *Counter_uint64, workerWGconnR printSpeedTable(dlPerc, upPerc, totalRX, totalTX, rxSpeed, txSpeed, rxMbps, txMbps, name, group) } - return nil } func printSpeedTable(dlPerc, upPerc int, totalRX uint64, totalTX uint64, rxSpeed int64, txSpeed int64, rxMbps float64, txMbps float64, provider string, group string) { diff --git a/ConnPool_test.go b/ConnPool_test.go new file mode 100644 index 0000000..90bdbb9 --- /dev/null +++ b/ConnPool_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "net" + gotesting "testing" // Workers.go declares a const named testing + "time" + + "github.com/go-while/go-loggedrwmutex" +) + +// Plan 3.4: CloseConn dereferenced a nil item, and a second close of the same +// item (readDotLines closes on read errors, the caller closes again) decremented +// openConns twice. Negative drift made KillConnPool stop waiting early and let +// GetConn dial past MaxConns. Each call must still pass the shared-conn token on. +func TestCloseConnNilSafeAndDecrementsOnce(t *gotesting.T) { + testCfg() + prov := &Provider{Name: "p", MaxConns: 2, OpenConns: 1, mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "p"}} + c := &ConnPool{provider: prov, openConns: 1, pool: make(chan *ConnItem, 2), + mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "pool-p"}} + a, b := net.Pipe() + defer b.Close() + item := &ConnItem{c: c, conn: a} + sharedCC := make(chan *ConnItem, 1) + + token := func(what string) { + t.Helper() + select { + case got := <-sharedCC: + if got != nil { + t.Fatalf("%s: want a nil token, got %#v", what, got) + } + case <-time.After(time.Second): + t.Fatalf("%s: no token sent to sharedCC", what) + } + } + + c.CloseConn(item, sharedCC) + token("first close") + c.CloseConn(item, sharedCC) // same item again + token("second close") + c.CloseConn(nil, sharedCC) // nil item + token("nil item") + c.CloseConn(nil, nil) // no panic, nothing to send + + if open, _ := c.GetStats(); open != 0 { + t.Fatalf("openConns: want 0 after closing one conn, got %d", open) + } + time.Sleep(50 * time.Millisecond) // DecrementOpenConns runs in a goroutine + prov.mux.RLock() + defer prov.mux.RUnlock() + if prov.OpenConns != 0 { + t.Fatalf("provider.OpenConns: want 0, got %d", prov.OpenConns) + } +} diff --git a/NetConn.go b/NetConn.go index 4aad11c..bdef636 100644 --- a/NetConn.go +++ b/NetConn.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "io" "log" @@ -336,6 +337,9 @@ func CMD_POST(connitem *ConnItem, item *segmentChanItem) (int, string, uint64, e // the number of bytes read (rxb), the content as a slice of strings, and any error encountered. // The function supports parsing headers, handling continued lines, and cleaning headers based on the configuration. // It also supports yenc decoding if enabled in the configuration. +// errReadAborted closes the rapidyenc pipe when readDotLines returns early. +var errReadAborted = errors.New("readDotLines aborted") + // WARNING: TODO! Be careful when reading X/OVER or X/HDR on large groups because you may run out of memory! func readDotLines(connitem *ConnItem, item *segmentChanItem, command string) (code int, rxb int, content []string, err error) { if connitem.conn == nil || connitem.srvtp == nil { @@ -391,6 +395,7 @@ func readDotLines(connitem *ConnItem, item *segmentChanItem, command string) (co start := time.Now() decodedBuf := make([]byte, cfg.opt.RapidYencBufSize) total := 0 + var result error for { n, err := rydecoder.Read(decodedBuf) if n > 0 { @@ -398,15 +403,15 @@ func readDotLines(connitem *ConnItem, item *segmentChanItem, command string) (co total += n } if err == io.EOF { - done <- nil break } if err != nil { - done <- err + result = err break } } dlog(cfg.opt.DebugRapidYenc, "readDotLines: rapidyenc decoder done for seg.Id='%s' @ '%s' decoded=%d bufsize=%d total=%d took=(%d µs)", *segId, connitem.c.provider.Name, len(*decoded), len(decodedBuf), total, time.Since(start).Microseconds()) + done <- result // last: after this the caller may release the decoder }(&item.segment.Id, &decodedData, ryDoneChan) case 3: @@ -451,7 +456,9 @@ func readDotLines(connitem *ConnItem, item *segmentChanItem, command string) (co cERR++ counter.Incr("ERR") returnAsyncCoreLimiter() - break readChan + // keep draining (broken skips the rest): breaking here left the reader + // blocked on a full decodeBodyChan forever + continue readChan } returnAsyncCoreLimiter() @@ -467,6 +474,25 @@ func readDotLines(connitem *ConnItem, item *segmentChanItem, command string) (co }(item.segment.Id, connitem.c.provider.Name, decodeBodyChan, counter, releaseDecoder) } // end if async } // end case 3 + + // An early return (read error, max size, write error, ...) used to leave the async + // decoder goroutine blocked forever: case 4 on the pipe that was never closed (and the + // deferred ReleaseDecoder handed the decoder back while it still read from it), + // case 3 on decodeBodyChan. The success path below closes them itself (finished). + finished := false + defer func() { + if finished { + return + } + if pipeWriter != nil { + pipeWriter.CloseWithError(errReadAborted) + <-ryDoneChan // the reader goroutine has stopped: ReleaseDecoder (deferred before) is safe now + } + if decodeBodyChan != nil { + close(decodeBodyChan) + <-releaseDecoder // it drains what is buffered, then exits + } + }() startReadLines := time.Now() i := 0 readlines: @@ -670,6 +696,7 @@ readlines: } // end for readlines + finished = true // from here on the pipe / decodeBodyChan are closed below // case 4: we are done reading lines, close the pipe writer from rapidyenc decoder if pipeWriter != nil { if _, err := pipeWriter.Write([]byte(DOT + CRLF)); err != nil { // write the final dot to the pipe diff --git a/NetConn_test.go b/NetConn_test.go index 1aefac7..864830c 100644 --- a/NetConn_test.go +++ b/NetConn_test.go @@ -4,6 +4,7 @@ import ( "bufio" "net" "net/textproto" + "runtime" "strings" gotesting "testing" // Workers.go declares a const named testing "time" @@ -59,3 +60,58 @@ func TestIHAVEReplyNot335KeepsConnUsable(t *gotesting.T) { } } } + +// Plan 3.7: with -yenctest=3/4 an early return from readDotLines (here: the server +// closes the conn mid-article) left the async decoder goroutine blocked forever, +// and with 4 the deferred ReleaseDecoder handed the decoder back while it was still in use. +func TestReadDotLinesEarlyReturnStopsDecoder(t *gotesting.T) { + testCfg() + old := *cfg.opt + defer func() { *cfg.opt = old }() + cfg.opt.YencCRC, cfg.opt.MaxArtSize, cfg.opt.RapidYencBufSize = true, 1<<20, 4096 + if async_core_chan == nil { // LaunchSession sets it up + async_core_chan = make(chan struct{}, 1) + async_core_chan <- struct{}{} + } + + for _, mode := range []int{3, 4} { + cfg.opt.YencTest = mode + cli, srv := net.Pipe() + go func() { + srv.Write([]byte("Subject: x\r\n\r\n=ybegin part=1 total=1 line=128 size=3 name=x\r\n=ypart begin=1 end=3\r\n")) + srv.Write([]byte("abc\r\n")) + srv.Close() // no "=yend", no final dot: ReadLine fails + }() + prov := &Provider{Name: "fake", mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "fake"}} + connitem := &ConnItem{c: &ConnPool{provider: prov}, conn: cli, srvtp: textproto.NewConn(cli), writer: bufio.NewWriter(cli)} + + done := make(chan error, 1) + go func() { + _, _, _, err := readDotLines(connitem, testItem(""), cmdARTICLE) + done <- err + }() + select { + case err := <-done: + if err == nil { + t.Fatalf("yenctest=%d: want a read error", mode) + } + case <-time.After(3 * time.Second): + t.Fatalf("yenctest=%d: readDotLines blocked", mode) + } + cli.Close() + + // the fix waits for the decoder goroutine before returning; allow a little slack anyway + deadline := time.Now().Add(time.Second) + for { + buf := make([]byte, 1<<20) + stacks := string(buf[:runtime.Stack(buf, true)]) + if !strings.Contains(stacks, ".readDotLines.func") { // package path differs under go test + break + } + if time.Now().After(deadline) { + t.Fatalf("yenctest=%d: async decoder goroutine still running after readDotLines returned", mode) + } + time.Sleep(20 * time.Millisecond) + } + } +} diff --git a/Workers.go b/Workers.go index 5b8a3ff..b730fbb 100644 --- a/Workers.go +++ b/Workers.go @@ -278,16 +278,9 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro defer s.checkersExit(provider, 1) timeOut := time.Now().Add(15 * time.Second) lastGood := time.Now() - toChan := make(chan struct{}, 1) - go func() { - for { - time.Sleep(3 * time.Second) - select { - case toChan <- struct{}{}: - default: - } - } - }() + tick := time.NewTicker(3 * time.Second) // was a goroutine per routine that never stopped + defer tick.Stop() + toChan := tick.C forGoCheckRoutine: for { select { @@ -346,16 +339,9 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro defer thisWorkerWG.Done() timeOut := time.Now().Add(15 * time.Second) lastGood := time.Now() - toChan := make(chan struct{}, 1) - go func() { - for { - time.Sleep(3 * time.Second) - select { - case toChan <- struct{}{}: - default: - } - } - }() + tick := time.NewTicker(3 * time.Second) // was a goroutine per routine that never stopped + defer tick.Stop() + toChan := tick.C forGoDownsRoutine: for { dlog(cfg.opt.DebugWorker, "GoDownsRoutine: (%d@'%s') wait on segCD len=%d", wid, provider.Name, len(segCD)) @@ -454,16 +440,9 @@ func (s *SESSION) GoWorker(wid int, provider *Provider, waitWorker *sync.WaitGro defer thisWorkerWG.Done() timeOut := time.Now().Add(15 * time.Second) lastGood := time.Now() - toChan := make(chan struct{}, 1) - go func() { - for { - time.Sleep(3 * time.Second) - select { - case toChan <- struct{}{}: - default: - } - } - }() + tick := time.NewTicker(3 * time.Second) // was a goroutine per routine that never stopped + defer tick.Stop() + toChan := tick.C forGoReupsRoutine: for { dlog(cfg.opt.DebugWorker, "ReupsRoutine: (%d@'%s') wait on segCD len=%d", wid, provider.Name, len(segCD)) diff --git a/review-the-codebase-for-jolly-petal.md b/review-the-codebase-for-jolly-petal.md index af30712..207cdf2 100644 --- a/review-the-codebase-for-jolly-petal.md +++ b/review-the-codebase-for-jolly-petal.md @@ -208,12 +208,19 @@ Written under `globalmux` in each provider's boot goroutine (`Workers.go:65-81`) - **Fix (corrected):** guard **only the decrement** (`if connitem.conn == nil` → skip `openConns--`), and **always** send the `nil` token to `sharedCC`. In the double-close case the second call is the only one that sends it. An early `return` at the top would leave sibling routines blocked at `ConnPool.go:729`. - The draft's "blocking send under `provider.mux`" is **dropped**: `checkCapabilities` passes `nil` for `sharedCC` (`NetConn.go:989`). A non-blocking `select` there is harmless. +**Result (2026-10-18):** fixed as planned. Only the close and the decrement are guarded (`closeConnOnce`, which runs only while `connitem.conn != nil`). The nil token is always sent, and a nil item no longer panics; `GetConn` did hit this, via `go c.CloseConn(nil, nil)` for a nil item taken from the pool. Test: `TestCloseConnNilSafeAndDecrementsOnce` (panics on the old code). + ### 3.5 `KillConnPool` waits unboundedly while holding `PoolsLock` (corrected) `ConnPool.go:144-178` holds the global `PoolsLock` across an unbounded `Sleep(10ms)` loop until `openConns == 0`. It spins only on a *positive* leak (a connection that is never returned), not on the negative drift from 3.4. `LaunchSession` calls it per provider (`Session.go:386-388`). **Fix:** drain under a short lock, wait outside `PoolsLock` with a deadline. Force-close only items that are idle in the pool; closing a `connitem` another goroutine is using is a data race. +**Result (2026-10-18):** fixed together with 3.11 and 3.12. +- `ConnPools` and `PoolsLock` are deleted (never read), which also removes the lock-order inversion. +- `KillConnPool` closes the idle conns and waits up to `killConnPoolWait` (5 s) for the ones in use, without a global lock. It never closes a conn another goroutine holds, and it logs a WARN if it gives up. +- No WARN in any verification run: after `waitWorker`/`waitPool` nothing is in use anymore. + ### 3.6 Blocking receives with no stop case (corrected) `ConnPool.go:435` (`<-c.pool`, "infinite wait"), `ConnPool.go:729` (`<-sharedCC`), `MemLimit.go:112` (`<-m.memchan`). `:436` dereferences `connitem.connid` without a nil check (latent). @@ -244,6 +251,16 @@ Written under `globalmux` in each provider's boot goroutine (`Workers.go:65-81`) **Fix:** stop channel per `ConnPool`, closed by `KillConnPool`; `time.Ticker` with `defer Stop()`; move the `NetConn` cleanup into a `defer` that uses a "done" flag, so the success path (`:669-684`) doesn't close `decodeBodyChan`/`pipeWriter` twice. +**Result (2026-10-18):** fixed, except the `WriteYenc` spin (latent, unchanged). +- **Tickers:** the three per-routine tick goroutines are a `time.Ticker` with `defer Stop()`. +- **Pool goroutines:** `ConnPool.poolStop` is closed once by `KillConnPool`. `WatchOpenConnsThread` and the per-pool `Speedmeter` select on it; the global `Speedmeter` (nil `cp`) keeps running. +- **`readDotLines`:** a deferred cleanup with a `finished` flag closes the pipe (`CloseWithError`) or `decodeBodyChan` on every early return, then waits for the decoder goroutine. +- **(new) Worse than the table said** for `-yenctest=4`: the deferred `ReleaseDecoder` gave the decoder back to the pool while the leaked reader goroutine still read from it (use after release). The reader now signals `done` as its last action, and the cleanup waits for it before the release runs. +- **(new) `-yenctest=3`:** after a decode error the decoder goroutine stopped reading (`break`). If `decodeBodyChan` (6666 slots) was full at that moment, the reader blocked forever on the send. It now keeps draining (the `broken` flag skips the work). +- Test: `TestReadDotLinesEarlyReturnStopsDecoder` (fails on the old code). +- **Verified:** 4 sequential web jobs, goroutine dump 20 s after the last one. Old build: 122 goroutines (49 leaked worker tickers, plus watchers and speed meters of finished sessions). New: 10, and the 4 pool goroutines among them belong to the web processor's long-lived base session. +- **Note for 6.2:** `LaunchSession` replaces the global `core_chan`/`async_core_chan` when `YencCpu`/`YencAsyncCpu` changes, under `globalmux`, while `get/returnAsyncCoreLimiter` use them without it. Only reachable if per-job settings change those values. + ### 3.8 `MemLimiter` check-then-act gap (corrected; fix added) `MemLimit.go:74-90`: `memdata[item]` is set before the blocking wait, `memlocked` only after it at `:113`. No concrete path with two concurrent lockers on one item was found. A double `MemReturn` hits `log.Fatalf` (`:148`) only if the channel is full; otherwise it silently adds an extra slot and drives `memlocked` to −1. `MemAvail()` (`:67`) tests `m.waiting < m.mem_max`, which reports "available" with an empty `memchan`; it is used at `Workers.go:523`. @@ -273,10 +290,14 @@ Written under `globalmux` in each provider's boot goroutine (`Workers.go:65-81`) Keyed by a per-session index (`Utils.go:200`, `ConnPool.go:134/181`), so sessions overwrite each other. Harmless: nothing ever reads the map. **Fix:** delete it. +**Result (2026-10-18):** deleted, with 3.5. + ### 3.12 `PoolsLock` ↔ `provider.mux` lock-order inversion (latent) `NewConnPool` takes `provider.mux` (`ConnPool.go:88`) then `PoolsLock` (`:133`); `KillConnPool` takes them in the reverse order (`:144`, `:183`). Not live today, because each session copies its providers (`Utils.go:197-198`). Fix while the file is open: never hold `provider.mux` while taking `PoolsLock`. +**Result (2026-10-18):** gone with `PoolsLock` (3.5/3.11). + ### 3.13 The uncommitted band-aid locks (corrected; must land with their replacement) - `ConnPool.go:569-571`: `globalmux` around `start := time.Now()`. Protects nothing. **Revert.** From 0c3bd6e819af7ba5791331990d3960dbb7900848 Mon Sep 17 00:00:00 2001 From: go-while Date: Sun, 18 Oct 2026 09:31:27 +0000 Subject: [PATCH 128/152] MemLimiter: one owner record per slot, no double return, no exit on a second lock MemReturn checked memlocked and returned the slot in separate steps, so two concurrent returns of one item handed back two slots (or hit log.Fatalf on a full chan). memdata is now the single ownership record; it changes together with item.memlocked under m.mux, and only once the slot is taken. A second return is a logged no-op. A second MemLockWait on an item still waiting for a slot waits for the first one instead of calling os.Exit(1). MemAvail keeps its meaning (bound the downloads waiting for a slot): gating on free slots cost 25% throughput with -mem 4. waiting is atomic so MemAvail takes no lock, since pushDL calls it while holding item.mux. Co-Authored-By: Claude Opus 5.5 --- Config.go | 2 +- MemLimit.go | 121 ++++++++++++++----------- MemLimit_test.go | 73 ++++++++++++++- review-the-codebase-for-jolly-petal.md | 9 ++ 4 files changed, 149 insertions(+), 56 deletions(-) diff --git a/Config.go b/Config.go index 4b6d8e5..cb4fc14 100644 --- a/Config.go +++ b/Config.go @@ -226,7 +226,7 @@ type segmentChanItem struct { flaginYenc bool // if true, item is in writing to yenc cache flagisYenc bool // if true, item has been written to yenc cache flagCache bool // if true, item is cached - memlocked int // counts up if item is memlocked + memlocked int // 1 while the item holds a mem slot; only MemLimiter sets it, under its mux checkedOn int // counts up if item has been checked on a provider pushedDL int // a counter for debugging pushedUP int // a counter for debugging diff --git a/MemLimit.go b/MemLimit.go index 3d5b360..0b0b0cc 100644 --- a/MemLimit.go +++ b/MemLimit.go @@ -15,17 +15,21 @@ package main */ import ( - "log" - "os" + "sync/atomic" "github.com/go-while/go-loggedrwmutex" ) type MemLimiter struct { mem_max int - waiting int + waiting atomic.Int64 // callers inside MemLockWait waiting for a slot; atomic so MemAvail needs no lock memchan chan struct{} + // memdata holds the items that own a slot. It is the only record of who owns + // one: it changes together with item.memlocked, and both only under mux. memdata map[*segmentChanItem]bool + // pending holds the items still waiting for a slot. A second MemLockWait on + // such an item waits for the first one instead of taking a second slot. + pending map[*segmentChanItem]chan struct{} //mux sync.RWMutex mux *loggedrwmutex.LoggedSyncRWMutex // debug mutex } @@ -37,6 +41,7 @@ func NewMemLimiter(value int) *MemLimiter { memlim := &MemLimiter{ memchan: make(chan struct{}, value), memdata: make(map[*segmentChanItem]bool, value), + pending: make(map[*segmentChanItem]chan struct{}), mem_max: value, mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "MemLimiter"}, } @@ -64,37 +69,52 @@ func (m *MemLimiter) ViewData() (data []string) { return } // end func memlim.ViewData -func (m *MemLimiter) MemAvail() (retbool bool) { - m.mux.RLock() - retbool = (m.waiting < m.mem_max) - m.mux.RUnlock() - return +// MemAvail reports whether fewer than mem_max callers are waiting for a slot. +// pushDL uses it to bound the downloads queued behind a full limiter. It does +// not mean a slot is free: queued downloads wait in MemLockWait, holding no +// memory, so they take a slot the moment one comes back. Gating on free slots +// instead (plan 3.8) cost ~25% throughput with -mem 4: every freed slot then +// waited for the next divider pass before a download was queued for it. +// +// It takes no lock: pushDL calls it while holding item.mux. +func (m *MemLimiter) MemAvail() bool { + return m.waiting.Load() < int64(m.mem_max) } // MemLockWait takes a memory slot for item. It waits until a slot is free or // until stop is closed (a nil stop waits forever). It returns false if it gave // up because of stop: the item then holds no slot and must not be downloaded. // It returns true if the item holds a slot (also when it already held one). +// +// Lock order: m.mux before item.mux. Callers must not hold item.mux. func (m *MemLimiter) MemLockWait(item *segmentChanItem, who string, stop <-chan struct{}) bool { - item.mux.Lock() - if item.memlocked > 0 { - item.mux.Unlock() - dlog(always, "MemLockWait called on already memlocked item seg.Id='%s' who='%s'", item.segment.Id, who) - return true - } - item.mux.Unlock() - GCounter.Incr("MemLockWait") defer GCounter.Decr("MemLockWait") - m.mux.Lock() - if m.memdata[item] { - dlog(always, "ERROR ! MemLimit tried to lock an item already in mem! seg.Id='%s' who='%s'", item.segment.Id, who) - os.Exit(1) // this is a bug! we should never try to lock an item already in mem! + for { + m.mux.Lock() + if m.memdata[item] { + m.mux.Unlock() + dlog(cfg.opt.DebugMemlim, "MemLockWait called on already memlocked item seg.Id='%s' who='%s'", item.segment.Id, who) + return true + } + other := m.pending[item] + if other == nil { + break // we wait for the slot, still holding m.mux + } + m.mux.Unlock() + // someone else is already waiting for a slot for this item: wait for them, then look again + dlog(always, "MemLockWait: seg.Id='%s' is already waiting for a slot, who='%s' waits for it", item.segment.Id, who) + select { + case <-other: + case <-stop: + return false + } } - m.memdata[item] = true // flag as in mem - dlog(cfg.opt.DebugMemlim, "MemLockWait avail=%d/%d m.waiting=%d who='%s'", len(m.memchan), m.mem_max, m.waiting, who) - m.waiting++ + done := make(chan struct{}) + m.pending[item] = done + waiting := m.waiting.Add(1) + dlog(cfg.opt.DebugMemlim, "MemLockWait avail=%d/%d m.waiting=%d who='%s'", len(m.memchan), m.mem_max, waiting, who) m.mux.Unlock() // take a free slot right away if there is one, so a closed stop never wins over a free slot @@ -110,45 +130,42 @@ func (m *MemLimiter) MemLockWait(item *segmentChanItem, who string, stop <-chan } } + m.mux.Lock() + m.waiting.Add(-1) + delete(m.pending, item) + if got { + m.memdata[item] = true + item.mux.Lock() + item.memlocked = 1 + item.mux.Unlock() + } + close(done) + m.mux.Unlock() + if !got { - // roll back, or the next MemLockWait on this item hits the os.Exit above - m.mux.Lock() - m.waiting-- - delete(m.memdata, item) - m.mux.Unlock() GCounter.Incr("TOTAL_MemLockAborted") dlog(cfg.opt.DebugMemlim, "MemLockWait aborted by stop seg.Id='%s' who='%s'", item.segment.Id, who) return false } - - item.mux.Lock() - item.memlocked++ - item.mux.Unlock() GCounter.Incr("TOTAL_MemLock") - - m.mux.Lock() - m.waiting-- - dlog(cfg.opt.DebugMemlim, "NewMemLock gotSLOT seg.Id='%s' m.waiting=%d who='%s'", item.segment.Id, m.waiting, who) - m.mux.Unlock() + dlog(cfg.opt.DebugMemlim, "NewMemLock gotSLOT seg.Id='%s' who='%s'", item.segment.Id, who) return true } // end func memlim.MemLockWait -// MemReturn removes the item from mem and returns the slot to the chan -// it is called after the item has been uploaded or written to cache -// it is also called if no upload and is written to cache +// MemReturn gives the item's slot back. It is called after the item has been +// uploaded, written to cache, or retired. Calling it for an item that holds no +// slot (or calling it twice) is logged and does nothing. func (m *MemLimiter) MemReturn(who string, item *segmentChanItem) { - //dlog(cfg.opt.DebugMemlim, "MemReturn free seg.Id='%s' who='%s'", item.segment.Id, who) - item.mux.RLock() - if item.memlocked == 0 { + m.mux.Lock() + if !m.memdata[item] { + m.mux.Unlock() dlog(always, "MemReturn called on non-memlocked item seg.Id='%s' who='%s'", item.segment.Id, who) - item.mux.RUnlock() return // not memlocked, nothing to do } - item.mux.RUnlock() - - // remove map entry from mem - m.mux.Lock() delete(m.memdata, item) + item.mux.Lock() + item.memlocked = 0 + item.mux.Unlock() m.mux.Unlock() // return the slot @@ -156,12 +173,8 @@ func (m *MemLimiter) MemReturn(who string, item *segmentChanItem) { case m.memchan <- struct{}{}: // return mem slot into chan GCounter.Incr("TOTAL_MemReturned") default: - // wtf chan is full?? that's a bug! - // this is a bug! we should never return a slot to a full chan! - log.Fatalf("ERROR on MemReturn chan is full seg.Id='%s' who='%s'", item.segment.Id, who) + // only one owner per slot, so the chan can't be full here + dlog(always, "BUG: MemReturn: slot chan is full seg.Id='%s' who='%s'", item.segment.Id, who) } - item.mux.Lock() - item.memlocked-- - item.mux.Unlock() dlog(cfg.opt.DebugMemlim, "MemReturned seg.Id='%s' who='%s'", item.segment.Id, who) } // end func memlim.MemReturn diff --git a/MemLimit_test.go b/MemLimit_test.go index 0d1226a..9180cd8 100644 --- a/MemLimit_test.go +++ b/MemLimit_test.go @@ -47,7 +47,7 @@ func TestMemLockWaitAbortsOnStopAndRollsBack(t *gotesting.T) { t.Fatal("MemLockWait did not return after stop was closed") } m.mux.RLock() - waiting, inData := m.waiting, m.memdata[waiter] + waiting, inData := m.waiting.Load(), m.memdata[waiter] m.mux.RUnlock() if waiting != 0 || inData || waiter.memlocked != 0 { t.Fatalf("no rollback: waiting=%d memdata=%t memlocked=%d", waiting, inData, waiter.memlocked) @@ -62,3 +62,74 @@ func TestMemLockWaitAbortsOnStopAndRollsBack(t *gotesting.T) { t.Fatalf("used slots = %d, want 1", used) } } + +// Plan 3.8: MemReturn checked memlocked, then returned the slot. Two concurrent +// returns of one item both passed the check and handed back two slots. +func TestMemReturnTwiceReturnsOneSlot(t *gotesting.T) { + testCfg() + if GCounter == nil { + GCounter = NewCounter(10) + } + const slots = 64 + m := NewMemLimiter(slots) + for i := 0; i < 500; i++ { + it := testItem("r") + if !m.MemLockWait(it, "test", nil) { + t.Fatal("lock failed with free slots") + } + start := make(chan struct{}) + done := make(chan struct{}, 2) + for j := 0; j < 2; j++ { + go func() { <-start; m.MemReturn("test", it); done <- struct{}{} }() + } + close(start) + <-done + <-done + if used, _ := m.Usage(); used != 0 { + t.Fatalf("round %d: used slots = %d after returning one item twice, want 0", i, used) + } + if got := len(m.memchan); got != slots { + t.Fatalf("round %d: %d free slots, want %d", i, got, slots) + } + if it.memlocked != 0 { + t.Fatalf("round %d: memlocked = %d, want 0", i, it.memlocked) + } + } +} + +// Plan 3.8: a second MemLockWait on an item whose first lock was still waiting +// called os.Exit(1). It now waits for the first one and shares its slot. +func TestMemLockWaitSameItemTwiceSharesOneSlot(t *gotesting.T) { + testCfg() + if GCounter == nil { + GCounter = NewCounter(10) + } + m := NewMemLimiter(1) + holder, it := testItem("holder"), testItem("it") + m.MemLockWait(holder, "test", nil) + + res := make(chan bool, 2) + go func() { res <- m.MemLockWait(it, "first", nil) }() + time.Sleep(100 * time.Millisecond) // let the first one start waiting + go func() { res <- m.MemLockWait(it, "second", nil) }() + time.Sleep(100 * time.Millisecond) + + m.MemReturn("test", holder) + for i := 0; i < 2; i++ { + select { + case ok := <-res: + if !ok { + t.Fatal("lock reported no slot") + } + case <-time.After(2 * time.Second): + t.Fatal("second MemLockWait on the same item never returned") + } + } + if used, _ := m.Usage(); used != 1 { + t.Fatalf("used slots = %d, want 1", used) + } + m.MemReturn("test", it) + if used, _ := m.Usage(); used != 0 { + t.Fatalf("used slots = %d after return, want 0", used) + } +} diff --git a/review-the-codebase-for-jolly-petal.md b/review-the-codebase-for-jolly-petal.md index 207cdf2..c511946 100644 --- a/review-the-codebase-for-jolly-petal.md +++ b/review-the-codebase-for-jolly-petal.md @@ -267,6 +267,15 @@ Written under `globalmux` in each provider's boot goroutine (`Workers.go:65-81`) **Fix:** set `memdata` and `memlocked` together, under one lock, after the slot is acquired. Make a second `MemReturn` a logged no-op instead of a fatal error. Base `MemAvail` on `len(m.memchan)`. +**Result (2026-10-18):** fixed, except `MemAvail` (kept on purpose, see below). +- `memdata` is now the only record of who owns a slot. It changes together with `item.memlocked` (now 0/1), both under `m.mux`, and only after the slot was taken. Lock order is `m.mux` → `item.mux`; no caller holds `item.mux` when calling `MemLockWait`/`MemReturn`. +- `MemReturn` checks and removes ownership in one critical section. A second return (or one for an item without a slot) is a logged no-op; the `log.Fatalf` on a full chan is now a logged `BUG`. +- A second `MemLockWait` on an item still waiting (was `os.Exit(1)`) now waits for the first one and shares its slot. +- **`MemAvail` unchanged in meaning.** Gating `pushDL` on free slots made a `-mem 4` run 25% slower (38 s → 50 s, twice, both orders): each freed slot waited for the next divider pass before a download was queued for it. The old test ("fewer than `mem_max` callers waiting") only bounds how many downloads wait in `MemLockWait`, and they hold no memory. `waiting` is now an `atomic.Int64`, so `MemAvail` takes no lock: `pushDL` calls it while holding `item.mux`, which would otherwise invert the new lock order. +- Tests: `TestMemReturnTwiceReturnsOneSlot` (old code: `log.Fatalf` "chan is full") and `TestMemLockWaitSameItemTwiceSharesOneSlot` (old code: `os.Exit(1)`). +- Verified with `-race` against the local servers: A→B, same group, slot leak, `-mem 4` (39 s, old build 38 s), kill B during uploads (ends in 32 s). No races, no "non-memlocked" warnings. +- **Noted:** with `-mem 4` and 4 connections per server, uploads run at about 2 articles/s (~65 KiB/s) to the local server on both builds. This is the race build, but it looks slow; worth a look with a normal build later. + ### 3.9 `s.preBoot`/`s.active` guarded by two mutexes (corrected; fix added) `Session.go:170-178`/`:520-524` use `s.mux`; `Workers.go:13-25` uses `globalmux`. `LaunchSession` and `GoBootWorkers` run in the same goroutine (`Session.go:369`), so they don't race each other. The real race is `webserver.go:854` reading `active` under `globalmux` while `closeSession` writes it under `s.mux`. The nil dereference at `Workers.go:15` and the early returns before the `defer` at `:27` are unreachable (`LaunchSession` rejects a double launch at `:171`). From 2b1bf9b8a19a1ba792ee96212eea8089092e0400 Mon Sep 17 00:00:00 2001 From: go-while Date: Sun, 18 Oct 2026 09:33:19 +0000 Subject: [PATCH 129/152] Guard session preBoot/active with s.mux only GoBootWorkers wrote them under globalmux while LaunchSession and closeSession use s.mux, and the web stats poll read active with no session lock. All now use s.mux; the poll reads active together with providerList. waitDivider.Done is deferred before the early returns. Co-Authored-By: Claude Opus 5.5 --- Session.go | 4 ++-- Workers.go | 16 ++++++++-------- review-the-codebase-for-jolly-petal.md | 2 ++ webserver.go | 6 +++++- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/Session.go b/Session.go index f2d755a..7bd4813 100644 --- a/Session.go +++ b/Session.go @@ -75,8 +75,8 @@ type SESSION struct { workDone atomic.Uint64 // items finished by any check/downs/reups routine, ok or not (divider watchdog) //memDL map[string]map[*string]*segmentChanItem // (map[provider.group]map[ptr to segment.Id]ptr to Item) --- with -checkfirst queues items here. map will be create in GoWorkDivider() //memUP map[string]map[*string]*segmentChanItem // TODO: (map[provider.group]map[ptr to segment.Id]ptr to Item) --- with -uploadlater process uploads after downloads (possible only with cacheON) map will be create in GoWorkDivider() - preBoot bool // will be set to false before workers start - active bool // will be set to true when processing starts // REVIEW!TODO!FIXME: set to false is missing!! + preBoot bool // true from LaunchSession until GoBootWorkers starts the workers; guarded by mux + active bool // true while the workers run (GoBootWorkers until closeSession); guarded by mux preparationStartTime time.Time // flag // time when the session was started segmentCheckStartTime time.Time // flag // time when the segment check started segmentCheckEndTime time.Time // flag // time when the segment check ended diff --git a/Workers.go b/Workers.go index b730fbb..d28af2e 100644 --- a/Workers.go +++ b/Workers.go @@ -12,21 +12,21 @@ import ( // GoBootWorkers boots up all workers for a session func (s *SESSION) GoBootWorkers(waitDivider *sync.WaitGroup, workerWGconnReady *sync.WaitGroup, waitWorker *sync.WaitGroup, waitPool *sync.WaitGroup, byteSize int64) { - globalmux.Lock() + defer waitDivider.Done() // allows WorkDivider to run when this releases, also on the early returns if s == nil { - dlog(always, "ERROR in GoBootWorkers: session is nil... preBoot=%t ", s.preBoot) - globalmux.Unlock() + dlog(always, "ERROR in GoBootWorkers: session is nil") return - } else if s.active || !s.preBoot { + } + // preBoot/active are guarded by s.mux, like in LaunchSession and closeSession + s.mux.Lock() + if s.active || !s.preBoot { dlog(always, "ERROR in GoBootWorkers: session %d already booted... s.preBoot=%t s.active=%t", s.sessId, s.preBoot, s.active) - globalmux.Unlock() + s.mux.Unlock() return } s.preBoot = false s.active = true - globalmux.Unlock() - - defer waitDivider.Done() // allows WorkDivider to run when this releases + s.mux.Unlock() if cacheON && cfg.opt.CheckCacheOnBoot { cached := 0 diff --git a/review-the-codebase-for-jolly-petal.md b/review-the-codebase-for-jolly-petal.md index c511946..efad38f 100644 --- a/review-the-codebase-for-jolly-petal.md +++ b/review-the-codebase-for-jolly-petal.md @@ -282,6 +282,8 @@ Written under `globalmux` in each provider's boot goroutine (`Workers.go:65-81`) **Fix:** guard both fields only with `s.mux` (or make them `atomic.Bool`), and read them that way in `webserver.go`. Move `defer waitDivider.Done()` above the early returns anyway; it costs nothing. +**Result (2026-10-18):** fixed. `GoBootWorkers` sets both fields under `s.mux` (no more `globalmux`), `defer waitDivider.Done()` is first, and the `s == nil` branch no longer dereferences `s` in its log line. `collectStats` reads `active` together with `providerList` under one `session.mux.RLock()`. Lock order there stays `globalmux` → `webProcessor.mux` → `session.mux`; nothing takes `globalmux` while holding a session's `mux`. The race detector never caught this one in the web runs (the write in `closeSession` and the poll never overlapped), so the check is: `-race` CLI A→B clean, web race run with both jobs completed, and the only 13 remaining reports are the job-field races in `webserver.go` (4.2). + ### 3.10 `WaitGroup.Add` after `Wait` has started (corrected) and a boot hang (new) `NewConnPool` (via `loadProviderList`, `Session.go:290`) spawns `Speedmeter` and `WatchOpenConnsThread`, which call `workerWGconnReady.Wait()` (`ConnPool.go:572`, `:802`) before the `Add(1)` loop at `Session.go:356-366`. `Wait()` can return early, which is harmless. **Dropped:** the "WaitGroup is reused" panic; it needs reuse after reaching 0, and this WaitGroup is local and never reused. At most `-race` reports it. diff --git a/webserver.go b/webserver.go index b86107f..e7af721 100644 --- a/webserver.go +++ b/webserver.go @@ -858,12 +858,16 @@ func collectStats() WebStats { defer webProcessor.mux.RUnlock() // Iterate through all active sessions for _, session := range webProcessor.sessMap { - if session == nil || !session.active { + if session == nil { continue } session.mux.RLock() + active := session.active // written under session.mux by GoBootWorkers and closeSession providerList := session.providerList // published under session.mux by loadProviderList session.mux.RUnlock() + if !active { + continue + } for _, prov := range providerList { // KillConnPool sets prov.ConnPool = nil and the workers update the // counters under prov.mux: copy everything under one read lock From 23454e85661ee42465ec2701d4a4cfd2f2a73b9c Mon Sep 17 00:00:00 2001 From: go-while Date: Sun, 18 Oct 2026 09:43:34 +0000 Subject: [PATCH 130/152] Bound the NNTP handshake and CAPABILITIES; accept 201 greetings connect() read the greeting and AUTHINFO replies with no deadline (UseNoDeadline makes ExtendConn a no-op), so a server that accepts and then stays silent blocked GetConn forever. The handshake now has a 30 s deadline, cleared before the conn is handed out; a timeout counts as a failed dial so the provider can be marked down for the session. checkCapabilities held provider.mux across two unbounded reads. It now has its own deadline and takes the lock only to store the result. The greeting was read with ReadCodeLine(200): a 201 greeting came back with an error and, without a login, the conn was leaked. Accept any 2xx. clearSession read len(sessMap) without the processor lock. Co-Authored-By: Claude Opus 5.5 --- ConnPool.go | 41 +++++++-- ConnPool_test.go | 110 +++++++++++++++++++++++++ NetConn.go | 26 ++++-- Session.go | 5 +- review-the-codebase-for-jolly-petal.md | 10 +++ 5 files changed, 174 insertions(+), 18 deletions(-) diff --git a/ConnPool.go b/ConnPool.go index 66be5ae..ac97a60 100644 --- a/ConnPool.go +++ b/ConnPool.go @@ -19,6 +19,7 @@ import ( "math/rand" "net" "net/textproto" + "os" "strings" "sync" "syscall" @@ -50,6 +51,11 @@ const ( var ( //noDeadLine time.Time // used as zero value! readDeadConn = time.Unix(1, 0) // no syscall needed. value is 1 second after "0" (1970-01-01 00:00:01 UTC) + + // connHandshakeTimeout bounds the greeting, login and CAPABILITIES exchange of a new conn. + // UseNoDeadline turns ExtendConn into a no-op, so without this a server that accepts the + // TCP connection and then sends nothing blocks the caller forever. A var so tests can shorten it. + connHandshakeTimeout = 30 * time.Second ) // holds an active connection to share around @@ -255,26 +261,43 @@ func (c *ConnPool) connect() (connitem *ConnItem, err error) { } // end switch dlog(cfg.opt.DebugConnPool, "ConPool connect raw conn established '%s' took=(%d ms)", c.provider.Name, time.Since(start).Milliseconds()) + // bound the greeting and login; cleared again once the conn is handed out + conn.SetDeadline(time.Now().Add(connHandshakeTimeout)) + defer func() { + if err != nil && errors.Is(err, os.ErrDeadlineExceeded) { + // accepted but silent: as useless as a refused dial, so GetConn may mark the provider down + err = fmt.Errorf("%w: handshake timed out after %v: %w", errDial, connHandshakeTimeout, err) + } + }() + time0 := time.Now() srvtp := textproto.NewConn(conn) dlog(cfg.opt.DebugConnPool, "ConnPool connect textproto.NewConn took=(%d ms) '%s'", time.Since(time0).Milliseconds(), c.provider.Name) time00 := time.Now() - code, msg, err := srvtp.ReadCodeLine(200) - dlog(cfg.opt.DebugConnPool, "ConnPool connect ReadCodeLine(20) took=(%d ms) '%s' code=%d msg='%s' err='%v'", time.Since(time00).Milliseconds(), c.provider.Name, code, msg, err) + // expectCode 2 accepts any 2xx. It was 200, so a "201 no posting" greeting came back + // with an error: NewConn then failed and the conn leaked when no login was configured. + code, msg, err := srvtp.ReadCodeLine(2) + dlog(cfg.opt.DebugConnPool, "ConnPool connect ReadCodeLine(2) took=(%d ms) '%s' code=%d msg='%s' err='%v'", time.Since(time00).Milliseconds(), c.provider.Name, code, msg, err) time000 := time.Now() - if code < nntpWelcomeCodeMin || code > nntpWelcomeCodeMax { // not a valid nntp welcome code - if conn != nil { - conn.Close() - } + if err != nil || code < nntpWelcomeCodeMin || code > nntpWelcomeCodeMax { // not a valid nntp welcome code + conn.Close() dlog(always, "ERROR in connect '%s' welcome banner code=%d err='%v'", c.provider.Name, code, err) - return nil, err + if err == nil { + err = fmt.Errorf("unexpected welcome banner code=%d msg='%s'", code, msg) + } + return nil, fmt.Errorf("error connect '%s': %w", c.provider.Name, err) } dlog(cfg.opt.DebugConnPool, "ConnPool connect welcome '%s' time0=(%d ms) time00=(%d ms) time000=(%d ms)", c.provider.Name, time.Since(time0).Milliseconds(), time.Since(time00).Milliseconds(), time.Since(time000).Milliseconds()) if !c.wants_auth { - return &ConnItem{c: c, connid: c.newconnid(), created: time.Now(), srvtp: srvtp, conn: conn, writer: bufio.NewWriter(conn)}, err + connitem = &ConnItem{c: c, connid: c.newconnid(), created: time.Now(), srvtp: srvtp, conn: conn, writer: bufio.NewWriter(conn)} + } else { + connitem, err = c.auth(srvtp, conn, start) } - return c.auth(srvtp, conn, start) + if connitem != nil && connitem.conn != nil { + connitem.conn.SetDeadline(time.Time{}) // no deadline from here on, see ExtendConn / UseNoDeadline + } + return connitem, err } // end func connect func (c *ConnPool) auth(srvtp *textproto.Conn, conn net.Conn, start time.Time) (connitem *ConnItem, err error) { diff --git a/ConnPool_test.go b/ConnPool_test.go index 90bdbb9..e910b3d 100644 --- a/ConnPool_test.go +++ b/ConnPool_test.go @@ -1,7 +1,10 @@ package main import ( + "errors" + "io" "net" + "net/textproto" gotesting "testing" // Workers.go declares a const named testing "time" @@ -52,3 +55,110 @@ func TestCloseConnNilSafeAndDecrementsOnce(t *gotesting.T) { t.Fatalf("provider.OpenConns: want 0, got %d", prov.OpenConns) } } + +func testPool(t *gotesting.T, addr string) *ConnPool { + t.Helper() + testCfg() + if GCounter == nil { + GCounter = NewCounter(10) + } + prov := &Provider{Name: "p", TCPMode: "tcp", MaxConns: 1, mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "p"}} + c := &ConnPool{provider: prov, rserver: addr, pool: make(chan *ConnItem, 1), + mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "pool-p"}} + prov.ConnPool = c + return c +} + +// serve accepts one conn, writes greeting (if any), then keeps it open and silent. +func serve(t *gotesting.T, greeting string) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) + go func() { + conn, err := ln.Accept() + if err != nil { + return + } + t.Cleanup(func() { conn.Close() }) + if greeting != "" { + io.WriteString(conn, greeting) + } + io.Copy(io.Discard, conn) + }() + return ln.Addr().String() +} + +func shortHandshake(t *gotesting.T) { + old := connHandshakeTimeout + connHandshakeTimeout = 300 * time.Millisecond + t.Cleanup(func() { connHandshakeTimeout = old }) +} + +// Plan 3.14: the greeting was read with no deadline (UseNoDeadline), so a server +// that accepts and stays silent blocked connect, and GetConn, forever. It now +// times out and counts as a failed dial, so GetConn can mark the provider down. +func TestConnectSilentServerTimesOut(t *gotesting.T) { + shortHandshake(t) + c := testPool(t, serve(t, "")) + type res struct { + item *ConnItem + err error + } + done := make(chan res, 1) + go func() { it, err := c.connect(); done <- res{it, err} }() + select { + case r := <-done: + if r.item != nil || !errors.Is(r.err, errDial) { + t.Fatalf("want no conn and an errDial error, got item=%v err=%v", r.item, r.err) + } + case <-time.After(5 * time.Second): // connect sleeps up to 1s before dialing + t.Fatal("connect blocked on a silent server") + } +} + +// Plan 3.14: the greeting was read with ReadCodeLine(200), so "201 no posting" +// came back as an error and the conn was never used (and leaked). +func TestConnectAccepts201Greeting(t *gotesting.T) { + shortHandshake(t) + c := testPool(t, serve(t, "201 no posting\r\n")) + it, err := c.connect() + if err != nil || it == nil { + t.Fatalf("201 greeting: want a conn and no error, got item=%v err=%v", it, err) + } + it.conn.Close() +} + +// Plan 3.14: checkCapabilities held provider.mux across two reads with no +// deadline. A server that never answers CAPABILITIES held the lock forever. +func TestCheckCapabilitiesTimesOutWithoutHoldingLock(t *gotesting.T) { + shortHandshake(t) + c := testPool(t, "") + c.openConns = 1 + a, b := net.Pipe() + defer b.Close() + go io.Copy(io.Discard, b) // reads the command, never answers + item := &ConnItem{c: c, conn: a, srvtp: textproto.NewConn(a)} + + done := make(chan error, 1) + go func() { done <- checkCapabilities(c.provider, item) }() + + time.Sleep(100 * time.Millisecond) + locked := make(chan struct{}) + go func() { c.provider.mux.Lock(); c.provider.mux.Unlock(); close(locked) }() + select { + case <-locked: + case <-time.After(5 * time.Second): + t.Fatal("provider.mux is held while waiting for the server") + } + select { + case err := <-done: + if err == nil { + t.Fatal("want an error from a server that never answers") + } + case <-time.After(5 * time.Second): + t.Fatal("checkCapabilities blocked on a silent server") + } +} diff --git a/NetConn.go b/NetConn.go index bdef636..e2a50d0 100644 --- a/NetConn.go +++ b/NetConn.go @@ -1018,10 +1018,13 @@ func checkCapabilities(provider *Provider, connitem *ConnItem) error { } rcode := 101 // expected - provider.mux.Lock() // mutex #9b71 checkCapabilities - defer provider.mux.Unlock() // mutex #9b71 checkCapabilities defer provider.ConnPool.CloseConn(connitem, nil) + // The conn has no read deadline (UseNoDeadline): give this exchange its own, so a + // server that stops answering can't block the boot. The conn is closed afterwards, + // so the deadline never reaches a reused conn. + connitem.conn.SetDeadline(time.Now().Add(connHandshakeTimeout)) + // No lock is held during the network I/O (it used to hold provider.mux across both reads). if !msg2srv(connitem.conn, "CAPABILITIES") { return fmt.Errorf("error '%s' checkCapabilities", provider.Name) } @@ -1033,24 +1036,25 @@ func checkCapabilities(provider *Provider, connitem *ConnItem) error { if err != nil { return fmt.Errorf("error '%s' checkCapabilities ReadDotLines err='%v'", provider.Name, err) } + var check, ihave, post, stream bool setpostProviders := 0 //dlog( "Read %d CAPAS from Provider %s", len(lines), provider.Name) for _, capability := range lines { //dlog( "CAPAS @ %s: %s", provider.Name, capability) switch strings.ToLower(capability) { case "check": - provider.capabilities.check = true + check = true case "ihave": - provider.capabilities.ihave = true + ihave = true setpostProviders++ case "post": - provider.capabilities.post = true + post = true setpostProviders++ case "takethis": - provider.capabilities.stream = true + stream = true setpostProviders++ case "streaming": - provider.capabilities.stream = true + stream = true setpostProviders++ } } @@ -1060,8 +1064,14 @@ func checkCapabilities(provider *Provider, connitem *ConnItem) error { if setpostProviders > 0 { GCounter.Incr("postProviders") } + provider.mux.Lock() // mutex #9b71 checkCapabilities + provider.capabilities.check = check + provider.capabilities.ihave = ihave + provider.capabilities.post = post + provider.capabilities.stream = stream // provider.NoUpload will be set to true if none capa is available - provider.NoUpload = (!provider.capabilities.ihave && !provider.capabilities.post && !provider.capabilities.stream) + provider.NoUpload = (!ihave && !post && !stream) + provider.mux.Unlock() // done return nil } // end func checkCapabilities diff --git a/Session.go b/Session.go index 7bd4813..9945103 100644 --- a/Session.go +++ b/Session.go @@ -486,7 +486,10 @@ func (p *PROCESSOR) newssid() uint64 { // If the session map exceeds the threshold, it removes the session from the processor map. // If the session is below the threshold, it closes the session partly. func (s *SESSION) clearSession() { - if len(s.proc.sessMap) > cfg.opt.SessThreshold { + s.proc.mux.RLock() + sessions := len(s.proc.sessMap) // the web job queue adds and removes sessions concurrently + s.proc.mux.RUnlock() + if sessions > cfg.opt.SessThreshold { s.proc.removeSession(s.sessId, s) // remove session from processor map return } diff --git a/review-the-codebase-for-jolly-petal.md b/review-the-codebase-for-jolly-petal.md index efad38f..fa50db2 100644 --- a/review-the-codebase-for-jolly-petal.md +++ b/review-the-codebase-for-jolly-petal.md @@ -346,6 +346,16 @@ Still races: - **`ConnPool.go:184` (worse than the draft said).** `provider.ConnPool = nil` runs in `KillConnPool` (`Session.go:387`) before `clearSession` (`:401`). Meanwhile `webserver.go:854-856` calls `prov.ConnPool.GetStats()` while `session.active` is still true. That is a reachable nil-pointer panic from a web poll. - `NetConn.go:987-989`: `checkCapabilities` holds `provider.mux` across two network reads with no deadline (`UseNoDeadline` makes `ExtendConn` a no-op). Add a deadline; don't hold the lock across I/O. +**Result (2026-10-18):** done. +- `clearSession` reads `len(sessMap)` under `p.mux.RLock()`. +- `ConnPool` nil panic: already fixed with the web wedge above (`9299fd8`): `collectStats` copies the pool pointer under `prov.mux` and skips a nil one. `KillConnPool` runs only after all workers have joined, and proxy sessions never call it, so no other reader can see the nil. +- `checkCapabilities` gives the exchange its own deadline (`connHandshakeTimeout`, 30 s), parses into locals with no lock held, and takes `provider.mux` only to store the result. +- **New, same cause:** `connect()` read the greeting and the AUTHINFO replies with no deadline either, so a server that accepts the TCP connection and then sends nothing blocked `GetConn` forever. The whole handshake now has the same 30 s deadline, cleared before the conn is handed out. A handshake timeout counts as `errDial`, so `GetConn` marks the provider down after `MaxConnErrors` tries (never in proxy mode). +- **New:** `connect()` read the greeting with `ReadCodeLine(200)`. A `201` greeting ("no posting", common on download-only servers) passed the range check but came back with an error, so without a login configured `NewConn` failed and leaked the conn. It now accepts any 2xx and closes the conn on every failure. +- Tests: `TestConnectSilentServerTimesOut`, `TestConnectAccepts201Greeting`, `TestCheckCapabilitiesTimesOutWithoutHoldingLock` (all three fail on the old code: two hang, one errors). +- Verified with `-race` against the local servers: B replaced by a listener that accepts and never answers: the old build was still hanging when killed at 180 s; the new one marks B down and ends at 105 s (3 × (30 s + 3 s retry sleep)). A→B (`-yenctest 4`), same group, B refused at boot (13 s), proxy login + STAT (223 / 430): unchanged, no races. +- **Still open:** in-session reads (ARTICLE/STAT/IHAVE/POST) still have no deadline because `UseNoDeadline = true` turns `ExtendConn` into a no-op, and pooled conns only get a deadline through `ExtendConn` on the shared-conn path. A server that goes silent mid-session can still block a routine until the session's stop. Turning deadlines on needs a deadline set before every command (parked conns would otherwise hit a stale one), so it is a design change, not a flag flip. + **Not races (dropped):** - `Workers.go:129`, `Routines.go:355-368`, `Workers.go:616-618`: written in the boot goroutine before the workers start, so the writes happen first. - `writeCsvFile` (`Utils.go:444-451`): runs after the workers are joined. From 4f1aeb271adc975e0903d79ff11aaa40d6bae6b7 Mon Sep 17 00:00:00 2001 From: go-while Date: Sun, 18 Oct 2026 14:03:43 +0000 Subject: [PATCH 131/152] Web: report bind errors, copy jobs instead of sharing them, no scans under locks StartWebServer binds and loads the TLS cert before serving and returns the error; main exits on it (a busy port used to leave the process running without a web server). The job queue moves to webjobs.go. Readers get copies and never write. Each job creates and keeps its own session (SessionID is set), so nothing matches sessions by path. GoWorkDivider publishes its per-pass counts in s.progress; running jobs read them and finished jobs keep the final counts, also when nobody polled. AddJob parses outside the lock and rejects broken NZBs. collectStats no longer holds globalmux or walks segment lists. New waiting state, panic recovery on the job goroutine, queue caps, 404/409 status codes. Per-job settings (D2): the config endpoints return 501 with the startup settings, /upload ignores the fields, and the dialogs are read-only. Co-Authored-By: Claude Opus 5.5 --- Session.go | 10 +- Workers.go | 8 + main.go | 33 +- review-the-codebase-for-jolly-petal.md | 17 + web/templates/jobs.html | 89 +-- webjobs.go | 354 +++++++++++ webjobs_test.go | 125 ++++ webserver.go | 795 +++++-------------------- 8 files changed, 689 insertions(+), 742 deletions(-) create mode 100644 webjobs.go create mode 100644 webjobs_test.go diff --git a/Session.go b/Session.go index 9945103..2df16a1 100644 --- a/Session.go +++ b/Session.go @@ -73,6 +73,7 @@ type SESSION struct { segmentChansReups map[string]chan *segmentChanItem // map[provider.Group]chan checkers map[string]*checkerGroup // live check routines per provider.Group, built with the channels workDone atomic.Uint64 // items finished by any check/downs/reups routine, ok or not (divider watchdog) + progress sessionProgress // counts published by GoWorkDivider for the web UI //memDL map[string]map[*string]*segmentChanItem // (map[provider.group]map[ptr to segment.Id]ptr to Item) --- with -checkfirst queues items here. map will be create in GoWorkDivider() //memUP map[string]map[*string]*segmentChanItem // TODO: (map[provider.group]map[ptr to segment.Id]ptr to Item) --- with -uploadlater process uploads after downloads (possible only with cacheON) map will be create in GoWorkDivider() preBoot bool // true from LaunchSession until GoBootWorkers starts the workers; guarded by mux @@ -219,7 +220,7 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s if s.nzbFile != nil { // nzbfile is still open and loaded, pass - } else if s.nzbFile == nil && nzbfilepath != "" { + } else if s.nzbFile == nil && s.nzbPath != "" { // we have a nzbfilepath and no nzbFile loaded yet // nzbfile has not been opened and parsed before (or has been closed) nzbfile, err := loadNzbFile(s.nzbPath) @@ -290,6 +291,7 @@ func (p *PROCESSOR) LaunchSession(s *SESSION, nzbfilepath string, waitSession *s s.mux.Lock() s.segmentList = segmentList s.mux.Unlock() + s.progress.total.Store(uint64(len(segmentList))) mibsize := float64(nzbfile.Bytes) / 1024 / 1024 artsize := mibsize / float64(len(segmentList)) * 1024 @@ -438,6 +440,12 @@ func (p *PROCESSOR) SetDirRefresh(seconds int64) { // private PROCESSOR/SESSION functions +// sessionProgress holds the counts of GoWorkDivider's last pass over the segment +// list. The web UI reads them instead of walking segmentList under locks (plan 4.3). +type sessionProgress struct { + total, checked, done, dead, cached, downloaded, uploaded, notarget atomic.Uint64 +} + func (p *PROCESSOR) newSession(nzbName string) (uint64, *SESSION) { // create session, every nzbfile will have its own session // this is called from LaunchSession and watchDirThread diff --git a/Workers.go b/Workers.go index d28af2e..f54248c 100644 --- a/Workers.go +++ b/Workers.go @@ -1220,6 +1220,14 @@ forever: dlog(always, " | Segment Check Done: took='%.1f sec'", took.Seconds()) } + s.progress.checked.Store(checked) + s.progress.done.Store(done) + s.progress.dead.Store(dead) + s.progress.cached.Store(cached) + s.progress.downloaded.Store(isdl) + s.progress.uploaded.Store(isup) + s.progress.notarget.Store(notarget) + // workDone also moves on failed attempts: slow failing work (a dead provider) is not stuck if progress := checked + isdl + isup + isyenc + cached + done + dead + fails + notarget + s.workDone.Load(); progress != lastProgress { lastProgress, lastProgressAt = progress, time.Now() diff --git a/main.go b/main.go index a3afffd..738ff00 100644 --- a/main.go +++ b/main.go @@ -113,29 +113,20 @@ func main() { dlog(cfg.opt.Debug, "main: wg.Wait()") wg.Wait() } else if webServerEnabled { - // boot web server + // boot web server: bind first, so a busy port or a bad cert exits right away (plan 4.1) + if err := StartWebServer(webServerAddr, thisProcessor); err != nil { + dlog(always, "ERROR StartWebServer: err='%v'", err) + os.Exit(1) + } + // a base session loads the providers for the web server, like the proxy's + s := &SESSION{ + proc: thisProcessor, + mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "WEBSESSION"}, + stopChan: make(chan struct{}), // closed only by s.stop() + proxy: true, // Set proxy flag to initialize providers without processing NZB + } wg.Add(1) // waitSession go func(wg *sync.WaitGroup) { - globalmux.Lock() - // set up a session to load providers for web server - // similar to proxy server setup - dlog(cfg.opt.Debug, "pre:thisProcessor.LaunchSession: web server") - s := &SESSION{ - proc: thisProcessor, - mux: &loggedrwmutex.LoggedSyncRWMutex{Name: "WEBSESSION"}, - stopChan: make(chan struct{}), // closed only by s.stop() - proxy: true, // Set proxy flag to initialize providers without processing NZB - } - webProcessor = thisProcessor - webWg = wg - globalmux.Unlock() - go func() { - if err := StartWebServer(webServerAddr, webProcessor, wg); err != nil { - dlog(always, "ERROR StartWebServer: err='%v'", err) - os.Exit(1) - } - }() - // Initialize session to load providers if err := thisProcessor.LaunchSession(s, "", wg); err != nil { dlog(always, "ERROR NewProcessor Boot Web Server: err='%v'", err) os.Exit(1) diff --git a/review-the-codebase-for-jolly-petal.md b/review-the-codebase-for-jolly-petal.md index fa50db2..2f93a6b 100644 --- a/review-the-codebase-for-jolly-petal.md +++ b/review-the-codebase-for-jolly-petal.md @@ -371,6 +371,8 @@ Still races: **Fix:** `net.Listen` synchronously and return its error; `tls.LoadX509KeyPair` up front when `-webssl`; then `go srv.Serve(ln)`. +**Result (2026-10-18):** fixed. `StartWebServer` loads the certificate, binds, and only then serves in the background (`srv.Serve`/`ServeTLS` on its own `http.Server`, kept in `webSrv` for a later `Shutdown`). `main` calls it synchronously, before the web base session starts. Busy port: the old build kept running without a web server (killed at 20 s); the new one exits 1 at once with the error. Missing certificate: exits 1 with the file error. `TestStartWebServerReportsBindError`. + ### 4.2 Shared job state written under a read lock `ListJobs` (`webserver.go:178-218`, writes at `:194-200`) and `collectStats` (writes at `:926-933` under `jobQueue.mu.RLock` from `:897`) both modify `*QueuedJob` under `RLock`. `ListJobs` returns live pointers that are marshalled after the lock is dropped (`:652-655`), while the completion goroutine writes `Status`/`Error`/`FinishedAt` (`:293-302`). @@ -379,6 +381,15 @@ Still races: **Fix:** readers must not modify jobs. The job updates its own progress (it currently advances only when someone polls, so it stays 0 with no browser open, even after completion). Return copies (`[]QueuedJob`) under the lock. +**Result (2026-10-18):** fixed; the job queue moved to `webjobs.go`. +- One `jobs` slice in insertion order (no `running`/`completed` lists), every field under `jq.mu`. `ListJobs`/`GetJob`/`runningJobs` return copies; nothing that reads writes. +- Each job holds its own session (`runJob` creates it with `newSession` and passes it to `LaunchSession`), and `SessionID` is set. No matching by path any more. `LaunchSession` now loads a supplied session's NZB from `s.nzbPath`. +- Progress: `GoWorkDivider` publishes its per-pass counts in `s.progress` (atomics). A running job's copy reads them; `finishLocked` stores the final counts, so a finished job shows its result even if nobody polled. +- `extractSessionStats` (and its unlocked `segmentList` check) is gone. +- Web race run (2 jobs, `/api/jobs` + `/api/stats` every 50 ms): **0 races** (was 13–26, all here). Both jobs completed, progress 1.0, session IDs 1 and 2. +- Tests: `TestListJobsReturnsCopies`, `TestJobProgressFromItsSession`, `TestJobStateErrors`. +- **New engine finding (not fixed):** a successful upload sets `uploadedTo` and deletes `missingOn`, but never sets `availableOn` (commented out in `Routines.go`). The divider's "done" test needs `availableOn` on every provider, so uploaded items are never "done". Each pass then counts them as "no upload target" (`notarget`), and every successful refresh ends through close case #8 instead of #0/#4. The run still ends correctly. `len(item.availableOn)` also counts `false` entries (`pushDL` ranges over it checking the value). The web counts uploaded items as done. + ### 4.3 Locks held across long work - `AddJob` (`:117-147`) holds `jq.mu` while it opens, gunzips and parses the whole NZB (`Utils.go:67-124`). It swallows the parse error (`err == nil &&`), so a corrupt file is accepted with HTTP 200. The parse is then redone by `LaunchSession` (`Session.go:211`). @@ -387,6 +398,8 @@ Still races: **Fix:** parse outside the lock and reject parse errors with 400. Keep running counters on the session (atomics updated by the workers) and have the web layer read those instead of scanning. +**Result (2026-10-18):** fixed. `AddJob` parses before taking `jq.mu` and returns the parse error (400; the upload is deleted). `collectStats` no longer holds `globalmux` (it reads the `memlim` pointer under a brief `RLock`), copies the active sessions' provider lists under `p.mux`, then reads providers without it. Job progress comes from the session atomics; no segment list is walked. + ### 4.4 Login, CSRF, timeouts, uploads (corrected; needs D1) - **No authentication** on any route (`:512-525`). The default bind is `127.0.0.1:11911`, but `WEB_JOB_QUEUE.md:115-118` tells users to expose it on `:443`. Delete that example. @@ -415,6 +428,8 @@ Still races: - Status codes: not-found returns 400 at `:663`/`:678`, conflict returns 400 at `:678`/`:699`. Use 404/409. - No graceful shutdown: add `srv.Shutdown(ctx)` on signal. +**Partial result (2026-10-18), with 4.2:** `waiting` state (`StartJob` sets it; `runJob` sets `running`/`StartedAt` once it holds `processingMu`); `SessionID` assigned and used; panic recovery on the job goroutine (`launch`); a failed `LaunchSession` removes its session from `sessMap`; pending jobs capped at 1000 (503), finished jobs at 50; second `webProcessor`/`webWg` assignment and the no-op WaitGroup removed; 404/409 instead of 400. **Still open:** stop/cancel endpoint, D3 mode checks, graceful shutdown. + ### 4.6 Templates (corrected) - `jobs.html:231-235`: the "commented out" Config button is live. `html/template` strips the `` lines as JS comments and keeps the `if (canConfig)` block, so the button renders and the modal PATCHes at `:401-402`. Use `/* … */`, or remove it under D2. @@ -538,6 +553,8 @@ All three answered by the user on 2026-10-17. - Delete the commented-out block in `processJob` (`webserver.go:332-348`); it couldn't have worked (6.1). - Fix `WEB_JOB_QUEUE.md:105`/`:179` to say per-job settings are planned, not active. - Fix `jobs.html:231-235` so the button state is deliberate (disabled), not an accident of the `` comment (4.6). + +**Partial result (2026-10-18), with 4.2:** the three endpoints return 501 with the startup settings in the body; `/upload` ignores the config fields and says so (`note`); the default-config dialog shows the startup settings read-only with a note; both dialogs have disabled fields and Save buttons; the ⚙ button is rendered disabled on purpose; the dead `processJob` block and the queue's `defaultConfig` are gone. Each job's `Config` shows the startup settings it runs with. **Still open:** `WEB_JOB_QUEUE.md` (with 6.3). - The real implementation (6.1 refactor) stays a separate later change, after Phases 2–3. **D3: Startup mode combinations. DECIDED: reject with a clear error.** diff --git a/web/templates/jobs.html b/web/templates/jobs.html index c6ad046..dc89614 100644 --- a/web/templates/jobs.html +++ b/web/templates/jobs.html @@ -47,6 +47,8 @@
Configure Job
+
Per-job settings are not supported yet: jobs run with the settings NZBreX was started with.
+
@@ -77,10 +79,11 @@
Configure Job
+
- - + +
@@ -95,7 +98,8 @@
Default Job Configuration
-

These settings will be applied to all newly uploaded NZB files.

+
Per-job settings are not supported yet: every job runs with these startup settings.
+
@@ -126,10 +130,11 @@
Default Job Configuration
+
- - + +
@@ -192,6 +197,8 @@
Default Job Configuration
let timelineHtml = ''; if (job.Status === 'queued') { timelineHtml = `Queued: ${formatTime(job.AddedAt)}`; + } else if (job.Status === 'waiting') { + timelineHtml = `Waiting for the running job to finish`; } else if (job.Status === 'running') { const duration = formatDuration(job.StartedAt, new Date()); const progressPct = (job.Progress * 100).toFixed(2); @@ -228,11 +235,10 @@
Default Job Configuration
if (canStart) { html += ``; } - if (canDelete) { html += ``; } @@ -250,6 +256,7 @@
Default Job Configuration
function getBadgeClass(status) { switch(status) { case 'queued': return 'bg-warning text-dark'; + case 'waiting': return 'bg-secondary'; case 'running': return 'bg-info text-dark'; case 'completed': return 'bg-success'; case 'failed': return 'bg-danger'; @@ -327,7 +334,8 @@
Default Job Configuration
const response = await fetch('/api/jobs/default-config'); const data = await response.json(); - if (response.ok && data.config) { + // 501: per-job settings are not supported yet; the reply carries the startup settings + if (data.config) { const config = data.config; document.getElementById('defCheckOnly').checked = config.CheckOnly || false; document.getElementById('defCheckFirst').checked = config.CheckFirst || false; @@ -353,69 +361,6 @@
Default Job Configuration
if (modal) modal.hide(); } - // Save default config - async function saveDefaultConfig() { - const config = { - CheckOnly: document.getElementById('defCheckOnly').checked, - CheckFirst: document.getElementById('defCheckFirst').checked, - YencCRC: document.getElementById('defYencCRC').checked, - YencTest: parseInt(document.getElementById('defYencTest').value), - YencCpu: parseInt(document.getElementById('defYencCpu').value), - Cachedir: document.getElementById('defCachedir').value - }; - - try { - const response = await fetch('/api/jobs/default-config', { - method: 'PUT', - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify(config) - }); - - if (response.ok) { - showNotification('Default configuration saved successfully', 'success'); - closeDefaultConfigModal(); - } else { - const data = await response.json(); - showNotification('Failed to save: ' + (data.error || 'Unknown error'), 'error'); - } - } catch (error) { - showNotification('Error saving default config: ' + error.message, 'error'); - } - } - - async function saveConfig() { - if (!currentJobId) return; - - const config = { - CheckOnly: document.getElementById('cfgCheckOnly').checked, - CheckFirst: document.getElementById('cfgCheckFirst').checked, - YencCRC: document.getElementById('cfgYencCRC').checked, - YencTest: parseInt(document.getElementById('cfgYencTest').value), - YencCpu: parseInt(document.getElementById('cfgYencCpu').value), - Cachedir: document.getElementById('cfgCachedir').value, - }; - - try { - const response = await fetch(`/api/jobs/${currentJobId}/config`, { - method: 'PATCH', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(config) - }); - const result = await response.json(); - if (response.ok) { - console.log('Config updated:', result); - closeConfigModal(); - loadJobs(); - } else { - alert('Failed to update config: ' + result.error); - } - } catch (error) { - alert('Failed to update config: ' + error.message); - } - } - function formatDuration(start, end) { if (!start || !end) return 'N/A'; diff --git a/webjobs.go b/webjobs.go new file mode 100644 index 0000000..70f95e2 --- /dev/null +++ b/webjobs.go @@ -0,0 +1,354 @@ +package main + +import ( + "errors" + "fmt" + "runtime/debug" + "sync" + "time" +) + +// Job states. A job moves queued → waiting → running → completed/failed. +// waiting: started, but another job still holds processingMu. +const ( + jobQueued = "queued" + jobWaiting = "waiting" + jobRunning = "running" + jobCompleted = "completed" + jobFailed = "failed" +) + +const ( + maxPendingJobs = 1000 // queued + waiting; AddJob refuses more + maxFinishedJobs = 50 // finished jobs kept for display; older ones are dropped +) + +var ( + errJobNotFound = errors.New("job not found") + errJobState = errors.New("job is in the wrong state") + errQueueFull = errors.New("job queue is full") +) + +// JobQueue manages the web jobs. Jobs run one at a time (processingMu), because +// sessions still share the global cfg. +// +// Locking: jq.mu guards every job field. Readers get copies (ListJobs, GetJob), +// never pointers. Lock order: jq.mu before PROCESSOR.mux (runJob calls +// newSession under jq.mu); nothing takes jq.mu while holding PROCESSOR.mux. +type JobQueue struct { + mu sync.RWMutex + jobs []*QueuedJob // every job, in the order it was added + nextID int + processingMu sync.Mutex // only one job runs at a time +} + +// QueuedJob is one uploaded NZB. The exported fields are the JSON the pages read. +type QueuedJob struct { + ID string + NZBPath string + NZBName string + Config JobConfig // the settings the job runs with: always the startup flags (D2) + Status string + AddedAt time.Time + StartedAt time.Time + FinishedAt time.Time + Error string + Progress float64 // 0..1: finished articles / total + TotalArticles uint64 `json:"total_articles"` + ArticlesProcessed uint64 `json:"articles_processed"` // finished: ok on all providers, dead, or nowhere to upload + ArticlesDone uint64 `json:"articles_done"` // ok on all providers, or uploaded + ArticlesCached uint64 `json:"articles_cached"` + ArticlesDead uint64 `json:"articles_dead"` + TotalSize int64 `json:"total_size"` // bytes + SessionID uint64 `json:"session_id"` + + session *SESSION // set when the job starts running; the job's own session, never matched by path +} + +// JobConfig holds per-job settings. They are not applied yet (D2): every job +// runs with the settings the process was started with. +type JobConfig struct { + CheckOnly bool + CheckFirst bool + YencCRC bool + YencTest int + YencCpu int + Cachedir string +} + +func NewJobQueue() *JobQueue { + return &JobQueue{nextID: 1} +} + +// startupJobConfig reports the settings every job runs with. +func startupJobConfig() JobConfig { + return JobConfig{ + CheckOnly: cfg.opt.CheckOnly, + CheckFirst: cfg.opt.CheckFirst, + YencCRC: cfg.opt.YencCRC, + YencTest: cfg.opt.YencTest, + YencCpu: cfg.opt.YencCpu, + Cachedir: cfg.opt.Cachedir, + } +} + +// AddJob parses the NZB and queues it. A file that doesn't parse is rejected. +// The parse runs before jq.mu is taken (it used to hold the lock for the whole parse). +func (jq *JobQueue) AddJob(nzbPath, nzbName string) (string, error) { + nzbfile, err := loadNzbFile(nzbPath) + if err != nil || nzbfile == nil { + if err == nil { + err = errors.New("empty nzb") + } + return "", fmt.Errorf("invalid NZB file: %w", err) + } + var totalArticles uint64 + for _, file := range nzbfile.Files { + totalArticles += uint64(file.TotalSegments) + } + job := &QueuedJob{ + NZBPath: nzbPath, + NZBName: nzbName, + Config: startupJobConfig(), + Status: jobQueued, + AddedAt: time.Now(), + TotalSize: nzbfile.Bytes, + TotalArticles: totalArticles, + } + + jq.mu.Lock() + defer jq.mu.Unlock() + pending := 0 + for _, j := range jq.jobs { + if j.Status == jobQueued || j.Status == jobWaiting { + pending++ + } + } + if pending >= maxPendingJobs { + return "", fmt.Errorf("%w: %d jobs are waiting", errQueueFull, pending) + } + job.ID = fmt.Sprintf("job-%d", jq.nextID) + jq.nextID++ + jq.jobs = append(jq.jobs, job) + return job.ID, nil +} + +// find returns the job with this ID. Caller holds jq.mu. +func (jq *JobQueue) find(jobID string) (int, *QueuedJob) { + for i, j := range jq.jobs { + if j.ID == jobID { + return i, j + } + } + return -1, nil +} + +// GetJob returns a copy of the job. +func (jq *JobQueue) GetJob(jobID string) (QueuedJob, bool) { + jq.mu.RLock() + defer jq.mu.RUnlock() + if _, j := jq.find(jobID); j != nil { + return j.snapshot(), true + } + return QueuedJob{}, false +} + +// ListJobs returns copies of all jobs: running, waiting, queued, then finished. +// Running jobs carry their session's live progress. Nothing is written. +func (jq *JobQueue) ListJobs() []QueuedJob { + jq.mu.RLock() + defer jq.mu.RUnlock() + out := make([]QueuedJob, 0, len(jq.jobs)) + for _, status := range []string{jobRunning, jobWaiting, jobQueued, ""} { + for _, j := range jq.jobs { + finished := j.Status == jobCompleted || j.Status == jobFailed + if (status == "" && finished) || j.Status == status { + out = append(out, j.snapshot()) + } + } + } + return out +} + +// runningJobs returns copies of the running jobs. +func (jq *JobQueue) runningJobs() []QueuedJob { + jq.mu.RLock() + defer jq.mu.RUnlock() + var out []QueuedJob + for _, j := range jq.jobs { + if j.Status == jobRunning { + out = append(out, j.snapshot()) + } + } + return out +} + +// snapshot copies the job; a running job gets its session's current progress. +// Caller holds jq.mu (read is enough: only the copy is changed). +func (j *QueuedJob) snapshot() QueuedJob { + c := *j + c.session = nil + if j.Status == jobRunning && j.session != nil { + c.setProgress(&j.session.progress) + } + return c +} + +// setProgress fills the counters from a session's published progress. +func (j *QueuedJob) setProgress(p *sessionProgress) { + total := p.total.Load() + if total == 0 { + return // NZB not loaded yet: keep the counts from AddJob + } + // The divider's "done" never counts uploaded items (an upload sets uploadedTo, not + // availableOn); they end up in notarget instead. ok = on every provider, or uploaded. + done, dead, notarget := p.done.Load(), p.dead.Load(), p.notarget.Load() + finished := min(done+dead+notarget, total) + j.TotalArticles = total + j.ArticlesDone = min(done+p.uploaded.Load(), total) + j.ArticlesProcessed = finished + j.ArticlesCached = p.cached.Load() + j.ArticlesDead = dead + j.Progress = float64(finished) / float64(total) +} + +// StartAllQueued starts every queued job. They run one after another. +func (jq *JobQueue) StartAllQueued() int { + jq.mu.Lock() + var ids []string + for _, j := range jq.jobs { + if j.Status == jobQueued { + ids = append(ids, j.ID) + } + } + jq.mu.Unlock() + started := 0 + for _, id := range ids { + if err := jq.StartJob(id); err != nil { + dlog(always, "Warning: Failed to start job %s: %v", id, err) + } else { + started++ + } + } + return started +} + +// StartJob moves a queued job to waiting and starts its goroutine. The job +// becomes running once it holds processingMu. +func (jq *JobQueue) StartJob(jobID string) error { + jq.mu.Lock() + defer jq.mu.Unlock() + _, job := jq.find(jobID) + if job == nil { + return fmt.Errorf("%w: %s", errJobNotFound, jobID) + } + if job.Status != jobQueued { + return fmt.Errorf("%w: %s is %s, not %s", errJobState, jobID, job.Status, jobQueued) + } + job.Status = jobWaiting + go jq.runJob(job) + return nil +} + +// runJob waits for its turn, runs the job's session and records the result. +func (jq *JobQueue) runJob(job *QueuedJob) { + jq.processingMu.Lock() + defer jq.processingMu.Unlock() + + jq.mu.Lock() + if job.Status != jobWaiting { + jq.mu.Unlock() // removed or canceled while waiting + return + } + if webProcessor == nil { + jq.finishLocked(job, errors.New("web processor not initialized")) + jq.mu.Unlock() + return + } + sessId, s := webProcessor.newSession(job.NZBPath) + job.session = s + job.SessionID = sessId + job.Status = jobRunning + job.StartedAt = time.Now() + jq.mu.Unlock() + + dlog(cfg.opt.Verbose, "Starting job %s sessId=%d NZB: %s", job.ID, sessId, job.NZBPath) + err := jq.launch(s) + if err != nil { + // LaunchSession's early error returns skip clearSession: don't leave the session in sessMap + webProcessor.removeSession(sessId, s) + } + + jq.mu.Lock() + jq.finishLocked(job, err) + jq.mu.Unlock() +} + +// launch runs the session. A panic on this goroutine fails the job instead of +// killing the web server. Panics on worker goroutines can't be caught here. +func (jq *JobQueue) launch(s *SESSION) (err error) { + defer func() { + if r := recover(); r != nil { + dlog(always, "ERROR: job session %d panicked: %v\n%s", s.sessId, r, debug.Stack()) + err = fmt.Errorf("panic: %v", r) + } + }() + if err := webProcessor.LaunchSession(s, "", nil); err != nil { + return fmt.Errorf("failed to launch session: %w", err) + } + return nil +} + +// finishLocked records the job's result and drops the oldest finished jobs +// beyond maxFinishedJobs. Caller holds jq.mu. +func (jq *JobQueue) finishLocked(job *QueuedJob, err error) { + if job.session != nil { + job.setProgress(&job.session.progress) // final counts, also when nobody polled + job.session = nil // let the session be collected + } + if err != nil { + job.Status = jobFailed + job.Error = err.Error() + dlog(always, "ERROR: Job %s failed: %v", job.ID, err) + } else { + job.Status = jobCompleted + dlog(always, "Job %s completed successfully", job.ID) + } + job.FinishedAt = time.Now() + + finished := 0 + for _, j := range jq.jobs { + if j.Status == jobCompleted || j.Status == jobFailed { + finished++ + } + } + if drop := finished - maxFinishedJobs; drop > 0 { + kept := jq.jobs[:0] + for _, j := range jq.jobs { + if drop > 0 && (j.Status == jobCompleted || j.Status == jobFailed) { + drop-- + continue + } + kept = append(kept, j) + } + clear(jq.jobs[len(kept):]) + jq.jobs = kept + } +} + +// DeleteJob removes a queued or finished job. +func (jq *JobQueue) DeleteJob(jobID string) error { + jq.mu.Lock() + defer jq.mu.Unlock() + i, job := jq.find(jobID) + if job == nil { + return fmt.Errorf("%w: %s", errJobNotFound, jobID) + } + switch job.Status { + case jobQueued, jobCompleted, jobFailed: + default: + return fmt.Errorf("%w: cannot delete %s job %s", errJobState, job.Status, jobID) + } + jq.jobs = append(jq.jobs[:i], jq.jobs[i+1:]...) + return nil +} diff --git a/webjobs_test.go b/webjobs_test.go new file mode 100644 index 0000000..8c66715 --- /dev/null +++ b/webjobs_test.go @@ -0,0 +1,125 @@ +package main + +import ( + "errors" + "net" + "os" + "path/filepath" + gotesting "testing" // Workers.go declares a const named testing +) + +const testNZB = ` + + + alt.test + + a1@test + a2@test + + + +` + +func writeFile(t *gotesting.T, name, data string) string { + t.Helper() + p := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(p, []byte(data), 0600); err != nil { + t.Fatal(err) + } + return p +} + +// Plan 4.3: AddJob swallowed the parse error and accepted a corrupt file. +func TestAddJobRejectsBrokenNZB(t *gotesting.T) { + testCfg() + jq := NewJobQueue() + if _, err := jq.AddJob(writeFile(t, "bad.nzb", "this is not xml"), "bad.nzb"); err == nil { + t.Fatal("a broken NZB was accepted") + } + id, err := jq.AddJob(writeFile(t, "ok.nzb", testNZB), "ok.nzb") + if err != nil { + t.Fatalf("valid NZB rejected: %v", err) + } + job, ok := jq.GetJob(id) + if !ok || job.TotalArticles != 2 || job.Status != jobQueued { + t.Fatalf("job = %+v ok=%t, want 2 articles, queued", job, ok) + } +} + +// Plan 4.2: ListJobs handed out live pointers and wrote progress into them +// under a read lock. It now returns copies; changing one changes nothing. +func TestListJobsReturnsCopies(t *gotesting.T) { + testCfg() + jq := NewJobQueue() + id, err := jq.AddJob(writeFile(t, "ok.nzb", testNZB), "ok.nzb") + if err != nil { + t.Fatal(err) + } + jobs := jq.ListJobs() + jobs[0].Status = "tampered" + if job, _ := jq.GetJob(id); job.Status != jobQueued { + t.Fatalf("changing a listed job changed the queue: status=%q", job.Status) + } +} + +// A running job's progress comes from its own session, and the final counts +// are kept when it finishes (it used to advance only while someone polled). +func TestJobProgressFromItsSession(t *gotesting.T) { + testCfg() + jq := NewJobQueue() + id, _ := jq.AddJob(writeFile(t, "ok.nzb", testNZB), "ok.nzb") + s := &SESSION{} + s.progress.total.Store(4) + s.progress.done.Store(1) + s.progress.dead.Store(1) + jq.mu.Lock() + _, job := jq.find(id) + job.Status, job.session = jobRunning, s + jq.mu.Unlock() + + if got, _ := jq.GetJob(id); got.Progress != 0.5 || got.ArticlesProcessed != 2 || got.TotalArticles != 4 { + t.Fatalf("running job: progress=%v processed=%d total=%d, want 0.5 2 4", got.Progress, got.ArticlesProcessed, got.TotalArticles) + } + s.progress.done.Store(3) + jq.mu.Lock() + jq.finishLocked(job, nil) + jq.mu.Unlock() + if got, _ := jq.GetJob(id); got.Status != jobCompleted || got.Progress != 1 || got.ArticlesDone != 3 { + t.Fatalf("finished job: %+v, want completed with progress 1 and 3 done", got) + } +} + +func TestJobStateErrors(t *gotesting.T) { + testCfg() + jq := NewJobQueue() + if err := jq.StartJob("job-404"); !errors.Is(err, errJobNotFound) { + t.Fatalf("start unknown job: err=%v, want errJobNotFound", err) + } + id, _ := jq.AddJob(writeFile(t, "ok.nzb", testNZB), "ok.nzb") + jq.mu.Lock() + _, job := jq.find(id) + job.Status = jobRunning + jq.mu.Unlock() + if err := jq.StartJob(id); !errors.Is(err, errJobState) { + t.Fatalf("start running job: err=%v, want errJobState", err) + } + if err := jq.DeleteJob(id); !errors.Is(err, errJobState) { + t.Fatalf("delete running job: err=%v, want errJobState", err) + } + if jobErrStatus(errJobNotFound) != 404 || jobErrStatus(errJobState) != 409 { + t.Fatal("wrong HTTP status mapping") + } +} + +// Plan 4.1: a busy port was only logged from a goroutine; StartWebServer returned nil. +func TestStartWebServerReportsBindError(t *gotesting.T) { + testCfg() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + if err := StartWebServer(ln.Addr().String(), &PROCESSOR{}); err == nil { + t.Fatal("StartWebServer on a busy port returned no error") + } +} diff --git a/webserver.go b/webserver.go index e7af721..5e6504c 100644 --- a/webserver.go +++ b/webserver.go @@ -1,14 +1,16 @@ package main import ( + "crypto/tls" "embed" + "errors" "fmt" "html/template" "log" + "net" "net/http" "os" "path/filepath" - "sync" "time" "github.com/gin-gonic/gin" @@ -23,373 +25,11 @@ var ( webServerSSL bool webServerCert string webServerKey string - webProcessor *PROCESSOR - webWg *sync.WaitGroup - jobQueue *JobQueue + webProcessor *PROCESSOR // set by StartWebServer before the server serves + jobQueue *JobQueue // set by StartWebServer before the server serves + webSrv *http.Server // the running server, for Shutdown ) -// JobQueue manages queued NZB processing jobs -type JobQueue struct { - mu sync.RWMutex - jobs []*QueuedJob - running map[string]*QueuedJob // jobID -> job - completed []*QueuedJob // completed/failed jobs - nextID int - processingMu sync.Mutex // Ensures only one job processes at a time (prevents global cfg conflicts) - defaultConfig JobConfig // Default configuration for new jobs -} - -// QueuedJob represents a job in the queue -type QueuedJob struct { - ID string - NZBPath string - NZBName string - Config JobConfig - Status string // "queued", "running", "completed", "failed" - AddedAt time.Time - StartedAt time.Time - FinishedAt time.Time - Error string - Progress float64 - TotalArticles uint64 `json:"total_articles"` - ArticlesProcessed uint64 `json:"articles_processed"` // done count - ArticlesDone uint64 `json:"articles_done"` // done count (same as ArticlesProcessed) - ArticlesCached uint64 `json:"articles_cached"` // cached count - ArticlesDead uint64 `json:"articles_dead"` // dead count - TotalSize int64 `json:"total_size"` // Total size in bytes - SessionID uint64 `json:"session_id"` // Link to SESSION -} - -// JobConfig holds configuration options for a job -type JobConfig struct { - CheckOnly bool - CheckFirst bool - YencCRC bool - YencTest int - YencCpu int - Cachedir string -} - -// SessionStats holds real-time statistics extracted from a SESSION -type SessionStats struct { - TotalArticles uint64 - ArticlesDone uint64 - ArticlesCached uint64 - ArticlesDead uint64 - ArticlesChecked uint64 -} - -// NewJobQueue creates a new job queue -func NewJobQueue() *JobQueue { - globalmux.Lock() - defer globalmux.Unlock() - - // Initialize with default config from current global config - defaultConfig := JobConfig{ - CheckOnly: false, - CheckFirst: false, - YencCRC: false, - YencTest: 4, - YencCpu: 0, - Cachedir: "", - } - - // If cfg is already initialized, use its values as defaults - if cfg != nil && cfg.opt != nil { - defaultConfig.CheckOnly = cfg.opt.CheckOnly - defaultConfig.CheckFirst = cfg.opt.CheckFirst - defaultConfig.YencCRC = cfg.opt.YencCRC - defaultConfig.YencTest = cfg.opt.YencTest - defaultConfig.YencCpu = cfg.opt.YencCpu - defaultConfig.Cachedir = cfg.opt.Cachedir - } - - return &JobQueue{ - jobs: make([]*QueuedJob, 0), - running: make(map[string]*QueuedJob), - completed: make([]*QueuedJob, 0), - nextID: 1, - defaultConfig: defaultConfig, - } -} - -// AddJob adds a new job to the queue -func (jq *JobQueue) AddJob(nzbPath, nzbName string, config JobConfig) string { - jq.mu.Lock() - defer jq.mu.Unlock() - - jobID := fmt.Sprintf("job-%d", jq.nextID) - jq.nextID++ - - job := &QueuedJob{ - ID: jobID, - NZBPath: nzbPath, - NZBName: nzbName, - Config: config, - Status: "queued", - AddedAt: time.Now(), - } - - // Try to parse NZB to get size and article count - nzbfile, err := loadNzbFile(nzbPath) - if err == nil && nzbfile != nil { - var totalArticles uint64 - - for _, file := range nzbfile.Files { - totalArticles += uint64(file.TotalSegments) - } - - job.TotalSize = nzbfile.Bytes - job.TotalArticles = totalArticles - } - - jq.jobs = append(jq.jobs, job) - return jobID -} - -// GetJob retrieves a job by ID -func (jq *JobQueue) GetJob(jobID string) *QueuedJob { - jq.mu.RLock() - defer jq.mu.RUnlock() - - // Check running jobs first - if job, ok := jq.running[jobID]; ok { - return job - } - - // Check queued jobs - for _, job := range jq.jobs { - if job.ID == jobID { - return job - } - } - - // Check completed jobs - for _, job := range jq.completed { - if job.ID == jobID { - return job - } - } - - return nil -} - -// ListJobs returns all jobs (queued, running, and completed) -func (jq *JobQueue) ListJobs() []*QueuedJob { - jq.mu.RLock() - defer jq.mu.RUnlock() - - allJobs := make([]*QueuedJob, 0, len(jq.jobs)+len(jq.running)+len(jq.completed)) - - // Add running jobs with real-time stats - for _, job := range jq.running { - // Extract session stats for running jobs - if webProcessor != nil { - webProcessor.mux.RLock() - for _, session := range webProcessor.sessMap { - if session != nil && session.nzbPath == job.NZBPath { - sessionStats := extractSessionStats(session) - if sessionStats != nil { - // Update job with real-time statistics - job.ArticlesDone = sessionStats.ArticlesDone - job.ArticlesProcessed = sessionStats.ArticlesDone - job.ArticlesCached = sessionStats.ArticlesCached - job.ArticlesDead = sessionStats.ArticlesDead - job.TotalArticles = sessionStats.TotalArticles - if sessionStats.TotalArticles > 0 { - job.Progress = float64(sessionStats.ArticlesDone) / float64(sessionStats.TotalArticles) - } - } - break - } - } - webProcessor.mux.RUnlock() - } - allJobs = append(allJobs, job) - } - - // Add queued jobs - allJobs = append(allJobs, jq.jobs...) - - // Add completed jobs - allJobs = append(allJobs, jq.completed...) - - return allJobs -} - -// GetDefaultConfig returns a copy of the default job configuration -func (jq *JobQueue) GetDefaultConfig() JobConfig { - jq.mu.RLock() - defer jq.mu.RUnlock() - return jq.defaultConfig -} - -// SetDefaultConfig updates the default job configuration -func (jq *JobQueue) SetDefaultConfig(config JobConfig) { - jq.mu.Lock() - defer jq.mu.Unlock() - jq.defaultConfig = config -} - -// StartAllQueued starts all queued jobs (they will process serially) -func (jq *JobQueue) StartAllQueued() (int, error) { - jq.mu.Lock() - - // Get all queued job IDs - queuedIDs := make([]string, 0, len(jq.jobs)) - for _, job := range jq.jobs { - if job.Status == "queued" { - queuedIDs = append(queuedIDs, job.ID) - } - } - jq.mu.Unlock() - - // Start each job - started := 0 - for _, jobID := range queuedIDs { - if err := jq.StartJob(jobID); err != nil { - dlog(always, "Warning: Failed to start job %s: %v", jobID, err) - } else { - started++ - } - } - - return started, nil -} - -// StartJob starts a queued job -func (jq *JobQueue) StartJob(jobID string) error { - jq.mu.Lock() - - // Find the job in queue - var jobIndex = -1 - var job *QueuedJob - for i, j := range jq.jobs { - if j.ID == jobID { - jobIndex = i - job = j - break - } - } - - if jobIndex == -1 { - jq.mu.Unlock() - return fmt.Errorf("job %s not found in queue", jobID) - } - - // Remove from queue - jq.jobs = append(jq.jobs[:jobIndex], jq.jobs[jobIndex+1:]...) - - // Mark as running - job.Status = "running" - job.StartedAt = time.Now() - jq.running[jobID] = job - jq.mu.Unlock() - - // Start processing in background - go func() { - err := jq.processJob(job) - - jq.mu.Lock() - if err != nil { - job.Status = "failed" - job.Error = err.Error() - dlog(always, "ERROR: Job %s failed: %v", jobID, err) - } else { - job.Status = "completed" - dlog(always, "Job %s completed successfully", jobID) - } - job.FinishedAt = time.Now() - delete(jq.running, jobID) - - // Move to completed list - jq.completed = append(jq.completed, job) - - // Keep only last 50 completed jobs to prevent memory leak - if len(jq.completed) > 50 { - jq.completed = jq.completed[len(jq.completed)-50:] - } - - jq.mu.Unlock() - }() - - return nil -} - -// processJob processes a single job -func (jq *JobQueue) processJob(job *QueuedJob) error { - if webProcessor == nil { - return fmt.Errorf("web processor not initialized") - } - - // CRITICAL: Lock to prevent concurrent job execution - // This is necessary because cfg.opt is a GLOBAL variable shared across all jobs. - // Without this lock, concurrent jobs would overwrite each other's configuration. - // TODO: Refactor to pass per-job config to LaunchSession instead of using global cfg - jq.processingMu.Lock() - defer jq.processingMu.Unlock() - - // Apply job-specific configuration to global cfg - /* disabled: data race! - oldConfig := cfg.opt - defer func() { - cfg.opt = oldConfig // Restore original config - }() - - // Override with job config - cfg.opt.CheckOnly = job.Config.CheckOnly - cfg.opt.CheckFirst = job.Config.CheckFirst - cfg.opt.YencCRC = job.Config.YencCRC - cfg.opt.YencTest = job.Config.YencTest - cfg.opt.YencCpu = job.Config.YencCpu - if job.Config.Cachedir != "" { - cfg.opt.Cachedir = job.Config.Cachedir - } - */ - wg := &sync.WaitGroup{} - wg.Add(1) - - dlog(cfg.opt.Verbose, "Starting job %s with NZB: %s (serialized execution due to global cfg)", job.ID, job.NZBPath) - - // Launch session with NZB file (pass nil for session, file path for NZB) - err := webProcessor.LaunchSession(nil, job.NZBPath, wg) - if err != nil { - return fmt.Errorf("failed to launch session: %w", err) - } - - wg.Wait() - return nil -} - -// DeleteJob removes a job from the queue (only if not running) -func (jq *JobQueue) DeleteJob(jobID string) error { - jq.mu.Lock() - defer jq.mu.Unlock() - - // Can't delete running jobs - if _, ok := jq.running[jobID]; ok { - return fmt.Errorf("cannot delete running job %s", jobID) - } - - // Find and remove from queued jobs - for i, job := range jq.jobs { - if job.ID == jobID { - jq.jobs = append(jq.jobs[:i], jq.jobs[i+1:]...) - return nil - } - } - - // Find and remove from completed jobs - for i, job := range jq.completed { - if job.ID == jobID { - jq.completed = append(jq.completed[:i], jq.completed[i+1:]...) - return nil - } - } - - return fmt.Errorf("job %s not found", jobID) -} - // WebStats holds statistics for the web interface type WebStats struct { Uptime string `json:"uptime"` @@ -474,14 +114,11 @@ type JobInfo struct { DownloadBytes uint64 `json:"download_bytes"` } -// StartWebServer initializes and starts the Gin web server -func StartWebServer(addr string, processor *PROCESSOR, wg *sync.WaitGroup) error { - webServerEnabled = true - webServerAddr = addr +// StartWebServer sets up the routes, binds addr and starts serving in the +// background. A bind or certificate error is returned (plan 4.1: it used to be +// logged from a goroutine while the process went on without a web server). +func StartWebServer(addr string, processor *PROCESSOR) error { webProcessor = processor - webWg = wg - - // Initialize job queue jobQueue = NewJobQueue() // Set Gin to release mode for production @@ -520,50 +157,67 @@ func StartWebServer(addr string, processor *PROCESSOR, wg *sync.WaitGroup) error router.POST("/api/jobs/:id/start", handleJobStart) router.POST("/api/jobs/start-all", handleJobStartAll) router.DELETE("/api/jobs/:id", handleJobDelete) - router.PATCH("/api/jobs/:id/config", handleJobUpdateConfig) - router.GET("/api/jobs/default-config", handleGetDefaultConfig) - router.PUT("/api/jobs/default-config", handleSetDefaultConfig) + // per-job settings are not implemented yet (D2) + router.PATCH("/api/jobs/:id/config", handleJobConfigNotImplemented) + router.GET("/api/jobs/default-config", handleJobConfigNotImplemented) + router.PUT("/api/jobs/default-config", handleJobConfigNotImplemented) - // Determine protocol for logging - protocol := "http" + var tlsConf *tls.Config if webServerSSL { + cert, err := tls.LoadX509KeyPair(webServerCert, webServerKey) + if err != nil { + return fmt.Errorf("web server TLS cert='%s' key='%s': %w", webServerCert, webServerKey, err) + } + tlsConf = &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12} + } + ln, err := net.Listen("tcp", addr) + if err != nil { + return fmt.Errorf("web server listen on '%s': %w", addr, err) + } + srv := &http.Server{Handler: router, TLSConfig: tlsConf} + webSrv = srv + + protocol := "http" + if tlsConf != nil { protocol = "https" } - dlog(always, "Starting web server on %s://%s", protocol, addr) + dlog(always, "Web server listening on %s://%s", protocol, ln.Addr()) - // Run server in a goroutine go func() { var err error - if webServerSSL { - // Start HTTPS server with TLS - err = router.RunTLS(addr, webServerCert, webServerKey) + if tlsConf != nil { + err = srv.ServeTLS(ln, "", "") // certificate is in TLSConfig } else { - // Start HTTP server - err = router.Run(addr) + err = srv.Serve(ln) } - if err != nil { - dlog(always, "ERROR: Web server failed to start: %v", err) + if err != nil && !errors.Is(err, http.ErrServerClosed) { + dlog(always, "ERROR: web server stopped: %v", err) } }() - return nil } // handleIndex serves the main page func handleIndex(c *gin.Context) { - c.HTML(http.StatusOK, "index.html", gin.H{ - "title": "NZBreX - NZB Refresh", - "version": appVersion, - }) + c.HTML(http.StatusOK, "index.html", nil) } -// handleStats serves the statistics page +// handleStats serves the statistics page. The page loads its data from /api/stats. func handleStats(c *gin.Context) { - stats := collectStats() - c.HTML(http.StatusOK, "stats.html", gin.H{ - "title": "NZBreX Statistics", - "stats": stats, - }) + c.HTML(http.StatusOK, "stats.html", nil) +} + +// jobErrStatus maps a job queue error to an HTTP status. +func jobErrStatus(err error) int { + switch { + case errors.Is(err, errJobNotFound): + return http.StatusNotFound + case errors.Is(err, errJobState): + return http.StatusConflict + case errors.Is(err, errQueueFull): + return http.StatusServiceUnavailable + } + return http.StatusBadRequest } // handleUpload processes NZB file uploads and adds them to the job queue @@ -597,219 +251,93 @@ func handleUpload(c *gin.Context) { dlog(always, "Web: Uploaded NZB file: %s (size: %d bytes)", file.Filename, file.Size) - // Start with default config from queue, then override with form data if provided - config := jobQueue.GetDefaultConfig() - - // Override with form data if provided - if checkonly := c.PostForm("checkonly"); checkonly != "" { - config.CheckOnly = checkonly == "true" - } - if checkfirst := c.PostForm("checkfirst"); checkfirst != "" { - config.CheckFirst = checkfirst == "true" - } - if crc32 := c.PostForm("crc32"); crc32 != "" { - config.YencCRC = crc32 == "true" - } - if cachedir := c.PostForm("cachedir"); cachedir != "" { - config.Cachedir = cachedir - } - - // Parse optional int fields - if yencTest := c.PostForm("yenctest"); yencTest != "" { - if val, err := fmt.Sscanf(yencTest, "%d", &config.YencTest); err == nil && val == 1 { - if config.YencTest < 1 || config.YencTest > 4 { - config.YencTest = 4 - } - } - } - if yencCpu := c.PostForm("yenccpu"); yencCpu != "" { - fmt.Sscanf(yencCpu, "%d", &config.YencCpu) + jobID, err := jobQueue.AddJob(dst, file.Filename) + if err != nil { + os.Remove(dst) + c.JSON(jobErrStatus(err), gin.H{"error": err.Error()}) + return } - // Add job to queue - jobID := jobQueue.AddJob(dst, file.Filename, config) - - c.JSON(http.StatusOK, gin.H{ + resp := gin.H{ "message": "File uploaded and added to queue", "filename": file.Filename, "size": file.Size, "job_id": jobID, - "status": "queued", - }) + "status": jobQueued, + } + for _, field := range []string{"checkonly", "checkfirst", "crc32", "cachedir", "yenctest", "yenccpu"} { + if c.PostForm(field) != "" { + resp["note"] = "per-job settings are not supported yet and were ignored: the job runs with the startup settings" + break + } + } + c.JSON(http.StatusOK, resp) } -// handleJobsList displays the jobs queue page +// handleJobsList displays the jobs queue page. The page loads its data from /api/jobs. func handleJobsList(c *gin.Context) { - jobs := jobQueue.ListJobs() - c.HTML(http.StatusOK, "jobs.html", gin.H{ - "title": "Job Queue", - "jobs": jobs, - }) + c.HTML(http.StatusOK, "jobs.html", nil) } // handleAPIJobsList returns all jobs as JSON func handleAPIJobsList(c *gin.Context) { - jobs := jobQueue.ListJobs() c.JSON(http.StatusOK, gin.H{ - "jobs": jobs, + "jobs": jobQueue.ListJobs(), }) } // handleJobStart starts a queued job func handleJobStart(c *gin.Context) { jobID := c.Param("id") - if err := jobQueue.StartJob(jobID); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + c.JSON(jobErrStatus(err), gin.H{"error": err.Error()}) return } - c.JSON(http.StatusOK, gin.H{ "message": "Job started successfully", "job_id": jobID, }) } -// handleJobDelete deletes a queued job +// handleJobDelete deletes a queued or finished job func handleJobDelete(c *gin.Context) { jobID := c.Param("id") - if err := jobQueue.DeleteJob(jobID); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + c.JSON(jobErrStatus(err), gin.H{"error": err.Error()}) return } - c.JSON(http.StatusOK, gin.H{ "message": "Job deleted successfully", "job_id": jobID, }) } -// handleJobUpdateConfig updates a job's configuration -func handleJobUpdateConfig(c *gin.Context) { - jobID := c.Param("id") - - job := jobQueue.GetJob(jobID) - if job == nil { - c.JSON(http.StatusNotFound, gin.H{"error": "Job not found"}) - return - } - - if job.Status != "queued" { - c.JSON(http.StatusBadRequest, gin.H{"error": "Cannot update config of non-queued job"}) - return - } - - // Update configuration from JSON body - var newConfig JobConfig - if err := c.ShouldBindJSON(&newConfig); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid configuration"}) - return - } - - jobQueue.mu.Lock() - job.Config = newConfig - jobQueue.mu.Unlock() - - c.JSON(http.StatusOK, gin.H{ - "message": "Job configuration updated", - "job_id": jobID, - "config": newConfig, +// handleJobConfigNotImplemented answers the per-job and default settings +// endpoints. Jobs run with the startup settings until per-job settings exist (D2). +func handleJobConfigNotImplemented(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{ + "error": "per-job settings are not supported yet: jobs run with the startup settings", + "config": startupJobConfig(), }) } // handleJobStartAll starts all queued jobs func handleJobStartAll(c *gin.Context) { - started, err := jobQueue.StartAllQueued() - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - + started := jobQueue.StartAllQueued() c.JSON(http.StatusOK, gin.H{ "message": fmt.Sprintf("Started %d job(s)", started), "count": started, }) } -// handleGetDefaultConfig returns the default job configuration -func handleGetDefaultConfig(c *gin.Context) { - config := jobQueue.GetDefaultConfig() - c.JSON(http.StatusOK, gin.H{ - "config": config, - }) -} - -// handleSetDefaultConfig updates the default job configuration -func handleSetDefaultConfig(c *gin.Context) { - var newConfig JobConfig - if err := c.ShouldBindJSON(&newConfig); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid configuration"}) - return - } - - jobQueue.SetDefaultConfig(newConfig) - - c.JSON(http.StatusOK, gin.H{ - "message": "Default configuration updated", - "config": newConfig, - }) -} - // handleAPIStats returns statistics as JSON for AJAX requests func handleAPIStats(c *gin.Context) { stats := collectStats() c.JSON(http.StatusOK, stats) } -// extractSessionStats extracts real-time statistics from a SESSION -// This function counts the various states of segments in the session -func extractSessionStats(s *SESSION) *SessionStats { - if s == nil { - return nil - } - s.mux.RLock() - defer s.mux.RUnlock() - if s.segmentList == nil { - return nil - } - - stats := &SessionStats{ - TotalArticles: uint64(len(s.segmentList)), - } - providersCnt := len(s.providerList) - - // Count articles by state - for _, item := range s.segmentList { - item.mux.RLock() - - // Count cached articles - if item.flagCache { - stats.ArticlesCached++ - } - - // Count checked articles - if item.checkedOn > 0 { - stats.ArticlesChecked++ - } - - // Count done articles (available on all providers or uploaded) - if len(item.missingOn) == 0 && len(item.availableOn) == providersCnt { - stats.ArticlesDone++ - } - - // Count dead articles (not available anywhere) - if len(item.availableOn) == 0 && len(item.missingOn) == providersCnt { - stats.ArticlesDead++ - } - - item.mux.RUnlock() - } - - return stats -} - -// collectStats gathers current statistics for display +// collectStats gathers current statistics for display. It only copies +// counters: no segment list is walked and no lock is held for long (plan 4.3). func collectStats() WebStats { stats := WebStats{ Uptime: formatDuration(time.Since(booted)), @@ -818,9 +346,12 @@ func collectStats() WebStats { ActiveJobs: []JobInfo{}, } - // Get memory usage - if memlim != nil { - stats.MemUsed, stats.MemMax = memlim.Usage() + // memlim is created by the first LaunchSession, under globalmux + globalmux.RLock() + ml := memlim + globalmux.RUnlock() + if ml != nil { + stats.MemUsed, stats.MemMax = ml.Usage() } // Get global counters if available @@ -846,17 +377,14 @@ func collectStats() WebStats { stats.OpenConns = stats.TotalNewConns - stats.TotalDisConns } - // Collect provider statistics from running/completed jobs - globalmux.RLock() - defer globalmux.RUnlock() - if webProcessor != nil { - // closure + defer: a panic in here must never leave webProcessor.mux read-locked, - // or removeSession() blocks forever and the whole web server wedges + // copy the active sessions' provider lists, then read the providers without p.mux + var providerLists [][]*Provider func() { + // closure + defer: a panic in here must never leave webProcessor.mux read-locked, + // or removeSession() blocks forever and the whole web server wedges webProcessor.mux.RLock() defer webProcessor.mux.RUnlock() - // Iterate through all active sessions for _, session := range webProcessor.sessMap { if session == nil { continue @@ -865,97 +393,68 @@ func collectStats() WebStats { active := session.active // written under session.mux by GoBootWorkers and closeSession providerList := session.providerList // published under session.mux by loadProviderList session.mux.RUnlock() - if !active { - continue - } - for _, prov := range providerList { - // KillConnPool sets prov.ConnPool = nil and the workers update the - // counters under prov.mux: copy everything under one read lock - prov.mux.RLock() - cp := prov.ConnPool - articles := prov.articles - downloadSpeed := prov.speed.downloadSpeed // stored by Speedmeter - uploadSpeed := prov.speed.uploadSpeed - prov.mux.RUnlock() - if cp == nil { - continue // pool already closed: session is shutting down - } - openConns, idleConns := cp.GetStats() - - // Get bandwidth from provider counter - provRX := cp.Counter.GetValue("TOTAL_RXbytes") - provTX := cp.Counter.GetValue("TOTAL_TXbytes") - - provStats := ProviderStats{ - Name: prov.Name, - Group: prov.Group, - OpenConns: openConns, - IdleConns: idleConns, - Checks: articles.checked, - Downloads: articles.downloaded, - Uploads: articles.refreshed, - Checked: articles.checked, - Available: articles.available, - Missing: articles.missing, - Downloaded: articles.downloaded, - Uploaded: articles.refreshed, - TotalRXbyte: provRX, - TotalTXbyte: provTX, - TotalDownload: provRX, // For JSON compatibility - TotalUpload: provTX, // For JSON compatibility - DownloadSpeed: downloadSpeed, // bytes/sec (from Speedmeter) - UploadSpeed: uploadSpeed, // bytes/sec (from Speedmeter) - } - stats.Providers = append(stats.Providers, provStats) + if active { + providerLists = append(providerLists, providerList) } } }() - } - - // Collect active job information - if jobQueue != nil { - jobQueue.mu.RLock() - for _, job := range jobQueue.running { - // Try to find the session for this job and get real-time stats - var sessionStats *SessionStats - if webProcessor != nil { - webProcessor.mux.RLock() - for _, session := range webProcessor.sessMap { - if session != nil && session.nzbPath == job.NZBPath { - sessionStats = extractSessionStats(session) - break - } - } - webProcessor.mux.RUnlock() - } - - jobInfo := JobInfo{ - ID: job.ID, - NZBName: job.NZBName, - Status: job.Status, - StartedAt: job.StartedAt, - StartTime: job.StartedAt, // For JSON compatibility - Progress: job.Progress, - } - - // Update job with session stats if available - if sessionStats != nil { - jobInfo.TotalSegments = sessionStats.TotalArticles - jobInfo.ArticlesProcessed = sessionStats.ArticlesDone - // Update the job object itself with real-time progress - job.ArticlesDone = sessionStats.ArticlesDone - job.ArticlesProcessed = sessionStats.ArticlesDone - job.ArticlesCached = sessionStats.ArticlesCached - job.ArticlesDead = sessionStats.ArticlesDead - job.TotalArticles = sessionStats.TotalArticles - if sessionStats.TotalArticles > 0 { - job.Progress = float64(sessionStats.ArticlesDone) / float64(sessionStats.TotalArticles) + for _, providerList := range providerLists { + for _, prov := range providerList { + // KillConnPool sets prov.ConnPool = nil and the workers update the + // counters under prov.mux: copy everything under one read lock + prov.mux.RLock() + cp := prov.ConnPool + articles := prov.articles + downloadSpeed := prov.speed.downloadSpeed // stored by Speedmeter + uploadSpeed := prov.speed.uploadSpeed + prov.mux.RUnlock() + if cp == nil { + continue // pool already closed: session is shutting down } + openConns, idleConns := cp.GetStats() + + // Get bandwidth from provider counter + provRX := cp.Counter.GetValue("TOTAL_RXbytes") + provTX := cp.Counter.GetValue("TOTAL_TXbytes") + + stats.Providers = append(stats.Providers, ProviderStats{ + Name: prov.Name, + Group: prov.Group, + OpenConns: openConns, + IdleConns: idleConns, + Checks: articles.checked, + Downloads: articles.downloaded, + Uploads: articles.refreshed, + Checked: articles.checked, + Available: articles.available, + Missing: articles.missing, + Downloaded: articles.downloaded, + Uploaded: articles.refreshed, + TotalRXbyte: provRX, + TotalTXbyte: provTX, + TotalDownload: provRX, // For JSON compatibility + TotalUpload: provTX, // For JSON compatibility + DownloadSpeed: downloadSpeed, // bytes/sec (from Speedmeter) + UploadSpeed: uploadSpeed, // bytes/sec (from Speedmeter) + }) } + } + } - stats.ActiveJobs = append(stats.ActiveJobs, jobInfo) + // running jobs, with their sessions' live progress (copies: nothing is written) + if jobQueue != nil { + for _, job := range jobQueue.runningJobs() { + stats.ActiveJobs = append(stats.ActiveJobs, JobInfo{ + ID: job.ID, + NZBName: job.NZBName, + Status: job.Status, + StartedAt: job.StartedAt, + StartTime: job.StartedAt, // For JSON compatibility + TotalSegments: job.TotalArticles, + Progress: job.Progress, + ArticlesProcessed: job.ArticlesProcessed, + }) } - jobQueue.mu.RUnlock() } // Calculate computed fields From 453edcdb2c99f3c0b43c4d9be39eb9d931b4128b Mon Sep 17 00:00:00 2001 From: go-while Date: Sun, 18 Oct 2026 14:12:58 +0000 Subject: [PATCH 132/152] Web login, CSRF guard, timeouts and safe uploads; shared passwd code Login (D1): HTTP basic auth against its own -webpasswd file, users via -webadduser. Without it the server only starts on loopback. Verified logins are cached briefly, the file is reloaded when it changes. The passwd parsing moves to passwd.go and is shared with the proxy, which keeps its own map: a reload swaps in a new map only when it parsed and has users (it used to clear the live map first), and unknown users cost a bcrypt comparison like wrong passwords (timing oracle), expiry checked. Every state-changing request needs X-NZBreX-Request: 1, a same-site Sec-Fetch-Site and a matching Origin. Without a login the Host header is checked against an allowlist (DNS rebinding). Server timeouts, 128 MiB upload cap, no trusted proxies. Uploads go to a private MkdirTemp dir under unique names, written with O_EXCL instead of gin's SaveUploadedFile, and are deleted with their job. Bootstrap is embedded (Debian libjs-bootstrap5 5.3.5) instead of loaded from a CDN. Co-Authored-By: Claude Opus 5.5 --- Flags.go | 41 +++++ Proxy.go | 2 +- ProxyHelper.go | 220 +++--------------------- WEB_JOB_QUEUE.md | 52 +++--- passwd.go | 197 +++++++++++++++++++++ review-the-codebase-for-jolly-petal.md | 16 ++ web/static/bootstrap.bundle.min.js | 7 + web/static/bootstrap.min.css | 6 + web/{templates => static}/style.css | 0 web/templates/base.html | 59 ------- web/templates/index.html | 7 +- web/templates/jobs.html | 11 +- web/templates/stats.html | 4 +- webauth.go | 228 +++++++++++++++++++++++++ webauth_test.go | 207 ++++++++++++++++++++++ webjobs.go | 14 ++ webserver.go | 125 +++++++++++--- 17 files changed, 885 insertions(+), 311 deletions(-) create mode 100644 passwd.go create mode 100644 web/static/bootstrap.bundle.min.js create mode 100644 web/static/bootstrap.min.css rename web/{templates => static}/style.css (100%) delete mode 100644 web/templates/base.html create mode 100644 webauth.go create mode 100644 webauth_test.go diff --git a/Flags.go b/Flags.go index 5abd775..e7d91dc 100644 --- a/Flags.go +++ b/Flags.go @@ -96,6 +96,9 @@ func ParseFlags() { flag.BoolVar(&webServerSSL, "webssl", false, "[true|false] enable HTTPS/TLS for web server (default: false)") flag.StringVar(&webServerCert, "webssl-crt", "fullchain.pem", "path to TLS certificate file for web server (default: fullchain.pem in current dir)") flag.StringVar(&webServerKey, "webssl-key", "privkey.pem", "path to TLS private key file for web server (default: privkey.pem in current dir)") + flag.StringVar(&webPasswdFile, "webpasswd", "", "web server login: passwd file with the web users (same format as the proxy's, separate users). Without it the web server only runs on a loopback address (default: empty = no login)") + flag.StringVar(&webAddUser, "webadduser", "", "adds a user to the -webpasswd file and exits: 'username|password' (both at least 10 characters) or 'auto|auto' for random credentials") + flag.StringVar(&webHosts, "webhosts", "", "comma separated host names the web server answers to (Host header), e.g. 'nzbrex.example.org'. Always enforced without a login (localhost, 127.0.0.1, ::1 and the -webaddr host are allowed)") // cosmetics: segmentBar needs fixing: only when everything else works! //flag.BoolVar(&cfg.opt.Bar, "bar", false, "show progress bars") // FIXME TODO //flag.BoolVar(&cfg.opt.Colors, "colors", false, "adds colors to s") // FIXME TODO @@ -187,6 +190,10 @@ func ParseFlags() { // Redact password before logging os.Exit(0) } + if webAddUser != "" { + addWebUser(webAddUser) + os.Exit(0) + } // test rapidyenc decoder if testrapidyenc { // this is only for testing rapidyenc decoder @@ -419,3 +426,37 @@ func rapidyencDecoderFilesTest() (errs []error) { } // end for range files return errs } // end func rapidyencDecoderFilesTest + +// addWebUser handles -webadduser 'username|password'. The user goes to a +// .new.* file, which the web server merges on its next load. Web +// users don't expire (ExpireAt 0) and have no connection or posting rights. +func addWebUser(spec string) { + if webPasswdFile == "" { + log.Fatalf("-webadduser needs -webpasswd to name the web passwd file") + } + parts := strings.SplitN(spec, "|", 2) + if len(parts) != 2 { + log.Fatalf("-webadduser expects format: username|password") + } + username, password := strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]) + auto := username == "auto" && password == "auto" + if auto { + var err error + if username, password, err = generateRandomHexCredentials(); err != nil { + log.Fatalf("-webadduser: %v", err) + } + } + if len(username) < minUsernameLen || len(password) < 10 { + log.Fatalf("-webadduser: username and password must be at least 10 characters long") + } + if strings.ContainsAny(username, "|:") { + log.Fatalf("-webadduser: username must not contain '|' or ':'") + } + newFile := fmt.Sprintf("%s.new.%d.%d", webPasswdFile, time.Now().Unix(), mrand.Intn(65535)) + if err := appendPasswdUser(&UserData{Username: username, Password: password}, newFile); err != nil { + log.Fatalf("Failed to add web user to %s: %v", newFile, err) + } + if auto { + dlog(always, "-webadduser: auto generated credentials: username='%s' password='%s'", username, password) + } +} diff --git a/Proxy.go b/Proxy.go index 60b5e82..3be626a 100644 --- a/Proxy.go +++ b/Proxy.go @@ -104,7 +104,7 @@ func StartNNTPServer(addr string, passwdFilePath string, certFile string, keyFil dlog(always, "To manually add users, use the '-proxyadduser' command or craft a new .passwd file manually") dlog(always, "Example -proxyadduser \"HelloWorld|NotAsecurePassword|5|42d\" creates a user with 5 conns and 42 days to expiration") - } else if len(passwdMap) == 0 { + } else if proxyUserCount() == 0 { dlog(always, "Warning: Passwd file '%s' loaded, but no users found or all entries were invalid.", passwdFilePath) } diff --git a/ProxyHelper.go b/ProxyHelper.go index 498326b..adec087 100644 --- a/ProxyHelper.go +++ b/ProxyHelper.go @@ -6,15 +6,10 @@ import ( "fmt" "io" "log" - mrand "math/rand" "net" - "os" - "path/filepath" "strconv" "strings" "time" - - "golang.org/x/crypto/bcrypt" ) func LinesWriter(cliwriter *bufio.Writer, conn net.Conn, code int, item *segmentChanItem) (txb uint64, err error) { @@ -75,207 +70,48 @@ func (ps *ProxySession) printCapabilities() { dw.Close() } -// Passwords in the file are expected to be bcrypt hash strings. -// File format: username:bcrypt_hash:maxconns:ExpireAt_unix_timestamp +// loadPasswdFile merges filename.new.* into filename and loads the proxy users. +// The live map is replaced only by a successfully parsed, non-empty one (plan 5.2: +// it used to be cleared first, so a failed or empty reload locked everyone out +// until restart). Emptying the file therefore keeps the previous users. func loadPasswdFile(filename string) error { - // Before loading, merge any .new.* files into the main passwd file - dir := filepath.Dir(filename) - base := filepath.Base(filename) - if dir == "" { - dir = "." - } - pattern := fmt.Sprintf("%s.new.", base) - entries, err := os.ReadDir(dir) - if err == nil { - for _, entry := range entries { - if entry.IsDir() { - continue - } - name := entry.Name() - if strings.HasPrefix(name, pattern) { - newfile := filepath.Join(dir, name) - // Append contents to main passwd file - func() { - mainf, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640) - if err != nil { - log.Printf("Could not open %s for appending: %v", filename, err) - return - } - defer mainf.Close() - f, err := os.Open(newfile) - if err != nil { - log.Printf("Could not open %s: %v", newfile, err) - return - } - defer f.Close() - if _, err := io.Copy(mainf, f); err != nil { - log.Printf("Failed to import %s into %s: %v", newfile, filename, err) - } else { - log.Printf("Imported new user(s) from %s into %s", newfile, filename) - os.Remove(newfile) - } - }() - } - } - } - - file, err := os.Open(filename) + mergeNewPasswdFiles(filename) + users, err := parsePasswdFile(filename, passwdOpts{}) if err != nil { - return fmt.Errorf("failed to open passwd file %s: %w", filename, err) + return err } - defer file.Close() - - proxyMutex.Lock() - defer proxyMutex.Unlock() - - // Clear existing map before loading to support reloading - for k := range passwdMap { - delete(passwdMap, k) - } - - scanner := bufio.NewScanner(file) - lineNumber := 0 - usersLoaded := 0 - newpasswdMap := make(map[string]*UserData) // Temporary map to hold new users - for scanner.Scan() { - lineNumber++ - line := strings.TrimSpace(scanner.Text()) - if line == "" || strings.HasPrefix(line, "#") { // Skip empty lines and comments - continue - } - parts := strings.SplitN(line, "|", 5) // username|bcrypt_hash|maxconns|ExpireAt|(no)post - if len(parts) != 5 { - log.Printf("Skipping malformed line in passwd file (%d) (expected 5 parts, got %d): %s", lineNumber, len(parts), line) - continue - } - // check the splitted line parts - username := strings.TrimSpace(parts[0]) - bcryptHashFromFile := parts[1] - maxconnsStr := parts[2] - ExpireAtStr := parts[3] - posting := strings.HasPrefix(strings.ToLower(parts[4]), "post") - - maxconns, err := strconv.ParseInt(maxconnsStr, 10, 64) - if err != nil { - log.Printf("Skipping line in passwd file (%d) for user '%s' due to invalid maxconns ('%s'): %v", lineNumber, username, maxconnsStr, err) - continue - } - ExpireAt, err := strconv.ParseInt(ExpireAtStr, 10, 64) - if err != nil { - log.Printf("Skipping line in passwd file (%d) for user '%s' due to invalid ExpireAt ('%s'): %v", lineNumber, username, ExpireAtStr, err) - continue - } - if ExpireAt < time.Now().Unix() { - log.Printf("Skipping line in passwd file (%d) for user '%s' due to expired account (ExpireAt: %d, Current: %d)", lineNumber, username, ExpireAt, time.Now().Unix()) - continue - } - - if len(username) < 10 { - log.Printf("Skipping line in passwd file (%d) due to short username: %s", lineNumber, line) - continue - } - if len(bcryptHashFromFile) < 60 { - // Bcrypt hashes are typically 60 characters long, so we check for that. - // If the hash is empty or too short, we skip this user. - log.Printf("Skipping user '%s' on line in passwd file (%d) due to empty/short password hash.", username, lineNumber) - continue - } - - // check if the username already exists in the map - if _, exists := newpasswdMap[username]; exists { - delete(newpasswdMap, username) // Remove existing entry to avoid duplicates - log.Printf("WARNING: Skipping duplicate user '%s' on line %d in passwd file.", username, lineNumber) - continue // Skip duplicate usernames - } - newpasswdMap[username] = &UserData{ - Username: username, - Password: bcryptHashFromFile, // Store the bcrypt hash directly - MaxConns: int(maxconns), - ExpireAt: ExpireAt, - Posting: posting, - } - usersLoaded++ - } - - if err := scanner.Err(); err != nil { - return fmt.Errorf("error reading passwd file %s: %w", filename, err) - } - - if usersLoaded == 0 { - log.Printf("Warning: No users loaded from passwd file '%s'.", filename) - } else { - passwdMap = newpasswdMap - log.Printf("Successfully loaded %d users from '%s'.", usersLoaded, filename) + if len(users) == 0 { + log.Printf("Warning: No users loaded from passwd file '%s', keeping the %d users loaded before.", filename, proxyUserCount()) + return nil } + proxyMutex.Lock() + passwdMap = users // a new map: readers holding the old one are unaffected + proxyMutex.Unlock() + log.Printf("Successfully loaded %d users from '%s'.", len(users), filename) return nil } -// verifyPassword checks if the provided plaintext password matches the stored bcrypt hash for the user. +// proxyUserCount returns the number of loaded proxy users. +func proxyUserCount() int { + proxyMutex.RLock() + defer proxyMutex.RUnlock() + return len(passwdMap) +} + +// verifyPassword checks a proxy user's password and expiry. The map is never +// changed after it is published, so it is read outside proxyMutex. func verifyPassword(user string, plainPassword string) bool { proxyMutex.RLock() - userData, ok := passwdMap[user] + users := passwdMap proxyMutex.RUnlock() - - if !ok { - return false // User not found - } - time.Sleep(time.Duration(mrand.Intn(128)) * time.Millisecond) // small delay - // userData.Password holds the full bcrypt hash string from the .passwd file - err := bcrypt.CompareHashAndPassword([]byte(userData.Password), []byte(plainPassword)) - return err == nil // If err is nil, the password matches + _, ok := verifyUser(users, user, plainPassword) + return ok } -// addUserToPasswdFile adds a new user to the passwd file with a bcrypt hashed password. +// addUserToProxyPasswdFile adds a new user to the passwd file with a bcrypt hashed password. // The UserData.Password field should contain the plaintext password for the new user. func addUserToProxyPasswdFile(userData *UserData, filename string) error { - if userData.Password == "" { - return fmt.Errorf("cannot add user '%s': plaintext password is empty", userData.Username) - } - - // Generate bcrypt hash from the plaintext password - // bcrypt.DefaultCost is 10. You can increase this for more security (e.g., 12-14), - // but it will also be slower to hash and verify. - hashedPasswordBytes, err := bcrypt.GenerateFromPassword([]byte(userData.Password), bcrypt.DefaultCost) - if err != nil { - return fmt.Errorf("failed to generate bcrypt hash for user '%s': %w", userData.Username, err) - } - bcrpytHashString := string(hashedPasswordBytes) - - // Format the line to be appended to the file - // username:bcrypt_hash_string:maxconns:ExpireAt - posting := "nopost" // Default ACL for new users, can be changed if needed - if userData.Posting { - posting = "post" // Set ACL to post if userData.Posting is true - } - newUserLine := fmt.Sprintf("%s|%s|%d|%d|%s\n", userData.Username, bcrpytHashString, userData.MaxConns, userData.ExpireAt, posting) - - // Open the file in append mode, create if it doesn't exist - file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640) - if err != nil { - return fmt.Errorf("failed to open passwd file '%s' for appending: %w", filename, err) - } - defer file.Close() - - if _, err := file.WriteString(newUserLine); err != nil { - return fmt.Errorf("failed to write new user '%s' to passwd file '%s': %w", userData.Username, filename, err) - } - - log.Printf("Successfully added user '%s' to '%s' with a bcrypt hashed password.", userData.Username, filename) - // Optionally, reload passwdMap or add the new user directly to the in-memory map - // For simplicity here, we assume a restart or separate reload mechanism if immediate in-memory update is needed. - // To update in-memory map immediately: - /* TODO - proxyMutex.Lock() - passwdMap[userData.Username] = &UserData{ - Username: userData.Username, - Password: bcrpytHashString, // Store the newly generated hash - MaxConns: userData.MaxConns, - ExpireAt: userData.ExpireAt, - } - proxyMutex.Unlock() - log.Printf("User '%s' also updated in the in-memory passwdMap.", userData.Username) - */ - return nil + return appendPasswdUser(userData, filename) } const DefaultPasswordLength = 8 // 8 bytes will give us 16 hex characters, which is a common length for usernames/passwords diff --git a/WEB_JOB_QUEUE.md b/WEB_JOB_QUEUE.md index 370f53f..5f14b07 100644 --- a/WEB_JOB_QUEUE.md +++ b/WEB_JOB_QUEUE.md @@ -108,44 +108,38 @@ When job finishes: ### Start Web Server ```bash -# HTTP (default) +# local only, no login (allowed only on a loopback address; a warning is logged) ./NZBreX -web -webaddr "127.0.0.1:11911" -cd /cache -provider provider.json -# HTTPS -./NZBreX -web -webssl \ - -webssl-crt "/etc/letsencrypt/fullchain.pem" \ - -webssl-key "/etc/letsencrypt/privkey.pem" \ - -webaddr ":443" -cd /cache -provider provider.json +# with login: create a web user first (its own file, not the proxy's .proxypasswd) +./NZBreX -webpasswd /etc/nzbrex/webpasswd -webadduser 'webadmin01|a-long-password' +./NZBreX -web -webaddr "127.0.0.1:11911" -webpasswd /etc/nzbrex/webpasswd -cd /cache -provider provider.json ``` -### Upload File with Configuration (cURL) -```bash -# Upload with default config -curl -F "file=@myfile.nzb" http://127.0.0.1:11911/upload - -# Upload with custom config -curl -F "file=@myfile.nzb" \ - -F "checkonly=true" \ - -F "crc32=true" \ - -F "cachedir=/custom/cache" \ - http://127.0.0.1:11911/upload -``` +Without `-webpasswd` the server refuses to start on anything but a loopback +address. To reach it from elsewhere, set `-webpasswd`, prefer `-webssl` (TLS 1.2+), +and list the host names it is reached by in `-webhosts`. Exposing it directly on +the internet is not recommended: it drives your provider accounts. -### Manage Jobs (API) +### API (cURL) +Every request that changes something (POST, PUT, PATCH, DELETE) needs the header +`X-NZBreX-Request: 1`; browsers can't send it cross-site, which blocks CSRF. +With a login, add `-u user:password`. ```bash -# List all jobs -curl http://127.0.0.1:11911/api/jobs +# Upload (max 128 MiB). Per-job settings are not supported yet: jobs run with +# the startup settings, and config fields in the form are ignored. +curl -u webadmin01:a-long-password -H "X-NZBreX-Request: 1" \ + -F "file=@myfile.nzb" http://127.0.0.1:11911/upload -# Start job -curl -X POST http://127.0.0.1:11911/api/jobs/job-1/start +# List all jobs +curl -u webadmin01:a-long-password http://127.0.0.1:11911/api/jobs -# Update job config -curl -X PATCH http://127.0.0.1:11911/api/jobs/job-1/config \ - -H "Content-Type: application/json" \ - -d '{"CheckOnly":true,"YencCRC":true}' +# Start one job, or all queued jobs (they run one after another) +curl -u webadmin01:a-long-password -H "X-NZBreX-Request: 1" -X POST http://127.0.0.1:11911/api/jobs/job-1/start +curl -u webadmin01:a-long-password -H "X-NZBreX-Request: 1" -X POST http://127.0.0.1:11911/api/jobs/start-all -# Delete job -curl -X DELETE http://127.0.0.1:11911/api/jobs/job-1 +# Delete a queued or finished job +curl -u webadmin01:a-long-password -H "X-NZBreX-Request: 1" -X DELETE http://127.0.0.1:11911/api/jobs/job-1 ``` ## Web Interface Features diff --git a/passwd.go b/passwd.go new file mode 100644 index 0000000..05d57c1 --- /dev/null +++ b/passwd.go @@ -0,0 +1,197 @@ +package main + +import ( + "bufio" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/crypto/bcrypt" +) + +// Passwd files hold one user per line, passwords as bcrypt hashes: +// +// username|bcrypt_hash|maxconns|ExpireAt_unix|(no)post +// +// The proxy (-proxypasswdfile) and the web server (-webpasswd) use the same +// format but separate files and separate maps: proxy users get no web access. +// The web server ignores maxconns and post. ExpireAt 0 means "never" only where +// passwdOpts.neverExpires allows it (the web file); proxy accounts always expire. + +const minUsernameLen = 10 + +type passwdOpts struct { + neverExpires bool // ExpireAt 0 = no expiry +} + +// passwdHashCost is the bcrypt cost for new users (tests lower it). +var passwdHashCost = bcrypt.DefaultCost + +// dummyHash is compared on a miss, so an unknown user costs as much as a wrong +// password (plan 5.2: the miss path returned before bcrypt, a timing oracle). +// Made on first use, not at startup. +var dummyHash = sync.OnceValue(func() []byte { + h, _ := bcrypt.GenerateFromPassword([]byte("nzbrex-no-such-user"), bcrypt.DefaultCost) + return h +}) + +// parsePasswdFile reads filename into a new map. Malformed, short, duplicate and +// expired entries are skipped with a log line. It never touches a live map. +func parsePasswdFile(filename string, opts passwdOpts) (map[string]*UserData, error) { + file, err := os.Open(filename) + if err != nil { + return nil, fmt.Errorf("failed to open passwd file %s: %w", filename, err) + } + defer file.Close() + + users := make(map[string]*UserData) + dupes := make(map[string]bool) + now := time.Now().Unix() + scanner := bufio.NewScanner(file) + lineNumber := 0 + for scanner.Scan() { + lineNumber++ + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { // Skip empty lines and comments + continue + } + parts := strings.SplitN(line, "|", 5) // username|bcrypt_hash|maxconns|ExpireAt|(no)post + if len(parts) != 5 { + log.Printf("Skipping malformed line %d in passwd file '%s' (expected 5 parts, got %d)", lineNumber, filename, len(parts)) + continue + } + username := strings.TrimSpace(parts[0]) + hash := parts[1] + maxconns, err := strconv.ParseInt(parts[2], 10, 64) + if err != nil { + log.Printf("Skipping line %d in passwd file '%s' for user '%s': invalid maxconns '%s'", lineNumber, filename, username, parts[2]) + continue + } + expireAt, err := strconv.ParseInt(parts[3], 10, 64) + if err != nil { + log.Printf("Skipping line %d in passwd file '%s' for user '%s': invalid ExpireAt '%s'", lineNumber, filename, username, parts[3]) + continue + } + if !(expireAt == 0 && opts.neverExpires) && expireAt < now { + log.Printf("Skipping line %d in passwd file '%s' for user '%s': expired (ExpireAt: %d, now: %d)", lineNumber, filename, username, expireAt, now) + continue + } + if len(username) < minUsernameLen { + log.Printf("Skipping line %d in passwd file '%s': username shorter than %d characters", lineNumber, filename, minUsernameLen) + continue + } + if len(hash) < 60 { + // bcrypt hashes are 60 characters long + log.Printf("Skipping user '%s' on line %d in passwd file '%s': empty or short password hash", username, lineNumber, filename) + continue + } + if dupes[username] { + continue + } + if _, exists := users[username]; exists { + // a name listed twice is ambiguous: drop it entirely + delete(users, username) + dupes[username] = true + log.Printf("WARNING: Skipping duplicate user '%s' on line %d in passwd file '%s'.", username, lineNumber, filename) + continue + } + users[username] = &UserData{ + Username: username, + Password: hash, + MaxConns: int(maxconns), + ExpireAt: expireAt, + Posting: strings.HasPrefix(strings.ToLower(parts[4]), "post"), + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error reading passwd file %s: %w", filename, err) + } + return users, nil +} + +// verifyUser checks user and password against users. It costs one bcrypt +// comparison whether or not the user exists, and rejects expired accounts. +func verifyUser(users map[string]*UserData, user, plainPassword string) (*UserData, bool) { + userData, ok := users[user] + hash := dummyHash() + if ok { + hash = []byte(userData.Password) + } + match := bcrypt.CompareHashAndPassword(hash, []byte(plainPassword)) == nil + if !ok || !match { + return nil, false + } + if userData.ExpireAt != 0 && userData.ExpireAt < time.Now().Unix() { + return nil, false + } + return userData, true +} + +// mergeNewPasswdFiles appends filename.new.* files (written by -proxyadduser / +// -webadduser) to filename and removes them. +func mergeNewPasswdFiles(filename string) { + dir := filepath.Dir(filename) + pattern := filepath.Base(filename) + ".new." + entries, err := os.ReadDir(dir) + if err != nil { + return + } + for _, entry := range entries { + if entry.IsDir() || !strings.HasPrefix(entry.Name(), pattern) { + continue + } + newfile := filepath.Join(dir, entry.Name()) + func() { + mainf, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640) + if err != nil { + log.Printf("Could not open %s for appending: %v", filename, err) + return + } + defer mainf.Close() + f, err := os.Open(newfile) + if err != nil { + log.Printf("Could not open %s: %v", newfile, err) + return + } + defer f.Close() + if _, err := io.Copy(mainf, f); err != nil { + log.Printf("Failed to import %s into %s: %v", newfile, filename, err) + return + } + log.Printf("Imported new user(s) from %s into %s", newfile, filename) + os.Remove(newfile) + }() + } +} + +// appendPasswdUser hashes userData.Password (plaintext) and appends the user to filename. +func appendPasswdUser(userData *UserData, filename string) error { + if userData.Password == "" { + return fmt.Errorf("cannot add user '%s': plaintext password is empty", userData.Username) + } + hashed, err := bcrypt.GenerateFromPassword([]byte(userData.Password), passwdHashCost) + if err != nil { + return fmt.Errorf("failed to generate bcrypt hash for user '%s': %w", userData.Username, err) + } + posting := "nopost" + if userData.Posting { + posting = "post" + } + line := fmt.Sprintf("%s|%s|%d|%d|%s\n", userData.Username, hashed, userData.MaxConns, userData.ExpireAt, posting) + file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640) + if err != nil { + return fmt.Errorf("failed to open passwd file '%s' for appending: %w", filename, err) + } + defer file.Close() + if _, err := file.WriteString(line); err != nil { + return fmt.Errorf("failed to write new user '%s' to passwd file '%s': %w", userData.Username, filename, err) + } + log.Printf("Successfully added user '%s' to '%s' with a bcrypt hashed password.", userData.Username, filename) + return nil +} diff --git a/review-the-codebase-for-jolly-petal.md b/review-the-codebase-for-jolly-petal.md index 2f93a6b..3046e8b 100644 --- a/review-the-codebase-for-jolly-petal.md +++ b/review-the-codebase-for-jolly-petal.md @@ -415,6 +415,18 @@ Still races: - `gin.Default()` trusts all proxies (`0.0.0.0/0`): call `router.SetTrustedProxies(nil)`. - Bootstrap comes from `cdn.jsdelivr.net` without SRI in all four templates and breaks on an offline host. Embed it locally (the directory is already embedded). +**Result (2026-10-18):** done. +- **Login (D1):** `passwd.go` holds the shared code: `parsePasswdFile` (into a new map), `verifyUser` (bcrypt against a dummy hash on a miss, expiry checked), `mergeNewPasswdFiles`, `appendPasswdUser`. The proxy keeps its global `passwdMap` through thin wrappers; the web server has its own map (`webauth.go`): HTTP basic auth against `-webpasswd`, users added with `-webadduser 'user|password'` (written to `.new.*`, merged on load; web users have `ExpireAt 0` = never, allowed only in the web file). The file is reloaded when it changes. Verified logins are cached for 5 min (key = SHA-256 of user and password); at most 4 bcrypt checks run at once. No `-webpasswd`: refuses to start unless bound to loopback, and warns there. A file with no usable users is a start error. +- **CSRF:** every POST/PUT/PATCH/DELETE needs `X-NZBreX-Request: 1` (forces a preflight the server never answers); `Sec-Fetch-Site` must be `same-origin`/`none` and `Origin` must match `Host` when sent. `index.html`/`jobs.html` send the header. +- **Host allowlist:** without a login always enforced (localhost, 127.0.0.1, ::1, the `-webaddr` host, `-webhosts`), 421 otherwise. With a login only when `-webhosts` is set. +- **Server:** `ReadHeaderTimeout` 10 s, `IdleTimeout` 60 s; uploads capped at 128 MiB with `http.MaxBytesReader` (413); `MaxMultipartMemory` 8 MiB; `SetTrustedProxies(nil)`; TLS `MinVersion` 1.2 (with 4.1). +- **Uploads:** `os.MkdirTemp` (0700) at start; each upload is written by our own code with `O_CREATE|O_EXCL` under a unique `-` (an existing file or a planted symlink makes it fail). The job keeps the client's name for display. The file is deleted when the job finishes or is deleted. +- **Bootstrap:** embedded from Debian's `libjs-bootstrap5` 5.3.5 (`web/static/`, MIT headers kept, source-map comments removed); `style.css` moved there; served by `StaticFS`. +- `WEB_JOB_QUEUE.md`: the `:443` example is gone; the usage section documents `-webpasswd`, the header and `-u` (rest of the file: 6.3). +- Tests: `TestWebGuardBlocksCrossSiteAndUnknownHosts`, `TestWebAuthLoginCacheAndReload`, `TestStartWebServerRefusesPublicWithoutLogin`, `TestSaveUploadUniqueAndNoSymlinkFollow`. +- Verified with `-race` against the local servers: web race run with login and header, 2 jobs completed, 0 races. Probes: no login 401, wrong password 401, pages and static files 200, POST without header 403, foreign Origin 403, form upload without header 403, 140 MB upload 413, two uploads of one name → two files, delete removes the file, public bind without `-webpasswd` exits 1. Proxy login still works (281 / 481). +- **Not done:** graceful shutdown removes the upload dir (4.5). A brute-force limit beyond bcrypt's cost and the 4-check cap. + ### 4.5 Job lifecycle (corrected) - **Panic recovery** on the job goroutine (`:290`): add it, but it only catches panics on that goroutine. Worker goroutines and the runtime's `fatal error: concurrent map` are not recoverable there; Phase 2 is the real fix. @@ -438,6 +450,8 @@ Still races: - `index.html:169-176` `showNotification` inserts `${message}` without `escapeHtml`. Latent (callers at `:80`, `:85` pass static text). Escape it anyway. - Fine as-is: templates and CSS are embedded (`webserver.go:17-18`, `:495`, `:503`); no `template.HTML` or `text/template`. +**Partial result (2026-10-18):** with 4.2/4.4: `base.html` deleted; `handleJobsList`/`handleStats` (and `handleIndex`) pass no data; `showNotification` escapes its message; the Config button is deliberately disabled (D2). **Still open:** nothing else in 4.6. + --- ## Phase 5: NNTP proxy (corrected) @@ -468,10 +482,12 @@ The proxy runs alone in its own mode (see Context), so its bugs affect only prox - `ARTICLE ` resets the selected group (`Server.go:352`), which §6.2.1.2 forbids. **The reset is load-bearing:** `Server.go:413`/`:431` treat `ps.Group != "" && ps.MsgNum > 0` as "request by number". Add a local `byNumber` flag first, or message-ID requests get misrouted. `:357` also moves `MsgNum` before the request has succeeded. - Article number field: `0` is correct for message-ID requests (§6.2.1.2). Only numeric requests are wrong (the number is hardcoded and also lands in the message-ID slot, `ProxyHelper.go:40`, `Server.go:510`). Use the upstream response line for numeric requests. - **`passwdMap` wiped in place.** `ProxyHelper.go:132-134` clears the live map before parsing. On `usersLoaded == 0` (`:204`) or `scanner.Err` (`:200`) nothing is restored. Reload runs only after a successful `verifyPassword` (`Server.go:167` before `:173-188`), so once the map is empty nobody can log in again until restart. **Fix:** parse into a new map and swap it in only on success. Policy note: emptying the file on purpose then no longer locks everyone out. + **Fixed (2026-10-18, with 4.4):** `loadPasswdFile` parses into a new map and swaps it in only if it parsed and has users; emptying the file keeps the old users (logged). `TestLoadPasswdFileKeepsUsersOnBadReload`. - **(dropped)** "The `.new.*` merge lets anyone add users": this merge is how `-proxyadduser` delivers new users (`Flags.go:176`), and anyone with write access to that directory can already replace `.proxypasswd`. **Keep it.** - **`ProxySessions` keyed by username** (`Server.go:240`, deleted at `:44`): never read anywhere. Delete the map, or switch it to `map[uint64]` keyed by `ps.id`. - **Plaintext password kept** in `ps.Password` (`Server.go:232`), never read. Delete the field. - **Timing oracle for unknown users** (`ProxyHelper.go:219-221` returns before the sleep and bcrypt). Run bcrypt against a dummy hash of the same cost (`DefaultCost`) on the miss path. + **Fixed (2026-10-18, with 4.4):** `verifyUser` compares against a lazily made `DefaultCost` dummy hash on a miss and checks expiry; the random sleep in `verifyPassword` is gone (`Server.go` keeps its own). `TestVerifyUserUnknownCostsBcrypt`. - **`AUTHINFO PASS` split on whitespace** (`Server.go:100`, `:165`, plus `TrimSpace` at `:99`). (Corrected: RFC 4643 does not require taking the rest of the line; its grammar excludes spaces, and §2.3.2 makes whitespace a MAY.) The effect is the reverse of the draft: a password with spaces *fails*, and `correctpw ` is *accepted*. Optional: reject extra arguments. - **Client writes unchecked** almost everywhere (the `LinesWriter` error at `Server.go:481` is checked). Check them, so handlers stop fetching upstream for clients that are gone. - `ps.Close()` (`ProxyHelper.go:311-319`) logs the username after clearing it, and would skip the deferred `CountConns` decrement (`Server.go:35`). Currently unreachable. diff --git a/web/static/bootstrap.bundle.min.js b/web/static/bootstrap.bundle.min.js new file mode 100644 index 0000000..8a290d5 --- /dev/null +++ b/web/static/bootstrap.bundle.min.js @@ -0,0 +1,7 @@ +/*! + * Bootstrap v5.3.5 (https://getbootstrap.com/) + * Copyright 2011-2023 The Bootstrap Authors (https://github.com/twbs/bootstrap/graphs/contributors) + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE) + */(function(Et,S){typeof exports=="object"&&typeof module<"u"?module.exports=S():typeof define=="function"&&define.amd?define(S):(Et=typeof globalThis<"u"?globalThis:Et||self,Et.bootstrap=S())})(this,function(){"use strict";function Et(a,e){for(var r=0;ra.length)&&(e=a.length);for(var r=0,n=new Array(e);r=a.length?{done:!0}:{done:!1,value:a[n++]}}}throw new TypeError(`Invalid attempt to iterate non-iterable instance. +In order to be iterable, non-array objects must have a [Symbol.iterator]() method.`)}function ci(a,e){if(typeof a!="object"||a===null)return a;var r=a[Symbol.toPrimitive];if(r!==void 0){var n=r.call(a,e||"default");if(typeof n!="object")return n;throw new TypeError("@@toPrimitive must return a primitive value.")}return(e==="string"?String:Number)(a)}function fi(a){var e=ci(a,"string");return typeof e=="symbol"?e:String(e)}var et=new Map;const Te={set:function(e,r,n){et.has(e)||et.set(e,new Map);var t=et.get(e);if(!t.has(r)&&t.size!==0){console.error("Bootstrap doesn't allow more than one instance per element. Bound instance: "+Array.from(t.keys())[0]+".");return}t.set(r,n)},get:function(e,r){return et.has(e)&&et.get(e).get(r)||null},remove:function(e,r){if(et.has(e)){var n=et.get(e);n.delete(r),n.size===0&&et.delete(e)}}};var di=1e6,vi=1e3,we="transitionend",_r=function(e){return e&&window.CSS&&window.CSS.escape&&(e=e.replace(/#([^\s"#']+)/g,function(r,n){return"#"+CSS.escape(n)})),e},hi=function(e){return e==null?""+e:Object.prototype.toString.call(e).match(/\s([a-z]+)/i)[1].toLowerCase()},pi=function(e){do e+=Math.floor(Math.random()*di);while(document.getElementById(e));return e},_i=function(e){if(!e)return 0;var r=window.getComputedStyle(e),n=r.transitionDuration,t=r.transitionDelay,i=Number.parseFloat(n),o=Number.parseFloat(t);return!i&&!o?0:(n=n.split(",")[0],t=t.split(",")[0],(Number.parseFloat(n)+Number.parseFloat(t))*vi)},gr=function(e){e.dispatchEvent(new Event(we))},X=function(e){return!e||typeof e!="object"?!1:(typeof e.jquery<"u"&&(e=e[0]),typeof e.nodeType<"u")},rt=function(e){return X(e)?e.jquery?e[0]:e:typeof e=="string"&&e.length>0?document.querySelector(_r(e)):null},bt=function(e){if(!X(e)||e.getClientRects().length===0)return!1;var r=getComputedStyle(e).getPropertyValue("visibility")==="visible",n=e.closest("details:not([open])");if(!n)return r;if(n!==e){var t=e.closest("summary");if(t&&t.parentNode!==n||t===null)return!1}return r},nt=function(e){return!e||e.nodeType!==Node.ELEMENT_NODE||e.classList.contains("disabled")?!0:typeof e.disabled<"u"?e.disabled:e.hasAttribute("disabled")&&e.getAttribute("disabled")!=="false"},gi=function a(e){if(!document.documentElement.attachShadow)return null;if(typeof e.getRootNode=="function"){var r=e.getRootNode();return r instanceof ShadowRoot?r:null}return e instanceof ShadowRoot?e:e.parentNode?a(e.parentNode):null},Xt=function(){},xt=function(e){e.offsetHeight},mr=function(){return window.jQuery&&!document.body.hasAttribute("data-bs-no-jquery")?window.jQuery:null},Oe=[],mi=function(e){document.readyState==="loading"?(Oe.length||document.addEventListener("DOMContentLoaded",function(){for(var r=0,n=Oe;r1?0:t.touches[0].clientX-this._deltaX},r._handleSwipe=function(){var t=Math.abs(this._deltaX);if(!(t<=Gi)){var i=t/this._deltaX;this._deltaX=0,i&&R(i>0?this._config.rightCallback:this._config.leftCallback)}},r._initEvents=function(){var t=this;this._supportPointerEvents?(f.on(this._element,Bi,function(i){return t._start(i)}),f.on(this._element,Ki,function(i){return t._end(i)}),this._element.classList.add(zi)):(f.on(this._element,Wi,function(i){return t._start(i)}),f.on(this._element,ji,function(i){return t._move(i)}),f.on(this._element,Fi,function(i){return t._end(i)}))},r._eventIsPointerPenTouch=function(t){return this._supportPointerEvents&&(t.pointerType===Ui||t.pointerType===Yi)},e.isSupported=function(){return"ontouchstart"in document.documentElement||navigator.maxTouchPoints>0},S(e,null,[{key:"Default",get:function(){return qi}},{key:"DefaultType",get:function(){return Qi}},{key:"NAME",get:function(){return Hi}}]),e}($t),Jt,Xi="carousel",Zi="bs.carousel",it="."+Zi,kr=".data-api",Ji="ArrowLeft",ta="ArrowRight",ea=500,Vt="next",At="prev",Tt="left",te="right",ra="slide"+it,Me="slid"+it,na="keydown"+it,ia="mouseenter"+it,aa="mouseleave"+it,oa="dragstart"+it,sa="load"+it+kr,la="click"+it+kr,Pr="carousel",ee="active",ua="slide",ca="carousel-item-end",fa="carousel-item-start",da="carousel-item-next",va="carousel-item-prev",Mr=".active",Rr=".carousel-item",ha=Mr+Rr,pa=".carousel-item img",_a=".carousel-indicators",ga="[data-bs-slide], [data-bs-slide-to]",ma='[data-bs-ride="carousel"]',Ea=(Jt={},Jt[Ji]=te,Jt[ta]=Tt,Jt),ba={interval:5e3,keyboard:!0,pause:"hover",ride:!1,touch:!0,wrap:!0},ya={interval:"(number|boolean)",keyboard:"boolean",pause:"(string|boolean)",ride:"(boolean|string)",touch:"boolean",wrap:"boolean"},re=function(a){D(e,a);function e(n,t){var i;return i=a.call(this,n,t)||this,i._interval=null,i._activeElement=null,i._isSliding=!1,i.touchTimeout=null,i._swipeHelper=null,i._indicatorsElement=p.findOne(_a,i._element),i._addEventListeners(),i._config.ride===Pr&&i.cycle(),i}var r=e.prototype;return r.next=function(){this._slide(Vt)},r.nextWhenVisible=function(){!document.hidden&&bt(this._element)&&this.next()},r.prev=function(){this._slide(At)},r.pause=function(){this._isSliding&&gr(this._element),this._clearInterval()},r.cycle=function(){var t=this;this._clearInterval(),this._updateInterval(),this._interval=setInterval(function(){return t.nextWhenVisible()},this._config.interval)},r._maybeEnableCycle=function(){var t=this;if(this._config.ride){if(this._isSliding){f.one(this._element,Me,function(){return t.cycle()});return}this.cycle()}},r.to=function(t){var i=this,o=this._getItems();if(!(t>o.length-1||t<0)){if(this._isSliding){f.one(this._element,Me,function(){return i.to(t)});return}var s=this._getItemIndex(this._getActive());if(s!==t){var l=t>s?Vt:At;this._slide(l,o[t])}}},r.dispose=function(){this._swipeHelper&&this._swipeHelper.dispose(),a.prototype.dispose.call(this)},r._configAfterMerge=function(t){return t.defaultInterval=t.interval,t},r._addEventListeners=function(){var t=this;this._config.keyboard&&f.on(this._element,na,function(i){return t._keydown(i)}),this._config.pause==="hover"&&(f.on(this._element,ia,function(){return t.pause()}),f.on(this._element,aa,function(){return t._maybeEnableCycle()})),this._config.touch&&Ir.isSupported()&&this._addTouchEventListeners()},r._addTouchEventListeners=function(){for(var t=this,i=y(p.find(pa,this._element)),o;!(o=i()).done;){var s=o.value;f.on(s,oa,function(c){return c.preventDefault()})}var l=function(){t._config.pause==="hover"&&(t.pause(),t.touchTimeout&&clearTimeout(t.touchTimeout),t.touchTimeout=setTimeout(function(){return t._maybeEnableCycle()},ea+t._config.interval))},u={leftCallback:function(){return t._slide(t._directionToOrder(Tt))},rightCallback:function(){return t._slide(t._directionToOrder(te))},endCallback:l};this._swipeHelper=new Ir(this._element,u)},r._keydown=function(t){if(!/input|textarea/i.test(t.target.tagName)){var i=Ea[t.key];i&&(t.preventDefault(),this._slide(this._directionToOrder(i)))}},r._getItemIndex=function(t){return this._getItems().indexOf(t)},r._setActiveIndicatorElement=function(t){if(this._indicatorsElement){var i=p.findOne(Mr,this._indicatorsElement);i.classList.remove(ee),i.removeAttribute("aria-current");var o=p.findOne('[data-bs-slide-to="'+t+'"]',this._indicatorsElement);o&&(o.classList.add(ee),o.setAttribute("aria-current","true"))}},r._updateInterval=function(){var t=this._activeElement||this._getActive();if(t){var i=Number.parseInt(t.getAttribute("data-bs-interval"),10);this._config.interval=i||this._config.defaultInterval}},r._slide=function(t,i){var o=this;if(i===void 0&&(i=null),!this._isSliding){var s=this._getActive(),l=t===Vt,u=i||Ce(this._getItems(),s,l,this._config.wrap);if(u!==s){var c=this._getItemIndex(u),d=function(A){return f.trigger(o._element,A,{relatedTarget:u,direction:o._orderToDirection(t),from:o._getItemIndex(s),to:c})},v=d(ra);if(!v.defaultPrevented&&!(!s||!u)){var h=!!this._interval;this.pause(),this._isSliding=!0,this._setActiveIndicatorElement(c),this._activeElement=u;var _=l?fa:ca,g=l?da:va;u.classList.add(g),xt(u),s.classList.add(_),u.classList.add(_);var m=function(){u.classList.remove(_,g),u.classList.add(ee),s.classList.remove(ee,g,_),o._isSliding=!1,d(Me)};this._queueCallback(m,s,this._isAnimated()),h&&this.cycle()}}}},r._isAnimated=function(){return this._element.classList.contains(ua)},r._getActive=function(){return p.findOne(ha,this._element)},r._getItems=function(){return p.find(Rr,this._element)},r._clearInterval=function(){this._interval&&(clearInterval(this._interval),this._interval=null)},r._directionToOrder=function(t){return W()?t===Tt?At:Vt:t===Tt?Vt:At},r._orderToDirection=function(t){return W()?t===At?Tt:te:t===At?te:Tt},e.jQueryInterface=function(t){return this.each(function(){var i=e.getOrCreateInstance(this,t);if(typeof t=="number"){i.to(t);return}if(typeof t=="string"){if(i[t]===void 0||t.startsWith("_")||t==="constructor")throw new TypeError('No method named "'+t+'"');i[t]()}})},S(e,null,[{key:"Default",get:function(){return ba}},{key:"DefaultType",get:function(){return ya}},{key:"NAME",get:function(){return Xi}}]),e}(K);f.on(document,la,ga,function(a){var e=p.getElementFromSelector(this);if(!(!e||!e.classList.contains(Pr))){a.preventDefault();var r=re.getOrCreateInstance(e),n=this.getAttribute("data-bs-slide-to");if(n){r.to(n),r._maybeEnableCycle();return}if(Z.getDataAttribute(this,"slide")==="next"){r.next(),r._maybeEnableCycle();return}r.prev(),r._maybeEnableCycle()}}),f.on(window,sa,function(){for(var a=p.find(ma),e=y(a),r;!(r=e()).done;){var n=r.value;re.getOrCreateInstance(n)}}),j(re);var Aa="collapse",Ta="bs.collapse",Ht="."+Ta,wa=".data-api",Oa="show"+Ht,Ca="shown"+Ht,Sa="hide"+Ht,Na="hidden"+Ht,Da="click"+Ht+wa,Re="show",wt="collapse",ne="collapsing",La="collapsed",Ia=":scope ."+wt+" ."+wt,ka="collapse-horizontal",Pa="width",Ma="height",Ra=".collapse.show, .collapse.collapsing",xe='[data-bs-toggle="collapse"]',xa={parent:null,toggle:!0},$a={parent:"(null|element)",toggle:"boolean"},$e=function(a){D(e,a);function e(n,t){var i;i=a.call(this,n,t)||this,i._isTransitioning=!1,i._triggerArray=[];for(var o=p.find(xe),s=y(o),l;!(l=s()).done;){var u=l.value,c=p.getSelectorFromElement(u),d=p.find(c).filter(function(v){return v===i._element});c!==null&&d.length&&i._triggerArray.push(u)}return i._initializeChildren(),i._config.parent||i._addAriaAndCollapsedClass(i._triggerArray,i._isShown()),i._config.toggle&&i.toggle(),i}var r=e.prototype;return r.toggle=function(){this._isShown()?this.hide():this.show()},r.show=function(){var t=this;if(!(this._isTransitioning||this._isShown())){var i=[];if(this._config.parent&&(i=this._getFirstLevelChildren(Ra).filter(function(_){return _!==t._element}).map(function(_){return e.getOrCreateInstance(_,{toggle:!1})})),!(i.length&&i[0]._isTransitioning)){var o=f.trigger(this._element,Oa);if(!o.defaultPrevented){for(var s=y(i),l;!(l=s()).done;){var u=l.value;u.hide()}var c=this._getDimension();this._element.classList.remove(wt),this._element.classList.add(ne),this._element.style[c]=0,this._addAriaAndCollapsedClass(this._triggerArray,!0),this._isTransitioning=!0;var d=function(){t._isTransitioning=!1,t._element.classList.remove(ne),t._element.classList.add(wt,Re),t._element.style[c]="",f.trigger(t._element,Ca)},v=c[0].toUpperCase()+c.slice(1),h="scroll"+v;this._queueCallback(d,this._element,!0),this._element.style[c]=this._element[h]+"px"}}}},r.hide=function(){var t=this;if(!(this._isTransitioning||!this._isShown())){var i=f.trigger(this._element,Sa);if(!i.defaultPrevented){var o=this._getDimension();this._element.style[o]=this._element.getBoundingClientRect()[o]+"px",xt(this._element),this._element.classList.add(ne),this._element.classList.remove(wt,Re);for(var s=y(this._triggerArray),l;!(l=s()).done;){var u=l.value,c=p.getElementFromSelector(u);c&&!this._isShown(c)&&this._addAriaAndCollapsedClass([u],!1)}this._isTransitioning=!0;var d=function(){t._isTransitioning=!1,t._element.classList.remove(ne),t._element.classList.add(wt),f.trigger(t._element,Na)};this._element.style[o]="",this._queueCallback(d,this._element,!0)}}},r._isShown=function(t){return t===void 0&&(t=this._element),t.classList.contains(Re)},r._configAfterMerge=function(t){return t.toggle=!!t.toggle,t.parent=rt(t.parent),t},r._getDimension=function(){return this._element.classList.contains(ka)?Pa:Ma},r._initializeChildren=function(){if(this._config.parent)for(var t=this._getFirstLevelChildren(xe),i=y(t),o;!(o=i()).done;){var s=o.value,l=p.getElementFromSelector(s);l&&this._addAriaAndCollapsedClass([s],this._isShown(l))}},r._getFirstLevelChildren=function(t){var i=p.find(Ia,this._config.parent);return p.find(t,this._config.parent).filter(function(o){return!i.includes(o)})},r._addAriaAndCollapsedClass=function(t,i){if(t.length)for(var o=y(t),s;!(s=o()).done;){var l=s.value;l.classList.toggle(La,!i),l.setAttribute("aria-expanded",i)}},e.jQueryInterface=function(t){var i={};return typeof t=="string"&&/show|hide/.test(t)&&(i.toggle=!1),this.each(function(){var o=e.getOrCreateInstance(this,i);if(typeof t=="string"){if(typeof o[t]>"u")throw new TypeError('No method named "'+t+'"');o[t]()}})},S(e,null,[{key:"Default",get:function(){return xa}},{key:"DefaultType",get:function(){return $a}},{key:"NAME",get:function(){return Aa}}]),e}(K);f.on(document,Da,xe,function(a){(a.target.tagName==="A"||a.delegateTarget&&a.delegateTarget.tagName==="A")&&a.preventDefault();for(var e=y(p.getMultipleElementsFromSelector(this)),r;!(r=e()).done;){var n=r.value;$e.getOrCreateInstance(n,{toggle:!1}).toggle()}}),j($e);var P="top",x="bottom",$="right",M="left",ie="auto",Ot=[P,x,$,M],lt="start",Ct="end",xr="clippingParents",Ve="viewport",St="popper",$r="reference",He=Ot.reduce(function(a,e){return a.concat([e+"-"+lt,e+"-"+Ct])},[]),We=[].concat(Ot,[ie]).reduce(function(a,e){return a.concat([e,e+"-"+lt,e+"-"+Ct])},[]),Vr="beforeRead",Hr="read",Wr="afterRead",jr="beforeMain",Fr="main",Br="afterMain",Kr="beforeWrite",Yr="write",Ur="afterWrite",zr=[Vr,Hr,Wr,jr,Fr,Br,Kr,Yr,Ur];function q(a){return a?(a.nodeName||"").toLowerCase():null}function Y(a){if(a==null)return window;if(a.toString()!=="[object Window]"){var e=a.ownerDocument;return e&&e.defaultView||window}return a}function Nt(a){var e=Y(a).Element;return a instanceof e||a instanceof Element}function F(a){var e=Y(a).HTMLElement;return a instanceof e||a instanceof HTMLElement}function Gr(a){if(typeof ShadowRoot>"u")return!1;var e=Y(a).ShadowRoot;return a instanceof e||a instanceof ShadowRoot}function Va(a){var e=a.state;Object.keys(e.elements).forEach(function(r){var n=e.styles[r]||{},t=e.attributes[r]||{},i=e.elements[r];!F(i)||!q(i)||(Object.assign(i.style,n),Object.keys(t).forEach(function(o){var s=t[o];s===!1?i.removeAttribute(o):i.setAttribute(o,s===!0?"":s)}))})}function Ha(a){var e=a.state,r={popper:{position:e.options.strategy,left:"0",top:"0",margin:"0"},arrow:{position:"absolute"},reference:{}};return Object.assign(e.elements.popper.style,r.popper),e.styles=r,e.elements.arrow&&Object.assign(e.elements.arrow.style,r.arrow),function(){Object.keys(e.elements).forEach(function(n){var t=e.elements[n],i=e.attributes[n]||{},o=Object.keys(e.styles.hasOwnProperty(n)?e.styles[n]:r[n]),s=o.reduce(function(l,u){return l[u]="",l},{});!F(t)||!q(t)||(Object.assign(t.style,s),Object.keys(i).forEach(function(l){t.removeAttribute(l)}))})}}const je={name:"applyStyles",enabled:!0,phase:"write",fn:Va,effect:Ha,requires:["computeStyles"]};function Q(a){return a.split("-")[0]}var ut=Math.max,ae=Math.min,Dt=Math.round;function Lt(a,e){e===void 0&&(e=!1);var r=a.getBoundingClientRect(),n=1,t=1;if(F(a)&&e){var i=a.offsetHeight,o=a.offsetWidth;o>0&&(n=Dt(r.width)/o||1),i>0&&(t=Dt(r.height)/i||1)}return{width:r.width/n,height:r.height/t,top:r.top/t,right:r.right/n,bottom:r.bottom/t,left:r.left/n,x:r.left/n,y:r.top/t}}function Fe(a){var e=Lt(a),r=a.offsetWidth,n=a.offsetHeight;return Math.abs(e.width-r)<=1&&(r=e.width),Math.abs(e.height-n)<=1&&(n=e.height),{x:a.offsetLeft,y:a.offsetTop,width:r,height:n}}function qr(a,e){var r=e.getRootNode&&e.getRootNode();if(a.contains(e))return!0;if(r&&Gr(r)){var n=e;do{if(n&&a.isSameNode(n))return!0;n=n.parentNode||n.host}while(n)}return!1}function J(a){return Y(a).getComputedStyle(a)}function Wa(a){return["table","td","th"].indexOf(q(a))>=0}function at(a){return((Nt(a)?a.ownerDocument:a.document)||window.document).documentElement}function oe(a){return q(a)==="html"?a:a.assignedSlot||a.parentNode||(Gr(a)?a.host:null)||at(a)}function Qr(a){return!F(a)||J(a).position==="fixed"?null:a.offsetParent}function ja(a){var e=navigator.userAgent.toLowerCase().indexOf("firefox")!==-1,r=navigator.userAgent.indexOf("Trident")!==-1;if(r&&F(a)){var n=J(a);if(n.position==="fixed")return null}for(var t=oe(a);F(t)&&["html","body"].indexOf(q(t))<0;){var i=J(t);if(i.transform!=="none"||i.perspective!=="none"||i.contain==="paint"||["transform","perspective"].indexOf(i.willChange)!==-1||e&&i.willChange==="filter"||e&&i.filter&&i.filter!=="none")return t;t=t.parentNode}return null}function Wt(a){for(var e=Y(a),r=Qr(a);r&&Wa(r)&&J(r).position==="static";)r=Qr(r);return r&&(q(r)==="html"||q(r)==="body"&&J(r).position==="static")?e:r||ja(a)||e}function Be(a){return["top","bottom"].indexOf(a)>=0?"x":"y"}function jt(a,e,r){return ut(a,ae(e,r))}function Fa(a,e,r){var n=jt(a,e,r);return n>r?r:n}function Xr(){return{top:0,right:0,bottom:0,left:0}}function Zr(a){return Object.assign({},Xr(),a)}function Jr(a,e){return e.reduce(function(r,n){return r[n]=a,r},{})}var Ba=function(e,r){return e=typeof e=="function"?e(Object.assign({},r.rects,{placement:r.placement})):e,Zr(typeof e!="number"?e:Jr(e,Ot))};function Ka(a){var e,r=a.state,n=a.name,t=a.options,i=r.elements.arrow,o=r.modifiersData.popperOffsets,s=Q(r.placement),l=Be(s),u=[M,$].indexOf(s)>=0,c=u?"height":"width";if(!(!i||!o)){var d=Ba(t.padding,r),v=Fe(i),h=l==="y"?P:M,_=l==="y"?x:$,g=r.rects.reference[c]+r.rects.reference[l]-o[l]-r.rects.popper[c],m=o[l]-r.rects.reference[l],b=Wt(i),A=b?l==="y"?b.clientHeight||0:b.clientWidth||0:0,T=g/2-m/2,E=d[h],w=A-v[c]-d[_],O=A/2-v[c]/2+T,C=jt(E,O,w),N=l;r.modifiersData[n]=(e={},e[N]=C,e.centerOffset=C-O,e)}}function Ya(a){var e=a.state,r=a.options,n=r.element,t=n===void 0?"[data-popper-arrow]":n;t!=null&&(typeof t=="string"&&(t=e.elements.popper.querySelector(t),!t)||qr(e.elements.popper,t)&&(e.elements.arrow=t))}const tn={name:"arrow",enabled:!0,phase:"main",fn:Ka,effect:Ya,requires:["popperOffsets"],requiresIfExists:["preventOverflow"]};function It(a){return a.split("-")[1]}var Ua={top:"auto",right:"auto",bottom:"auto",left:"auto"};function za(a){var e=a.x,r=a.y,n=window,t=n.devicePixelRatio||1;return{x:Dt(e*t)/t||0,y:Dt(r*t)/t||0}}function en(a){var e,r=a.popper,n=a.popperRect,t=a.placement,i=a.variation,o=a.offsets,s=a.position,l=a.gpuAcceleration,u=a.adaptive,c=a.roundOffsets,d=a.isFixed,v=o.x,h=v===void 0?0:v,_=o.y,g=_===void 0?0:_,m=typeof c=="function"?c({x:h,y:g}):{x:h,y:g};h=m.x,g=m.y;var b=o.hasOwnProperty("x"),A=o.hasOwnProperty("y"),T=M,E=P,w=window;if(u){var O=Wt(r),C="clientHeight",N="clientWidth";if(O===Y(r)&&(O=at(r),J(O).position!=="static"&&s==="absolute"&&(C="scrollHeight",N="scrollWidth")),O=O,t===P||(t===M||t===$)&&i===Ct){E=x;var k=d&&w.visualViewport?w.visualViewport.height:O[C];g-=k-n.height,g*=l?1:-1}if(t===M||(t===P||t===x)&&i===Ct){T=$;var L=d&&w.visualViewport?w.visualViewport.width:O[N];h-=L-n.width,h*=l?1:-1}}var I=Object.assign({position:s},u&&Ua),U=c===!0?za({x:h,y:g}):{x:h,y:g};if(h=U.x,g=U.y,l){var V;return Object.assign({},I,(V={},V[E]=A?"0":"",V[T]=b?"0":"",V.transform=(w.devicePixelRatio||1)<=1?"translate("+h+"px, "+g+"px)":"translate3d("+h+"px, "+g+"px, 0)",V))}return Object.assign({},I,(e={},e[E]=A?g+"px":"",e[T]=b?h+"px":"",e.transform="",e))}function Ga(a){var e=a.state,r=a.options,n=r.gpuAcceleration,t=n===void 0?!0:n,i=r.adaptive,o=i===void 0?!0:i,s=r.roundOffsets,l=s===void 0?!0:s,u={placement:Q(e.placement),variation:It(e.placement),popper:e.elements.popper,popperRect:e.rects.popper,gpuAcceleration:t,isFixed:e.options.strategy==="fixed"};e.modifiersData.popperOffsets!=null&&(e.styles.popper=Object.assign({},e.styles.popper,en(Object.assign({},u,{offsets:e.modifiersData.popperOffsets,position:e.options.strategy,adaptive:o,roundOffsets:l})))),e.modifiersData.arrow!=null&&(e.styles.arrow=Object.assign({},e.styles.arrow,en(Object.assign({},u,{offsets:e.modifiersData.arrow,position:"absolute",adaptive:!1,roundOffsets:l})))),e.attributes.popper=Object.assign({},e.attributes.popper,{"data-popper-placement":e.placement})}const Ke={name:"computeStyles",enabled:!0,phase:"beforeWrite",fn:Ga,data:{}};var se={passive:!0};function qa(a){var e=a.state,r=a.instance,n=a.options,t=n.scroll,i=t===void 0?!0:t,o=n.resize,s=o===void 0?!0:o,l=Y(e.elements.popper),u=[].concat(e.scrollParents.reference,e.scrollParents.popper);return i&&u.forEach(function(c){c.addEventListener("scroll",r.update,se)}),s&&l.addEventListener("resize",r.update,se),function(){i&&u.forEach(function(c){c.removeEventListener("scroll",r.update,se)}),s&&l.removeEventListener("resize",r.update,se)}}const Ye={name:"eventListeners",enabled:!0,phase:"write",fn:function(){},effect:qa,data:{}};var Qa={left:"right",right:"left",bottom:"top",top:"bottom"};function le(a){return a.replace(/left|right|bottom|top/g,function(e){return Qa[e]})}var Xa={start:"end",end:"start"};function rn(a){return a.replace(/start|end/g,function(e){return Xa[e]})}function Ue(a){var e=Y(a),r=e.pageXOffset,n=e.pageYOffset;return{scrollLeft:r,scrollTop:n}}function ze(a){return Lt(at(a)).left+Ue(a).scrollLeft}function Za(a){var e=Y(a),r=at(a),n=e.visualViewport,t=r.clientWidth,i=r.clientHeight,o=0,s=0;return n&&(t=n.width,i=n.height,/^((?!chrome|android).)*safari/i.test(navigator.userAgent)||(o=n.offsetLeft,s=n.offsetTop)),{width:t,height:i,x:o+ze(a),y:s}}function Ja(a){var e,r=at(a),n=Ue(a),t=(e=a.ownerDocument)==null?void 0:e.body,i=ut(r.scrollWidth,r.clientWidth,t?t.scrollWidth:0,t?t.clientWidth:0),o=ut(r.scrollHeight,r.clientHeight,t?t.scrollHeight:0,t?t.clientHeight:0),s=-n.scrollLeft+ze(a),l=-n.scrollTop;return J(t||r).direction==="rtl"&&(s+=ut(r.clientWidth,t?t.clientWidth:0)-i),{width:i,height:o,x:s,y:l}}function Ge(a){var e=J(a),r=e.overflow,n=e.overflowX,t=e.overflowY;return/auto|scroll|overlay|hidden/.test(r+t+n)}function nn(a){return["html","body","#document"].indexOf(q(a))>=0?a.ownerDocument.body:F(a)&&Ge(a)?a:nn(oe(a))}function Ft(a,e){var r;e===void 0&&(e=[]);var n=nn(a),t=n===((r=a.ownerDocument)==null?void 0:r.body),i=Y(n),o=t?[i].concat(i.visualViewport||[],Ge(n)?n:[]):n,s=e.concat(o);return t?s:s.concat(Ft(oe(o)))}function qe(a){return Object.assign({},a,{left:a.x,top:a.y,right:a.x+a.width,bottom:a.y+a.height})}function to(a){var e=Lt(a);return e.top=e.top+a.clientTop,e.left=e.left+a.clientLeft,e.bottom=e.top+a.clientHeight,e.right=e.left+a.clientWidth,e.width=a.clientWidth,e.height=a.clientHeight,e.x=e.left,e.y=e.top,e}function an(a,e){return e===Ve?qe(Za(a)):Nt(e)?to(e):qe(Ja(at(a)))}function eo(a){var e=Ft(oe(a)),r=["absolute","fixed"].indexOf(J(a).position)>=0,n=r&&F(a)?Wt(a):a;return Nt(n)?e.filter(function(t){return Nt(t)&&qr(t,n)&&q(t)!=="body"}):[]}function ro(a,e,r){var n=e==="clippingParents"?eo(a):[].concat(e),t=[].concat(n,[r]),i=t[0],o=t.reduce(function(s,l){var u=an(a,l);return s.top=ut(u.top,s.top),s.right=ae(u.right,s.right),s.bottom=ae(u.bottom,s.bottom),s.left=ut(u.left,s.left),s},an(a,i));return o.width=o.right-o.left,o.height=o.bottom-o.top,o.x=o.left,o.y=o.top,o}function on(a){var e=a.reference,r=a.element,n=a.placement,t=n?Q(n):null,i=n?It(n):null,o=e.x+e.width/2-r.width/2,s=e.y+e.height/2-r.height/2,l;switch(t){case P:l={x:o,y:e.y-r.height};break;case x:l={x:o,y:e.y+e.height};break;case $:l={x:e.x+e.width,y:s};break;case M:l={x:e.x-r.width,y:s};break;default:l={x:e.x,y:e.y}}var u=t?Be(t):null;if(u!=null){var c=u==="y"?"height":"width";switch(i){case lt:l[u]=l[u]-(e[c]/2-r[c]/2);break;case Ct:l[u]=l[u]+(e[c]/2-r[c]/2);break}}return l}function kt(a,e){e===void 0&&(e={});var r=e,n=r.placement,t=n===void 0?a.placement:n,i=r.boundary,o=i===void 0?xr:i,s=r.rootBoundary,l=s===void 0?Ve:s,u=r.elementContext,c=u===void 0?St:u,d=r.altBoundary,v=d===void 0?!1:d,h=r.padding,_=h===void 0?0:h,g=Zr(typeof _!="number"?_:Jr(_,Ot)),m=c===St?$r:St,b=a.rects.popper,A=a.elements[v?m:c],T=ro(Nt(A)?A:A.contextElement||at(a.elements.popper),o,l),E=Lt(a.elements.reference),w=on({reference:E,element:b,strategy:"absolute",placement:t}),O=qe(Object.assign({},b,w)),C=c===St?O:E,N={top:T.top-C.top+g.top,bottom:C.bottom-T.bottom+g.bottom,left:T.left-C.left+g.left,right:C.right-T.right+g.right},k=a.modifiersData.offset;if(c===St&&k){var L=k[t];Object.keys(N).forEach(function(I){var U=[$,x].indexOf(I)>=0?1:-1,V=[P,x].indexOf(I)>=0?"y":"x";N[I]+=L[V]*U})}return N}function no(a,e){e===void 0&&(e={});var r=e,n=r.placement,t=r.boundary,i=r.rootBoundary,o=r.padding,s=r.flipVariations,l=r.allowedAutoPlacements,u=l===void 0?We:l,c=It(n),d=c?s?He:He.filter(function(_){return It(_)===c}):Ot,v=d.filter(function(_){return u.indexOf(_)>=0});v.length===0&&(v=d);var h=v.reduce(function(_,g){return _[g]=kt(a,{placement:g,boundary:t,rootBoundary:i,padding:o})[Q(g)],_},{});return Object.keys(h).sort(function(_,g){return h[_]-h[g]})}function io(a){if(Q(a)===ie)return[];var e=le(a);return[rn(a),e,rn(e)]}function ao(a){var e=a.state,r=a.options,n=a.name;if(!e.modifiersData[n]._skip){for(var t=r.mainAxis,i=t===void 0?!0:t,o=r.altAxis,s=o===void 0?!0:o,l=r.fallbackPlacements,u=r.padding,c=r.boundary,d=r.rootBoundary,v=r.altBoundary,h=r.flipVariations,_=h===void 0?!0:h,g=r.allowedAutoPlacements,m=e.options.placement,b=Q(m),A=b===m,T=l||(A||!_?[le(m)]:io(m)),E=[m].concat(T).reduce(function(Rt,st){return Rt.concat(Q(st)===ie?no(e,{placement:st,boundary:c,rootBoundary:d,padding:u,flipVariations:_,allowedAutoPlacements:g}):st)},[]),w=e.rects.reference,O=e.rects.popper,C=new Map,N=!0,k=E[0],L=0;L=0,zt=Ut?"width":"height",H=kt(e,{placement:I,boundary:c,rootBoundary:d,altBoundary:v,padding:u}),z=Ut?V?$:M:V?x:P;w[zt]>O[zt]&&(z=le(z));var ge=le(z),_t=[];if(i&&_t.push(H[U]<=0),s&&_t.push(H[z]<=0,H[ge]<=0),_t.every(function(Rt){return Rt})){k=I,N=!1;break}C.set(I,_t)}if(N)for(var me=_?3:1,fr=function(st){var qt=E.find(function(be){var gt=C.get(be);if(gt)return gt.slice(0,st).every(function(dr){return dr})});if(qt)return k=qt,"break"},Gt=me;Gt>0;Gt--){var Ee=fr(Gt);if(Ee==="break")break}e.placement!==k&&(e.modifiersData[n]._skip=!0,e.placement=k,e.reset=!0)}}const sn={name:"flip",enabled:!0,phase:"main",fn:ao,requiresIfExists:["offset"],data:{_skip:!1}};function ln(a,e,r){return r===void 0&&(r={x:0,y:0}),{top:a.top-e.height-r.y,right:a.right-e.width+r.x,bottom:a.bottom-e.height+r.y,left:a.left-e.width-r.x}}function un(a){return[P,$,x,M].some(function(e){return a[e]>=0})}function oo(a){var e=a.state,r=a.name,n=e.rects.reference,t=e.rects.popper,i=e.modifiersData.preventOverflow,o=kt(e,{elementContext:"reference"}),s=kt(e,{altBoundary:!0}),l=ln(o,n),u=ln(s,t,i),c=un(l),d=un(u);e.modifiersData[r]={referenceClippingOffsets:l,popperEscapeOffsets:u,isReferenceHidden:c,hasPopperEscaped:d},e.attributes.popper=Object.assign({},e.attributes.popper,{"data-popper-reference-hidden":c,"data-popper-escaped":d})}const cn={name:"hide",enabled:!0,phase:"main",requiresIfExists:["preventOverflow"],fn:oo};function so(a,e,r){var n=Q(a),t=[M,P].indexOf(n)>=0?-1:1,i=typeof r=="function"?r(Object.assign({},e,{placement:a})):r,o=i[0],s=i[1];return o=o||0,s=(s||0)*t,[M,$].indexOf(n)>=0?{x:s,y:o}:{x:o,y:s}}function lo(a){var e=a.state,r=a.options,n=a.name,t=r.offset,i=t===void 0?[0,0]:t,o=We.reduce(function(c,d){return c[d]=so(d,e.rects,i),c},{}),s=o[e.placement],l=s.x,u=s.y;e.modifiersData.popperOffsets!=null&&(e.modifiersData.popperOffsets.x+=l,e.modifiersData.popperOffsets.y+=u),e.modifiersData[n]=o}const fn={name:"offset",enabled:!0,phase:"main",requires:["popperOffsets"],fn:lo};function uo(a){var e=a.state,r=a.name;e.modifiersData[r]=on({reference:e.rects.reference,element:e.rects.popper,strategy:"absolute",placement:e.placement})}const Qe={name:"popperOffsets",enabled:!0,phase:"read",fn:uo,data:{}};function co(a){return a==="x"?"y":"x"}function fo(a){var e=a.state,r=a.options,n=a.name,t=r.mainAxis,i=t===void 0?!0:t,o=r.altAxis,s=o===void 0?!1:o,l=r.boundary,u=r.rootBoundary,c=r.altBoundary,d=r.padding,v=r.tether,h=v===void 0?!0:v,_=r.tetherOffset,g=_===void 0?0:_,m=kt(e,{boundary:l,rootBoundary:u,padding:d,altBoundary:c}),b=Q(e.placement),A=It(e.placement),T=!A,E=Be(b),w=co(E),O=e.modifiersData.popperOffsets,C=e.rects.reference,N=e.rects.popper,k=typeof g=="function"?g(Object.assign({},e.rects,{placement:e.placement})):g,L=typeof k=="number"?{mainAxis:k,altAxis:k}:Object.assign({mainAxis:0,altAxis:0},k),I=e.modifiersData.offset?e.modifiersData.offset[e.placement]:null,U={x:0,y:0};if(O){if(i){var V,Ut=E==="y"?P:M,zt=E==="y"?x:$,H=E==="y"?"height":"width",z=O[E],ge=z+m[Ut],_t=z-m[zt],me=h?-N[H]/2:0,fr=A===lt?C[H]:N[H],Gt=A===lt?-N[H]:-C[H],Ee=e.elements.arrow,Rt=h&&Ee?Fe(Ee):{width:0,height:0},st=e.modifiersData["arrow#persistent"]?e.modifiersData["arrow#persistent"].padding:Xr(),qt=st[Ut],be=st[zt],gt=jt(0,C[H],Rt[H]),dr=T?C[H]/2-me-gt-qt-L.mainAxis:fr-gt-qt-L.mainAxis,hu=T?-C[H]/2+me+gt+be+L.mainAxis:Gt+gt+be+L.mainAxis,vr=e.elements.arrow&&Wt(e.elements.arrow),pu=vr?E==="y"?vr.clientTop||0:vr.clientLeft||0:0,ti=(V=I?.[E])!=null?V:0,_u=z+dr-ti-pu,gu=z+hu-ti,ei=jt(h?ae(ge,_u):ge,z,h?ut(_t,gu):_t);O[E]=ei,U[E]=ei-z}if(s){var ri,mu=E==="x"?P:M,Eu=E==="x"?x:$,mt=O[w],ye=w==="y"?"height":"width",ni=mt+m[mu],ii=mt-m[Eu],hr=[P,M].indexOf(b)!==-1,ai=(ri=I?.[w])!=null?ri:0,oi=hr?ni:mt-C[ye]-N[ye]-ai+L.altAxis,si=hr?mt+C[ye]+N[ye]-ai-L.altAxis:ii,li=h&&hr?Fa(oi,mt,si):jt(h?oi:ni,mt,h?si:ii);O[w]=li,U[w]=li-mt}e.modifiersData[n]=U}}const dn={name:"preventOverflow",enabled:!0,phase:"main",fn:fo,requiresIfExists:["offset"]};function vo(a){return{scrollLeft:a.scrollLeft,scrollTop:a.scrollTop}}function ho(a){return a===Y(a)||!F(a)?Ue(a):vo(a)}function po(a){var e=a.getBoundingClientRect(),r=Dt(e.width)/a.offsetWidth||1,n=Dt(e.height)/a.offsetHeight||1;return r!==1||n!==1}function _o(a,e,r){r===void 0&&(r=!1);var n=F(e),t=F(e)&&po(e),i=at(e),o=Lt(a,t),s={scrollLeft:0,scrollTop:0},l={x:0,y:0};return(n||!n&&!r)&&((q(e)!=="body"||Ge(i))&&(s=ho(e)),F(e)?(l=Lt(e,!0),l.x+=e.clientLeft,l.y+=e.clientTop):i&&(l.x=ze(i))),{x:o.left+s.scrollLeft-l.x,y:o.top+s.scrollTop-l.y,width:o.width,height:o.height}}function go(a){var e=new Map,r=new Set,n=[];a.forEach(function(i){e.set(i.name,i)});function t(i){r.add(i.name);var o=[].concat(i.requires||[],i.requiresIfExists||[]);o.forEach(function(s){if(!r.has(s)){var l=e.get(s);l&&t(l)}}),n.push(i)}return a.forEach(function(i){r.has(i.name)||t(i)}),n}function mo(a){var e=go(a);return zr.reduce(function(r,n){return r.concat(e.filter(function(t){return t.phase===n}))},[])}function Eo(a){var e;return function(){return e||(e=new Promise(function(r){Promise.resolve().then(function(){e=void 0,r(a())})})),e}}function bo(a){var e=a.reduce(function(r,n){var t=r[n.name];return r[n.name]=t?Object.assign({},t,n,{options:Object.assign({},t.options,n.options),data:Object.assign({},t.data,n.data)}):n,r},{});return Object.keys(e).map(function(r){return e[r]})}var vn={placement:"bottom",modifiers:[],strategy:"absolute"};function hn(){for(var a=arguments.length,e=new Array(a),r=0;r"u")throw new TypeError("Bootstrap's dropdowns require Popper (https://popper.js.org/docs/v2/)");var t=this._element;this._config.reference==="parent"?t=this._parent:X(this._config.reference)?t=rt(this._config.reference):typeof this._config.reference=="object"&&(t=this._config.reference);var i=this._getPopperConfig();this._popper=Xe(t,this._menu,i)},r._isShown=function(){return this._menu.classList.contains(Pt)},r._getPlacement=function(){var t=this._parent;if(t.classList.contains(Ro))return zo;if(t.classList.contains(xo))return Go;if(t.classList.contains($o))return qo;if(t.classList.contains(Vo))return Qo;var i=getComputedStyle(this._menu).getPropertyValue("--bs-position").trim()==="end";return t.classList.contains(Mo)?i?Ko:Bo:i?Uo:Yo},r._detectNavbar=function(){return this._element.closest(Wo)!==null},r._getOffset=function(){var t=this,i=this._config.offset;return typeof i=="string"?i.split(",").map(function(o){return Number.parseInt(o,10)}):typeof i=="function"?function(o){return i(o,t._element)}:i},r._getPopperConfig=function(){var t={placement:this._getPlacement(),modifiers:[{name:"preventOverflow",options:{boundary:this._config.boundary}},{name:"offset",options:{offset:this._getOffset()}}]};return(this._inNavbar||this._config.display==="static")&&(Z.setDataAttribute(this._menu,"popper","static"),t.modifiers=[{name:"applyStyles",enabled:!1}]),G({},t,R(this._config.popperConfig,[void 0,t]))},r._selectMenuItem=function(t){var i=t.key,o=t.target,s=p.find(Fo,this._menu).filter(function(l){return bt(l)});s.length&&Ce(s,o,i===mn,!s.includes(o)).focus()},e.jQueryInterface=function(t){return this.each(function(){var i=e.getOrCreateInstance(this,t);if(typeof t=="string"){if(typeof i[t]>"u")throw new TypeError('No method named "'+t+'"');i[t]()}})},e.clearMenus=function(t){if(!(t.button===No||t.type==="keyup"&&t.key!==gn))for(var i=p.find(Ho),o=y(i),s;!(s=o()).done;){var l=s.value,u=e.getInstance(l);if(!(!u||u._config.autoClose===!1)){var c=t.composedPath(),d=c.includes(u._menu);if(!(c.includes(u._element)||u._config.autoClose==="inside"&&!d||u._config.autoClose==="outside"&&d)&&!(u._menu.contains(t.target)&&(t.type==="keyup"&&t.key===gn||/input|select|option|textarea|form/i.test(t.target.tagName)))){var v={relatedTarget:u._element};t.type==="click"&&(v.clickEvent=t),u._completeHide(v)}}}},e.dataApiKeydownHandler=function(t){var i=/input|textarea/i.test(t.target.tagName),o=t.key===Co,s=[So,mn].includes(t.key);if(!(!s&&!o)&&!(i&&!o)){t.preventDefault();var l=this.matches(ft)?this:p.prev(this,ft)[0]||p.next(this,ft)[0]||p.findOne(ft,t.delegateTarget.parentNode),u=e.getOrCreateInstance(l);if(s){t.stopPropagation(),u.show(),u._selectMenuItem(t);return}u._isShown()&&(t.stopPropagation(),u.hide(),l.focus())}},S(e,null,[{key:"Default",get:function(){return Xo}},{key:"DefaultType",get:function(){return Zo}},{key:"NAME",get:function(){return _n}}]),e}(K);f.on(document,bn,ft,dt.dataApiKeydownHandler),f.on(document,bn,ce,dt.dataApiKeydownHandler),f.on(document,En,dt.clearMenus),f.on(document,Po,dt.clearMenus),f.on(document,En,ft,function(a){a.preventDefault(),dt.getOrCreateInstance(this).toggle()}),j(dt);var yn="backdrop",Jo="fade",An="show",Tn="mousedown.bs."+yn,ts={className:"modal-backdrop",clickCallback:null,isAnimated:!1,isVisible:!0,rootElement:"body"},es={className:"string",clickCallback:"(function|null)",isAnimated:"boolean",isVisible:"boolean",rootElement:"(element|string)"},wn=function(a){D(e,a);function e(n){var t;return t=a.call(this)||this,t._config=t._getConfig(n),t._isAppended=!1,t._element=null,t}var r=e.prototype;return r.show=function(t){if(!this._config.isVisible){R(t);return}this._append();var i=this._getElement();this._config.isAnimated&&xt(i),i.classList.add(An),this._emulateAnimation(function(){R(t)})},r.hide=function(t){var i=this;if(!this._config.isVisible){R(t);return}this._getElement().classList.remove(An),this._emulateAnimation(function(){i.dispose(),R(t)})},r.dispose=function(){this._isAppended&&(f.off(this._element,Tn),this._element.remove(),this._isAppended=!1)},r._getElement=function(){if(!this._element){var t=document.createElement("div");t.className=this._config.className,this._config.isAnimated&&t.classList.add(Jo),this._element=t}return this._element},r._configAfterMerge=function(t){return t.rootElement=rt(t.rootElement),t},r._append=function(){var t=this;if(!this._isAppended){var i=this._getElement();this._config.rootElement.append(i),f.on(i,Tn,function(){R(t._config.clickCallback)}),this._isAppended=!0}},r._emulateAnimation=function(t){Er(t,this._getElement(),this._config.isAnimated)},S(e,null,[{key:"Default",get:function(){return ts}},{key:"DefaultType",get:function(){return es}},{key:"NAME",get:function(){return yn}}]),e}($t),rs="focustrap",ns="bs.focustrap",fe="."+ns,is="focusin"+fe,as="keydown.tab"+fe,os="Tab",ss="forward",On="backward",ls={autofocus:!0,trapElement:null},us={autofocus:"boolean",trapElement:"element"},Cn=function(a){D(e,a);function e(n){var t;return t=a.call(this)||this,t._config=t._getConfig(n),t._isActive=!1,t._lastTabNavDirection=null,t}var r=e.prototype;return r.activate=function(){var t=this;this._isActive||(this._config.autofocus&&this._config.trapElement.focus(),f.off(document,fe),f.on(document,is,function(i){return t._handleFocusin(i)}),f.on(document,as,function(i){return t._handleKeydown(i)}),this._isActive=!0)},r.deactivate=function(){this._isActive&&(this._isActive=!1,f.off(document,fe))},r._handleFocusin=function(t){var i=this._config.trapElement;if(!(t.target===document||t.target===i||i.contains(t.target))){var o=p.focusableChildren(i);o.length===0?i.focus():this._lastTabNavDirection===On?o[o.length-1].focus():o[0].focus()}},r._handleKeydown=function(t){t.key===os&&(this._lastTabNavDirection=t.shiftKey?On:ss)},S(e,null,[{key:"Default",get:function(){return ls}},{key:"DefaultType",get:function(){return us}},{key:"NAME",get:function(){return rs}}]),e}($t),Sn=".fixed-top, .fixed-bottom, .is-fixed, .sticky-top",Nn=".sticky-top",de="padding-right",Dn="margin-right",Je=function(){function a(){this._element=document.body}var e=a.prototype;return e.getWidth=function(){var n=document.documentElement.clientWidth;return Math.abs(window.innerWidth-n)},e.hide=function(){var n=this.getWidth();this._disableOverFlow(),this._setElementAttributes(this._element,de,function(t){return t+n}),this._setElementAttributes(Sn,de,function(t){return t+n}),this._setElementAttributes(Nn,Dn,function(t){return t-n})},e.reset=function(){this._resetElementAttributes(this._element,"overflow"),this._resetElementAttributes(this._element,de),this._resetElementAttributes(Sn,de),this._resetElementAttributes(Nn,Dn)},e.isOverflowing=function(){return this.getWidth()>0},e._disableOverFlow=function(){this._saveInitialAttribute(this._element,"overflow"),this._element.style.overflow="hidden"},e._setElementAttributes=function(n,t,i){var o=this,s=this.getWidth(),l=function(c){if(!(c!==o._element&&window.innerWidth>c.clientWidth+s)){o._saveInitialAttribute(c,t);var d=window.getComputedStyle(c).getPropertyValue(t);c.style.setProperty(t,i(Number.parseFloat(d))+"px")}};this._applyManipulationCallback(n,l)},e._saveInitialAttribute=function(n,t){var i=n.style.getPropertyValue(t);i&&Z.setDataAttribute(n,t,i)},e._resetElementAttributes=function(n,t){var i=function(s){var l=Z.getDataAttribute(s,t);if(l===null){s.style.removeProperty(t);return}Z.removeDataAttribute(s,t),s.style.setProperty(t,l)};this._applyManipulationCallback(n,i)},e._applyManipulationCallback=function(n,t){if(X(n)){t(n);return}for(var i=y(p.find(n,this._element)),o;!(o=i()).done;){var s=o.value;t(s)}},a}(),cs="modal",fs="bs.modal",B="."+fs,ds=".data-api",vs="Escape",hs="hide"+B,ps="hidePrevented"+B,Ln="hidden"+B,In="show"+B,_s="shown"+B,gs="resize"+B,ms="click.dismiss"+B,Es="mousedown.dismiss"+B,bs="keydown.dismiss"+B,ys="click"+B+ds,kn="modal-open",As="fade",Pn="show",tr="modal-static",Ts=".modal.show",ws=".modal-dialog",Os=".modal-body",Cs='[data-bs-toggle="modal"]',Ss={backdrop:!0,focus:!0,keyboard:!0},Ns={backdrop:"(boolean|string)",focus:"boolean",keyboard:"boolean"},Bt=function(a){D(e,a);function e(n,t){var i;return i=a.call(this,n,t)||this,i._dialog=p.findOne(ws,i._element),i._backdrop=i._initializeBackDrop(),i._focustrap=i._initializeFocusTrap(),i._isShown=!1,i._isTransitioning=!1,i._scrollBar=new Je,i._addEventListeners(),i}var r=e.prototype;return r.toggle=function(t){return this._isShown?this.hide():this.show(t)},r.show=function(t){var i=this;if(!(this._isShown||this._isTransitioning)){var o=f.trigger(this._element,In,{relatedTarget:t});o.defaultPrevented||(this._isShown=!0,this._isTransitioning=!0,this._scrollBar.hide(),document.body.classList.add(kn),this._adjustDialog(),this._backdrop.show(function(){return i._showElement(t)}))}},r.hide=function(){var t=this;if(!(!this._isShown||this._isTransitioning)){var i=f.trigger(this._element,hs);i.defaultPrevented||(this._isShown=!1,this._isTransitioning=!0,this._focustrap.deactivate(),this._element.classList.remove(Pn),this._queueCallback(function(){return t._hideModal()},this._element,this._isAnimated()))}},r.dispose=function(){f.off(window,B),f.off(this._dialog,B),this._backdrop.dispose(),this._focustrap.deactivate(),a.prototype.dispose.call(this)},r.handleUpdate=function(){this._adjustDialog()},r._initializeBackDrop=function(){return new wn({isVisible:!!this._config.backdrop,isAnimated:this._isAnimated()})},r._initializeFocusTrap=function(){return new Cn({trapElement:this._element})},r._showElement=function(t){var i=this;document.body.contains(this._element)||document.body.append(this._element),this._element.style.display="block",this._element.removeAttribute("aria-hidden"),this._element.setAttribute("aria-modal",!0),this._element.setAttribute("role","dialog"),this._element.scrollTop=0;var o=p.findOne(Os,this._dialog);o&&(o.scrollTop=0),xt(this._element),this._element.classList.add(Pn);var s=function(){i._config.focus&&i._focustrap.activate(),i._isTransitioning=!1,f.trigger(i._element,_s,{relatedTarget:t})};this._queueCallback(s,this._dialog,this._isAnimated())},r._addEventListeners=function(){var t=this;f.on(this._element,bs,function(i){if(i.key===vs){if(t._config.keyboard){t.hide();return}t._triggerBackdropTransition()}}),f.on(window,gs,function(){t._isShown&&!t._isTransitioning&&t._adjustDialog()}),f.on(this._element,Es,function(i){f.one(t._element,ms,function(o){if(!(t._element!==i.target||t._element!==o.target)){if(t._config.backdrop==="static"){t._triggerBackdropTransition();return}t._config.backdrop&&t.hide()}})})},r._hideModal=function(){var t=this;this._element.style.display="none",this._element.setAttribute("aria-hidden",!0),this._element.removeAttribute("aria-modal"),this._element.removeAttribute("role"),this._isTransitioning=!1,this._backdrop.hide(function(){document.body.classList.remove(kn),t._resetAdjustments(),t._scrollBar.reset(),f.trigger(t._element,Ln)})},r._isAnimated=function(){return this._element.classList.contains(As)},r._triggerBackdropTransition=function(){var t=this,i=f.trigger(this._element,ps);if(!i.defaultPrevented){var o=this._element.scrollHeight>document.documentElement.clientHeight,s=this._element.style.overflowY;s==="hidden"||this._element.classList.contains(tr)||(o||(this._element.style.overflowY="hidden"),this._element.classList.add(tr),this._queueCallback(function(){t._element.classList.remove(tr),t._queueCallback(function(){t._element.style.overflowY=s},t._dialog)},this._dialog),this._element.focus())}},r._adjustDialog=function(){var t=this._element.scrollHeight>document.documentElement.clientHeight,i=this._scrollBar.getWidth(),o=i>0;if(o&&!t){var s=W()?"paddingLeft":"paddingRight";this._element.style[s]=i+"px"}if(!o&&t){var l=W()?"paddingRight":"paddingLeft";this._element.style[l]=i+"px"}},r._resetAdjustments=function(){this._element.style.paddingLeft="",this._element.style.paddingRight=""},e.jQueryInterface=function(t,i){return this.each(function(){var o=e.getOrCreateInstance(this,t);if(typeof t=="string"){if(typeof o[t]>"u")throw new TypeError('No method named "'+t+'"');o[t](i)}})},S(e,null,[{key:"Default",get:function(){return Ss}},{key:"DefaultType",get:function(){return Ns}},{key:"NAME",get:function(){return cs}}]),e}(K);f.on(document,ys,Cs,function(a){var e=this,r=p.getElementFromSelector(this);["A","AREA"].includes(this.tagName)&&a.preventDefault(),f.one(r,In,function(i){i.defaultPrevented||f.one(r,Ln,function(){bt(e)&&e.focus()})});var n=p.findOne(Ts);n&&Bt.getInstance(n).hide();var t=Bt.getOrCreateInstance(r);t.toggle(this)}),Zt(Bt),j(Bt);var Ds="offcanvas",Ls="bs.offcanvas",tt="."+Ls,Mn=".data-api",Is="load"+tt+Mn,ks="Escape",Rn="show",xn="showing",$n="hiding",Ps="offcanvas-backdrop",Vn=".offcanvas.show",Ms="show"+tt,Rs="shown"+tt,xs="hide"+tt,Hn="hidePrevented"+tt,Wn="hidden"+tt,$s="resize"+tt,Vs="click"+tt+Mn,Hs="keydown.dismiss"+tt,Ws='[data-bs-toggle="offcanvas"]',js={backdrop:!0,keyboard:!0,scroll:!1},Fs={backdrop:"(boolean|string)",keyboard:"boolean",scroll:"boolean"},vt=function(a){D(e,a);function e(n,t){var i;return i=a.call(this,n,t)||this,i._isShown=!1,i._backdrop=i._initializeBackDrop(),i._focustrap=i._initializeFocusTrap(),i._addEventListeners(),i}var r=e.prototype;return r.toggle=function(t){return this._isShown?this.hide():this.show(t)},r.show=function(t){var i=this;if(!this._isShown){var o=f.trigger(this._element,Ms,{relatedTarget:t});if(!o.defaultPrevented){this._isShown=!0,this._backdrop.show(),this._config.scroll||new Je().hide(),this._element.setAttribute("aria-modal",!0),this._element.setAttribute("role","dialog"),this._element.classList.add(xn);var s=function(){(!i._config.scroll||i._config.backdrop)&&i._focustrap.activate(),i._element.classList.add(Rn),i._element.classList.remove(xn),f.trigger(i._element,Rs,{relatedTarget:t})};this._queueCallback(s,this._element,!0)}}},r.hide=function(){var t=this;if(this._isShown){var i=f.trigger(this._element,xs);if(!i.defaultPrevented){this._focustrap.deactivate(),this._element.blur(),this._isShown=!1,this._element.classList.add($n),this._backdrop.hide();var o=function(){t._element.classList.remove(Rn,$n),t._element.removeAttribute("aria-modal"),t._element.removeAttribute("role"),t._config.scroll||new Je().reset(),f.trigger(t._element,Wn)};this._queueCallback(o,this._element,!0)}}},r.dispose=function(){this._backdrop.dispose(),this._focustrap.deactivate(),a.prototype.dispose.call(this)},r._initializeBackDrop=function(){var t=this,i=function(){if(t._config.backdrop==="static"){f.trigger(t._element,Hn);return}t.hide()},o=!!this._config.backdrop;return new wn({className:Ps,isVisible:o,isAnimated:!0,rootElement:this._element.parentNode,clickCallback:o?i:null})},r._initializeFocusTrap=function(){return new Cn({trapElement:this._element})},r._addEventListeners=function(){var t=this;f.on(this._element,Hs,function(i){if(i.key===ks){if(t._config.keyboard){t.hide();return}f.trigger(t._element,Hn)}})},e.jQueryInterface=function(t){return this.each(function(){var i=e.getOrCreateInstance(this,t);if(typeof t=="string"){if(i[t]===void 0||t.startsWith("_")||t==="constructor")throw new TypeError('No method named "'+t+'"');i[t](this)}})},S(e,null,[{key:"Default",get:function(){return js}},{key:"DefaultType",get:function(){return Fs}},{key:"NAME",get:function(){return Ds}}]),e}(K);f.on(document,Vs,Ws,function(a){var e=this,r=p.getElementFromSelector(this);if(["A","AREA"].includes(this.tagName)&&a.preventDefault(),!nt(this)){f.one(r,Wn,function(){bt(e)&&e.focus()});var n=p.findOne(Vn);n&&n!==r&&vt.getInstance(n).hide();var t=vt.getOrCreateInstance(r);t.toggle(this)}}),f.on(window,Is,function(){for(var a=y(p.find(Vn)),e;!(e=a()).done;){var r=e.value;vt.getOrCreateInstance(r).show()}}),f.on(window,$s,function(){for(var a=y(p.find("[aria-modal][class*=show][class*=offcanvas-]")),e;!(e=a()).done;){var r=e.value;getComputedStyle(r).position!=="fixed"&&vt.getOrCreateInstance(r).hide()}}),Zt(vt),j(vt);var Bs=/^aria-[\w-]*$/i,jn={"*":["class","dir","id","lang","role",Bs],a:["target","href","title","rel"],area:[],b:[],br:[],col:[],code:[],dd:[],div:[],dl:[],dt:[],em:[],hr:[],h1:[],h2:[],h3:[],h4:[],h5:[],h6:[],i:[],img:["src","srcset","alt","title","width","height"],li:[],ol:[],p:[],pre:[],s:[],small:[],span:[],sub:[],sup:[],strong:[],u:[],ul:[]},Ks=new Set(["background","cite","href","itemtype","longdesc","poster","src","xlink:href"]),Ys=/^(?!javascript:)(?:[a-z0-9+.-]+:|[^&:/?#]*(?:[/?#]|$))/i,Us=function(e,r){var n=e.nodeName.toLowerCase();return r.includes(n)?Ks.has(n)?!!Ys.test(e.nodeValue):!0:r.filter(function(t){return t instanceof RegExp}).some(function(t){return t.test(n)})};function zs(a,e,r){var n;if(!a.length)return a;if(r&&typeof r=="function")return r(a);for(var t=new window.DOMParser,i=t.parseFromString(a,"text/html"),o=(n=[]).concat.apply(n,i.body.querySelectorAll("*")),s=y(o),l;!(l=s()).done;){var u,c=l.value,d=c.nodeName.toLowerCase();if(!Object.keys(e).includes(d)){c.remove();continue}for(var v=(u=[]).concat.apply(u,c.attributes),h=[].concat(e["*"]||[],e[d]||[]),_=y(v),g;!(g=_()).done;){var m=g.value;Us(m,h)||c.removeAttribute(m.nodeName)}}return i.body.innerHTML}var Gs="TemplateFactory",qs={allowList:jn,content:{},extraClass:"",html:!1,sanitize:!0,sanitizeFn:null,template:"
"},Qs={allowList:"object",content:"object",extraClass:"(string|function)",html:"boolean",sanitize:"boolean",sanitizeFn:"(null|function)",template:"string"},Xs={entry:"(string|element|function|null)",selector:"(string|element)"},Zs=function(a){D(e,a);function e(n){var t;return t=a.call(this)||this,t._config=t._getConfig(n),t}var r=e.prototype;return r.getContent=function(){var t=this;return Object.values(this._config.content).map(function(i){return t._resolvePossibleFunction(i)}).filter(Boolean)},r.hasContent=function(){return this.getContent().length>0},r.changeContent=function(t){return this._checkContent(t),this._config.content=G({},this._config.content,t),this},r.toHtml=function(){var t=document.createElement("div");t.innerHTML=this._maybeSanitize(this._config.template);for(var i=0,o=Object.entries(this._config.content);i
',title:"",trigger:"hover focus"},gl={allowList:"object",animation:"boolean",boundary:"(string|element)",container:"(string|element|boolean)",customClass:"(string|function)",delay:"(number|object)",fallbackPlacements:"array",html:"boolean",offset:"(array|string|function)",placement:"(string|function)",popperConfig:"(null|object|function)",sanitize:"boolean",sanitizeFn:"(null|function)",selector:"(string|boolean)",template:"string",title:"(string|element|function)",trigger:"string"},Yt=function(a){D(e,a);function e(n,t){var i;if(typeof pn>"u")throw new TypeError("Bootstrap's tooltips require Popper (https://popper.js.org/docs/v2/)");return i=a.call(this,n,t)||this,i._isEnabled=!0,i._timeout=0,i._isHovered=null,i._activeTrigger={},i._popper=null,i._templateFactory=null,i._newContent=null,i.tip=null,i._setListeners(),i._config.selector||i._fixTitle(),i}var r=e.prototype;return r.enable=function(){this._isEnabled=!0},r.disable=function(){this._isEnabled=!1},r.toggleEnabled=function(){this._isEnabled=!this._isEnabled},r.toggle=function(){if(this._isEnabled){if(this._isShown()){this._leave();return}this._enter()}},r.dispose=function(){clearTimeout(this._timeout),f.off(this._element.closest(Fn),Bn,this._hideModalHandler),this._element.getAttribute("data-bs-original-title")&&this._element.setAttribute("title",this._element.getAttribute("data-bs-original-title")),this._disposePopper(),a.prototype.dispose.call(this)},r.show=function(){var t=this;if(this._element.style.display==="none")throw new Error("Please use show on visible elements");if(this._isWithContent()&&this._isEnabled){var i=f.trigger(this._element,this.constructor.eventName(sl)),o=gi(this._element),s=(o||this._element.ownerDocument.documentElement).contains(this._element);if(!(i.defaultPrevented||!s)){this._disposePopper();var l=this._getTipElement();this._element.setAttribute("aria-describedby",l.getAttribute("id"));var u=this._config.container;if(this._element.ownerDocument.documentElement.contains(this.tip)||(u.append(l),f.trigger(this._element,this.constructor.eventName(ul))),this._popper=this._createPopper(l),l.classList.add(ve),"ontouchstart"in document.documentElement)for(var c=y((v=[]).concat.apply(v,document.body.children)),d;!(d=c()).done;){var v,h=d.value;f.on(h,"mouseover",Xt)}var _=function(){f.trigger(t._element,t.constructor.eventName(ll)),t._isHovered===!1&&t._leave(),t._isHovered=!1};this._queueCallback(_,this.tip,this._isAnimated())}}},r.hide=function(){var t=this;if(this._isShown()){var i=f.trigger(this._element,this.constructor.eventName(al));if(!i.defaultPrevented){var o=this._getTipElement();if(o.classList.remove(ve),"ontouchstart"in document.documentElement)for(var s=y((u=[]).concat.apply(u,document.body.children)),l;!(l=s()).done;){var u,c=l.value;f.off(c,"mouseover",Xt)}this._activeTrigger[nl]=!1,this._activeTrigger[rr]=!1,this._activeTrigger[Kt]=!1,this._isHovered=null;var d=function(){t._isWithActiveTrigger()||(t._isHovered||t._disposePopper(),t._element.removeAttribute("aria-describedby"),f.trigger(t._element,t.constructor.eventName(ol)))};this._queueCallback(d,this.tip,this._isAnimated())}}},r.update=function(){this._popper&&this._popper.update()},r._isWithContent=function(){return!!this._getTitle()},r._getTipElement=function(){return this.tip||(this.tip=this._createTipElement(this._newContent||this._getContentForTemplate())),this.tip},r._createTipElement=function(t){var i=this._getTemplateFactory(t).toHtml();if(!i)return null;i.classList.remove(er,ve),i.classList.add("bs-"+this.constructor.NAME+"-auto");var o=pi(this.constructor.NAME).toString();return i.setAttribute("id",o),this._isAnimated()&&i.classList.add(er),i},r.setContent=function(t){this._newContent=t,this._isShown()&&(this._disposePopper(),this.show())},r._getTemplateFactory=function(t){return this._templateFactory?this._templateFactory.changeContent(t):this._templateFactory=new Zs(G({},this._config,{content:t,extraClass:this._resolvePossibleFunction(this._config.customClass)})),this._templateFactory},r._getContentForTemplate=function(){var t;return t={},t[rl]=this._getTitle(),t},r._getTitle=function(){return this._resolvePossibleFunction(this._config.title)||this._element.getAttribute("data-bs-original-title")},r._initializeOnDelegatedTarget=function(t){return this.constructor.getOrCreateInstance(t.delegateTarget,this._getDelegateConfig())},r._isAnimated=function(){return this._config.animation||this.tip&&this.tip.classList.contains(er)},r._isShown=function(){return this.tip&&this.tip.classList.contains(ve)},r._createPopper=function(t){var i=R(this._config.placement,[this,t,this._element]),o=pl[i.toUpperCase()];return Xe(this._element,t,this._getPopperConfig(o))},r._getOffset=function(){var t=this,i=this._config.offset;return typeof i=="string"?i.split(",").map(function(o){return Number.parseInt(o,10)}):typeof i=="function"?function(o){return i(o,t._element)}:i},r._resolvePossibleFunction=function(t){return R(t,[this._element,this._element])},r._getPopperConfig=function(t){var i=this,o={placement:t,modifiers:[{name:"flip",options:{fallbackPlacements:this._config.fallbackPlacements}},{name:"offset",options:{offset:this._getOffset()}},{name:"preventOverflow",options:{boundary:this._config.boundary}},{name:"arrow",options:{element:"."+this.constructor.NAME+"-arrow"}},{name:"preSetPlacement",enabled:!0,phase:"beforeMain",fn:function(l){i._getTipElement().setAttribute("data-popper-placement",l.state.placement)}}]};return G({},o,R(this._config.popperConfig,[void 0,o]))},r._setListeners=function(){for(var t=this,i=this._config.trigger.split(" "),o=y(i),s;!(s=o()).done;){var l=s.value;if(l==="click")f.on(this._element,this.constructor.eventName(cl),this._config.selector,function(d){var v=t._initializeOnDelegatedTarget(d);v.toggle()});else if(l!==il){var u=l===Kt?this.constructor.eventName(vl):this.constructor.eventName(fl),c=l===Kt?this.constructor.eventName(hl):this.constructor.eventName(dl);f.on(this._element,u,this._config.selector,function(d){var v=t._initializeOnDelegatedTarget(d);v._activeTrigger[d.type==="focusin"?rr:Kt]=!0,v._enter()}),f.on(this._element,c,this._config.selector,function(d){var v=t._initializeOnDelegatedTarget(d);v._activeTrigger[d.type==="focusout"?rr:Kt]=v._element.contains(d.relatedTarget),v._leave()})}}this._hideModalHandler=function(){t._element&&t.hide()},f.on(this._element.closest(Fn),Bn,this._hideModalHandler)},r._fixTitle=function(){var t=this._element.getAttribute("title");t&&(!this._element.getAttribute("aria-label")&&!this._element.textContent.trim()&&this._element.setAttribute("aria-label",t),this._element.setAttribute("data-bs-original-title",t),this._element.removeAttribute("title"))},r._enter=function(){var t=this;if(this._isShown()||this._isHovered){this._isHovered=!0;return}this._isHovered=!0,this._setTimeout(function(){t._isHovered&&t.show()},this._config.delay.show)},r._leave=function(){var t=this;this._isWithActiveTrigger()||(this._isHovered=!1,this._setTimeout(function(){t._isHovered||t.hide()},this._config.delay.hide))},r._setTimeout=function(t,i){clearTimeout(this._timeout),this._timeout=setTimeout(t,i)},r._isWithActiveTrigger=function(){return Object.values(this._activeTrigger).includes(!0)},r._getConfig=function(t){for(var i=Z.getDataAttributes(this._element),o=0,s=Object.keys(i);o"u")throw new TypeError('No method named "'+t+'"');i[t]()}})},S(e,null,[{key:"Default",get:function(){return _l}},{key:"DefaultType",get:function(){return gl}},{key:"NAME",get:function(){return Js}}]),e}(K);j(Yt);var ml="popover",El=".popover-header",bl=".popover-body",yl=G({},Yt.Default,{content:"",offset:[0,8],placement:"right",template:'',trigger:"click"}),Al=G({},Yt.DefaultType,{content:"(null|string|element|function)"}),Kn=function(a){D(e,a);function e(){return a.apply(this,arguments)||this}var r=e.prototype;return r._isWithContent=function(){return this._getTitle()||this._getContent()},r._getContentForTemplate=function(){var t;return t={},t[El]=this._getTitle(),t[bl]=this._getContent(),t},r._getContent=function(){return this._resolvePossibleFunction(this._config.content)},e.jQueryInterface=function(t){return this.each(function(){var i=e.getOrCreateInstance(this,t);if(typeof t=="string"){if(typeof i[t]>"u")throw new TypeError('No method named "'+t+'"');i[t]()}})},S(e,null,[{key:"Default",get:function(){return yl}},{key:"DefaultType",get:function(){return Al}},{key:"NAME",get:function(){return ml}}]),e}(Yt);j(Kn);var Tl="scrollspy",wl="bs.scrollspy",nr="."+wl,Ol=".data-api",Cl="activate"+nr,Yn="click"+nr,Sl="load"+nr+Ol,Nl="dropdown-item",Mt="active",Dl='[data-bs-spy="scroll"]',ir="[href]",Ll=".nav, .list-group",Un=".nav-link",Il=".nav-item",kl=".list-group-item",Pl=Un+", "+Il+" > "+Un+", "+kl,Ml=".dropdown",Rl=".dropdown-toggle",xl={offset:null,rootMargin:"0px 0px -25%",smoothScroll:!1,target:null,threshold:[.1,.5,1]},$l={offset:"(number|null)",rootMargin:"string",smoothScroll:"boolean",target:"element",threshold:"array"},ar=function(a){D(e,a);function e(n,t){var i;return i=a.call(this,n,t)||this,i._targetLinks=new Map,i._observableSections=new Map,i._rootElement=getComputedStyle(i._element).overflowY==="visible"?null:i._element,i._activeTarget=null,i._observer=null,i._previousScrollData={visibleEntryTop:0,parentScrollTop:0},i.refresh(),i}var r=e.prototype;return r.refresh=function(){this._initializeTargetsAndObservables(),this._maybeEnableSmoothScroll(),this._observer?this._observer.disconnect():this._observer=this._getNewObserver();for(var t=y(this._observableSections.values()),i;!(i=t()).done;){var o=i.value;this._observer.observe(o)}},r.dispose=function(){this._observer.disconnect(),a.prototype.dispose.call(this)},r._configAfterMerge=function(t){return t.target=rt(t.target)||document.body,t.rootMargin=t.offset?t.offset+"px 0px -30%":t.rootMargin,typeof t.threshold=="string"&&(t.threshold=t.threshold.split(",").map(function(i){return Number.parseFloat(i)})),t},r._maybeEnableSmoothScroll=function(){var t=this;this._config.smoothScroll&&(f.off(this._config.target,Yn),f.on(this._config.target,Yn,ir,function(i){var o=t._observableSections.get(i.target.hash);if(o){i.preventDefault();var s=t._rootElement||window,l=o.offsetTop-t._element.offsetTop;if(s.scrollTo){s.scrollTo({top:l,behavior:"smooth"});return}s.scrollTop=l}}))},r._getNewObserver=function(){var t=this,i={root:this._rootElement,threshold:this._config.threshold,rootMargin:this._config.rootMargin};return new IntersectionObserver(function(o){return t._observerCallback(o)},i)},r._observerCallback=function(t){var i=this,o=function(g){return i._targetLinks.get("#"+g.target.id)},s=function(g){i._previousScrollData.visibleEntryTop=g.target.offsetTop,i._process(o(g))},l=(this._rootElement||document.documentElement).scrollTop,u=l>=this._previousScrollData.parentScrollTop;this._previousScrollData.parentScrollTop=l;for(var c=y(t),d;!(d=c()).done;){var v=d.value;if(!v.isIntersecting){this._activeTarget=null,this._clearActiveClass(o(v));continue}var h=v.target.offsetTop>=this._previousScrollData.visibleEntryTop;if(u&&h){if(s(v),!l)return;continue}!u&&!h&&s(v)}},r._initializeTargetsAndObservables=function(){this._targetLinks=new Map,this._observableSections=new Map;for(var t=p.find(ir,this._config.target),i=y(t),o;!(o=i()).done;){var s=o.value;if(!(!s.hash||nt(s))){var l=p.findOne(decodeURI(s.hash),this._element);bt(l)&&(this._targetLinks.set(decodeURI(s.hash),s),this._observableSections.set(s.hash,l))}}},r._process=function(t){this._activeTarget!==t&&(this._clearActiveClass(this._config.target),this._activeTarget=t,t.classList.add(Mt),this._activateParents(t),f.trigger(this._element,Cl,{relatedTarget:t}))},r._activateParents=function(t){if(t.classList.contains(Nl)){p.findOne(Rl,t.closest(Ml)).classList.add(Mt);return}for(var i=y(p.parents(t,Ll)),o;!(o=i()).done;)for(var s=o.value,l=y(p.prev(s,Pl)),u;!(u=l()).done;){var c=u.value;c.classList.add(Mt)}},r._clearActiveClass=function(t){t.classList.remove(Mt);for(var i=p.find(ir+"."+Mt,t),o=y(i),s;!(s=o()).done;){var l=s.value;l.classList.remove(Mt)}},e.jQueryInterface=function(t){return this.each(function(){var i=e.getOrCreateInstance(this,t);if(typeof t=="string"){if(i[t]===void 0||t.startsWith("_")||t==="constructor")throw new TypeError('No method named "'+t+'"');i[t]()}})},S(e,null,[{key:"Default",get:function(){return xl}},{key:"DefaultType",get:function(){return $l}},{key:"NAME",get:function(){return Tl}}]),e}(K);f.on(window,Sl,function(){for(var a=y(p.find(Dl)),e;!(e=a()).done;){var r=e.value;ar.getOrCreateInstance(r)}}),j(ar);var Vl="tab",Hl="bs.tab",ht="."+Hl,Wl="hide"+ht,jl="hidden"+ht,Fl="show"+ht,Bl="shown"+ht,Kl="click"+ht,Yl="keydown"+ht,Ul="load"+ht,zl="ArrowLeft",zn="ArrowRight",Gl="ArrowUp",Gn="ArrowDown",or="Home",qn="End",pt="active",Qn="fade",sr="show",ql="dropdown",Xn=".dropdown-toggle",Ql=".dropdown-menu",lr=":not("+Xn+")",Xl='.list-group, .nav, [role="tablist"]',Zl=".nav-item, .list-group-item",Jl=".nav-link"+lr+", .list-group-item"+lr+', [role="tab"]'+lr,Zn='[data-bs-toggle="tab"], [data-bs-toggle="pill"], [data-bs-toggle="list"]',ur=Jl+", "+Zn,tu="."+pt+'[data-bs-toggle="tab"], .'+pt+'[data-bs-toggle="pill"], .'+pt+'[data-bs-toggle="list"]',he=function(a){D(e,a);function e(n){var t;return t=a.call(this,n)||this,t._parent=t._element.closest(Xl),t._parent?(t._setInitialAttributes(t._parent,t._getChildren()),f.on(t._element,Yl,function(i){return t._keydown(i)}),t):Qt(t)}var r=e.prototype;return r.show=function(){var t=this._element;if(!this._elemIsActive(t)){var i=this._getActiveElem(),o=i?f.trigger(i,Wl,{relatedTarget:t}):null,s=f.trigger(t,Fl,{relatedTarget:i});s.defaultPrevented||o&&o.defaultPrevented||(this._deactivate(i,t),this._activate(t,i))}},r._activate=function(t,i){var o=this;if(t){t.classList.add(pt),this._activate(p.getElementFromSelector(t));var s=function(){if(t.getAttribute("role")!=="tab"){t.classList.add(sr);return}t.removeAttribute("tabindex"),t.setAttribute("aria-selected",!0),o._toggleDropDown(t,!0),f.trigger(t,Bl,{relatedTarget:i})};this._queueCallback(s,t,t.classList.contains(Qn))}},r._deactivate=function(t,i){var o=this;if(t){t.classList.remove(pt),t.blur(),this._deactivate(p.getElementFromSelector(t));var s=function(){if(t.getAttribute("role")!=="tab"){t.classList.remove(sr);return}t.setAttribute("aria-selected",!1),t.setAttribute("tabindex","-1"),o._toggleDropDown(t,!1),f.trigger(t,jl,{relatedTarget:i})};this._queueCallback(s,t,t.classList.contains(Qn))}},r._keydown=function(t){if([zl,zn,Gl,Gn,or,qn].includes(t.key)){t.stopPropagation(),t.preventDefault();var i=this._getChildren().filter(function(l){return!nt(l)}),o;if([or,qn].includes(t.key))o=i[t.key===or?0:i.length-1];else{var s=[zn,Gn].includes(t.key);o=Ce(i,t.target,s,!0)}o&&(o.focus({preventScroll:!0}),e.getOrCreateInstance(o).show())}},r._getChildren=function(){return p.find(ur,this._parent)},r._getActiveElem=function(){var t=this;return this._getChildren().find(function(i){return t._elemIsActive(i)})||null},r._setInitialAttributes=function(t,i){this._setAttributeIfNotExists(t,"role","tablist");for(var o=y(i),s;!(s=o()).done;){var l=s.value;this._setInitialAttributesOnChild(l)}},r._setInitialAttributesOnChild=function(t){t=this._getInnerElement(t);var i=this._elemIsActive(t),o=this._getOuterElement(t);t.setAttribute("aria-selected",i),o!==t&&this._setAttributeIfNotExists(o,"role","presentation"),i||t.setAttribute("tabindex","-1"),this._setAttributeIfNotExists(t,"role","tab"),this._setInitialAttributesOnTargetPanel(t)},r._setInitialAttributesOnTargetPanel=function(t){var i=p.getElementFromSelector(t);i&&(this._setAttributeIfNotExists(i,"role","tabpanel"),t.id&&this._setAttributeIfNotExists(i,"aria-labelledby",""+t.id))},r._toggleDropDown=function(t,i){var o=this._getOuterElement(t);if(o.classList.contains(ql)){var s=function(u,c){var d=p.findOne(u,o);d&&d.classList.toggle(c,i)};s(Xn,pt),s(Ql,sr),o.setAttribute("aria-expanded",i)}},r._setAttributeIfNotExists=function(t,i,o){t.hasAttribute(i)||t.setAttribute(i,o)},r._elemIsActive=function(t){return t.classList.contains(pt)},r._getInnerElement=function(t){return t.matches(ur)?t:p.findOne(ur,t)},r._getOuterElement=function(t){return t.closest(Zl)||t},e.jQueryInterface=function(t){return this.each(function(){var i=e.getOrCreateInstance(this);if(typeof t=="string"){if(i[t]===void 0||t.startsWith("_")||t==="constructor")throw new TypeError('No method named "'+t+'"');i[t]()}})},S(e,null,[{key:"NAME",get:function(){return Vl}}]),e}(K);f.on(document,Kl,Zn,function(a){["A","AREA"].includes(this.tagName)&&a.preventDefault(),!nt(this)&&he.getOrCreateInstance(this).show()}),f.on(window,Ul,function(){for(var a=y(p.find(tu)),e;!(e=a()).done;){var r=e.value;he.getOrCreateInstance(r)}}),j(he);var eu="toast",ru="bs.toast",ot="."+ru,nu="mouseover"+ot,iu="mouseout"+ot,au="focusin"+ot,ou="focusout"+ot,su="hide"+ot,lu="hidden"+ot,uu="show"+ot,cu="shown"+ot,fu="fade",Jn="hide",pe="show",_e="showing",du={animation:"boolean",autohide:"boolean",delay:"number"},vu={animation:!0,autohide:!0,delay:5e3},cr=function(a){D(e,a);function e(n,t){var i;return i=a.call(this,n,t)||this,i._timeout=null,i._hasMouseInteraction=!1,i._hasKeyboardInteraction=!1,i._setListeners(),i}var r=e.prototype;return r.show=function(){var t=this,i=f.trigger(this._element,uu);if(!i.defaultPrevented){this._clearTimeout(),this._config.animation&&this._element.classList.add(fu);var o=function(){t._element.classList.remove(_e),f.trigger(t._element,cu),t._maybeScheduleHide()};this._element.classList.remove(Jn),xt(this._element),this._element.classList.add(pe,_e),this._queueCallback(o,this._element,this._config.animation)}},r.hide=function(){var t=this;if(this.isShown()){var i=f.trigger(this._element,su);if(!i.defaultPrevented){var o=function(){t._element.classList.add(Jn),t._element.classList.remove(_e,pe),f.trigger(t._element,lu)};this._element.classList.add(_e),this._queueCallback(o,this._element,this._config.animation)}}},r.dispose=function(){this._clearTimeout(),this.isShown()&&this._element.classList.remove(pe),a.prototype.dispose.call(this)},r.isShown=function(){return this._element.classList.contains(pe)},r._maybeScheduleHide=function(){var t=this;this._config.autohide&&(this._hasMouseInteraction||this._hasKeyboardInteraction||(this._timeout=setTimeout(function(){t.hide()},this._config.delay)))},r._onInteraction=function(t,i){switch(t.type){case"mouseover":case"mouseout":{this._hasMouseInteraction=i;break}case"focusin":case"focusout":{this._hasKeyboardInteraction=i;break}}if(i){this._clearTimeout();return}var o=t.relatedTarget;this._element===o||this._element.contains(o)||this._maybeScheduleHide()},r._setListeners=function(){var t=this;f.on(this._element,nu,function(i){return t._onInteraction(i,!0)}),f.on(this._element,iu,function(i){return t._onInteraction(i,!1)}),f.on(this._element,au,function(i){return t._onInteraction(i,!0)}),f.on(this._element,ou,function(i){return t._onInteraction(i,!1)})},r._clearTimeout=function(){clearTimeout(this._timeout),this._timeout=null},e.jQueryInterface=function(t){return this.each(function(){var i=e.getOrCreateInstance(this,t);if(typeof t=="string"){if(typeof i[t]>"u")throw new TypeError('No method named "'+t+'"');i[t](this)}})},S(e,null,[{key:"Default",get:function(){return vu}},{key:"DefaultType",get:function(){return du}},{key:"NAME",get:function(){return eu}}]),e}(K);return Zt(cr),j(cr),{Alert:ke,Button:Pe,Carousel:re,Collapse:$e,Dropdown:dt,Modal:Bt,Offcanvas:vt,Popover:Kn,ScrollSpy:ar,Tab:he,Toast:cr,Tooltip:Yt}}); + diff --git a/web/static/bootstrap.min.css b/web/static/bootstrap.min.css new file mode 100644 index 0000000..beec370 --- /dev/null +++ b/web/static/bootstrap.min.css @@ -0,0 +1,6 @@ +/*! + * Bootstrap v5.3.5 (https://getbootstrap.com/) + * Copyright 2011-2025 The Bootstrap Authors + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE) + */:root,[data-bs-theme=light]{--bs-blue: #0d6efd;--bs-indigo: #6610f2;--bs-purple: #6f42c1;--bs-pink: #d63384;--bs-red: #dc3545;--bs-orange: #fd7e14;--bs-yellow: #ffc107;--bs-green: #198754;--bs-teal: #20c997;--bs-cyan: #0dcaf0;--bs-black: #000;--bs-white: #fff;--bs-gray: #6c757d;--bs-gray-dark: #343a40;--bs-gray-100: #f8f9fa;--bs-gray-200: #e9ecef;--bs-gray-300: #dee2e6;--bs-gray-400: #ced4da;--bs-gray-500: #adb5bd;--bs-gray-600: #6c757d;--bs-gray-700: #495057;--bs-gray-800: #343a40;--bs-gray-900: #212529;--bs-primary: #0d6efd;--bs-secondary: #6c757d;--bs-success: #198754;--bs-info: #0dcaf0;--bs-warning: #ffc107;--bs-danger: #dc3545;--bs-light: #f8f9fa;--bs-dark: #212529;--bs-primary-rgb: 13, 110, 253;--bs-secondary-rgb: 108, 117, 125;--bs-success-rgb: 25, 135, 84;--bs-info-rgb: 13, 202, 240;--bs-warning-rgb: 255, 193, 7;--bs-danger-rgb: 220, 53, 69;--bs-light-rgb: 248, 249, 250;--bs-dark-rgb: 33, 37, 41;--bs-primary-text-emphasis: #052c65;--bs-secondary-text-emphasis: #2b2f32;--bs-success-text-emphasis: #0a3622;--bs-info-text-emphasis: #055160;--bs-warning-text-emphasis: #664d03;--bs-danger-text-emphasis: #58151c;--bs-light-text-emphasis: #495057;--bs-dark-text-emphasis: #495057;--bs-primary-bg-subtle: #cfe2ff;--bs-secondary-bg-subtle: #e2e3e5;--bs-success-bg-subtle: #d1e7dd;--bs-info-bg-subtle: #cff4fc;--bs-warning-bg-subtle: #fff3cd;--bs-danger-bg-subtle: #f8d7da;--bs-light-bg-subtle: #fcfcfd;--bs-dark-bg-subtle: #ced4da;--bs-primary-border-subtle: #9ec5fe;--bs-secondary-border-subtle: #c4c8cb;--bs-success-border-subtle: #a3cfbb;--bs-info-border-subtle: #9eeaf9;--bs-warning-border-subtle: #ffe69c;--bs-danger-border-subtle: #f1aeb5;--bs-light-border-subtle: #e9ecef;--bs-dark-border-subtle: #adb5bd;--bs-white-rgb: 255, 255, 255;--bs-black-rgb: 0, 0, 0;--bs-font-sans-serif: system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", "Noto Sans", "Liberation Sans", Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";--bs-font-monospace: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;--bs-gradient: linear-gradient(180deg, rgba(255, 255, 255, .15), rgba(255, 255, 255, 0));--bs-body-font-family: var(--bs-font-sans-serif);--bs-body-font-size: 1rem;--bs-body-font-weight: 400;--bs-body-line-height: 1.5;--bs-body-color: #212529;--bs-body-color-rgb: 33, 37, 41;--bs-body-bg: #fff;--bs-body-bg-rgb: 255, 255, 255;--bs-emphasis-color: #000;--bs-emphasis-color-rgb: 0, 0, 0;--bs-secondary-color: rgba(33, 37, 41, .75);--bs-secondary-color-rgb: 33, 37, 41;--bs-secondary-bg: #e9ecef;--bs-secondary-bg-rgb: 233, 236, 239;--bs-tertiary-color: rgba(33, 37, 41, .5);--bs-tertiary-color-rgb: 33, 37, 41;--bs-tertiary-bg: #f8f9fa;--bs-tertiary-bg-rgb: 248, 249, 250;--bs-heading-color: inherit;--bs-link-color: #0d6efd;--bs-link-color-rgb: 13, 110, 253;--bs-link-decoration: underline;--bs-link-hover-color: #0a58ca;--bs-link-hover-color-rgb: 10, 88, 202;--bs-code-color: #d63384;--bs-highlight-color: #212529;--bs-highlight-bg: #fff3cd;--bs-border-width: 1px;--bs-border-style: solid;--bs-border-color: #dee2e6;--bs-border-color-translucent: rgba(0, 0, 0, .175);--bs-border-radius: .375rem;--bs-border-radius-sm: .25rem;--bs-border-radius-lg: .5rem;--bs-border-radius-xl: 1rem;--bs-border-radius-xxl: 2rem;--bs-border-radius-2xl: var(--bs-border-radius-xxl);--bs-border-radius-pill: 50rem;--bs-box-shadow: 0 .5rem 1rem rgba(0, 0, 0, .15);--bs-box-shadow-sm: 0 .125rem .25rem rgba(0, 0, 0, .075);--bs-box-shadow-lg: 0 1rem 3rem rgba(0, 0, 0, .175);--bs-box-shadow-inset: inset 0 1px 2px rgba(0, 0, 0, .075);--bs-focus-ring-width: .25rem;--bs-focus-ring-opacity: .25;--bs-focus-ring-color: rgba(13, 110, 253, .25);--bs-form-valid-color: #198754;--bs-form-valid-border-color: #198754;--bs-form-invalid-color: #dc3545;--bs-form-invalid-border-color: #dc3545}[data-bs-theme=dark]{color-scheme:dark;--bs-body-color: #dee2e6;--bs-body-color-rgb: 222, 226, 230;--bs-body-bg: #212529;--bs-body-bg-rgb: 33, 37, 41;--bs-emphasis-color: #fff;--bs-emphasis-color-rgb: 255, 255, 255;--bs-secondary-color: rgba(222, 226, 230, .75);--bs-secondary-color-rgb: 222, 226, 230;--bs-secondary-bg: #343a40;--bs-secondary-bg-rgb: 52, 58, 64;--bs-tertiary-color: rgba(222, 226, 230, .5);--bs-tertiary-color-rgb: 222, 226, 230;--bs-tertiary-bg: #2b3035;--bs-tertiary-bg-rgb: 43, 48, 53;--bs-primary-text-emphasis: #6ea8fe;--bs-secondary-text-emphasis: #a7acb1;--bs-success-text-emphasis: #75b798;--bs-info-text-emphasis: #6edff6;--bs-warning-text-emphasis: #ffda6a;--bs-danger-text-emphasis: #ea868f;--bs-light-text-emphasis: #f8f9fa;--bs-dark-text-emphasis: #dee2e6;--bs-primary-bg-subtle: #031633;--bs-secondary-bg-subtle: #161719;--bs-success-bg-subtle: #051b11;--bs-info-bg-subtle: #032830;--bs-warning-bg-subtle: #332701;--bs-danger-bg-subtle: #2c0b0e;--bs-light-bg-subtle: #343a40;--bs-dark-bg-subtle: #1a1d20;--bs-primary-border-subtle: #084298;--bs-secondary-border-subtle: #41464b;--bs-success-border-subtle: #0f5132;--bs-info-border-subtle: #087990;--bs-warning-border-subtle: #997404;--bs-danger-border-subtle: #842029;--bs-light-border-subtle: #495057;--bs-dark-border-subtle: #343a40;--bs-heading-color: inherit;--bs-link-color: #6ea8fe;--bs-link-hover-color: #8bb9fe;--bs-link-color-rgb: 110, 168, 254;--bs-link-hover-color-rgb: 139, 185, 254;--bs-code-color: #e685b5;--bs-highlight-color: #dee2e6;--bs-highlight-bg: #664d03;--bs-border-color: #495057;--bs-border-color-translucent: rgba(255, 255, 255, .15);--bs-form-valid-color: #75b798;--bs-form-valid-border-color: #75b798;--bs-form-invalid-color: #ea868f;--bs-form-invalid-border-color: #ea868f}*,*:before,*:after{box-sizing:border-box}@media (prefers-reduced-motion: no-preference){:root{scroll-behavior:smooth}}body{margin:0;font-family:var(--bs-body-font-family);font-size:var(--bs-body-font-size);font-weight:var(--bs-body-font-weight);line-height:var(--bs-body-line-height);color:var(--bs-body-color);text-align:var(--bs-body-text-align);background-color:var(--bs-body-bg);-webkit-text-size-adjust:100%;-webkit-tap-highlight-color:rgba(0,0,0,0)}hr{margin:1rem 0;color:inherit;border:0;border-top:var(--bs-border-width) solid;opacity:.25}h6,.h6,h5,.h5,h4,.h4,h3,.h3,h2,.h2,h1,.h1{margin-top:0;margin-bottom:.5rem;font-weight:500;line-height:1.2;color:var(--bs-heading-color)}h1,.h1{font-size:calc(1.375rem + 1.5vw)}@media (min-width: 1200px){h1,.h1{font-size:2.5rem}}h2,.h2{font-size:calc(1.325rem + .9vw)}@media (min-width: 1200px){h2,.h2{font-size:2rem}}h3,.h3{font-size:calc(1.3rem + .6vw)}@media (min-width: 1200px){h3,.h3{font-size:1.75rem}}h4,.h4{font-size:calc(1.275rem + .3vw)}@media (min-width: 1200px){h4,.h4{font-size:1.5rem}}h5,.h5{font-size:1.25rem}h6,.h6{font-size:1rem}p{margin-top:0;margin-bottom:1rem}abbr[title]{text-decoration:underline dotted;cursor:help;text-decoration-skip-ink:none}address{margin-bottom:1rem;font-style:normal;line-height:inherit}ol,ul{padding-left:2rem}ol,ul,dl{margin-top:0;margin-bottom:1rem}ol ol,ul ul,ol ul,ul ol{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}b,strong{font-weight:bolder}small,.small{font-size:.875em}mark,.mark{padding:.1875em;color:var(--bs-highlight-color);background-color:var(--bs-highlight-bg)}sub,sup{position:relative;font-size:.75em;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:rgba(var(--bs-link-color-rgb),var(--bs-link-opacity, 1));text-decoration:underline}a:hover{--bs-link-color-rgb: var(--bs-link-hover-color-rgb)}a:not([href]):not([class]),a:not([href]):not([class]):hover{color:inherit;text-decoration:none}pre,code,kbd,samp{font-family:var(--bs-font-monospace);font-size:1em}pre{display:block;margin-top:0;margin-bottom:1rem;overflow:auto;font-size:.875em}pre code{font-size:inherit;color:inherit;word-break:normal}code{font-size:.875em;color:var(--bs-code-color);word-wrap:break-word}a>code{color:inherit}kbd{padding:.1875rem .375rem;font-size:.875em;color:var(--bs-body-bg);background-color:var(--bs-body-color);border-radius:.25rem}kbd kbd{padding:0;font-size:1em}figure{margin:0 0 1rem}img,svg{vertical-align:middle}table{caption-side:bottom;border-collapse:collapse}caption{padding-top:.5rem;padding-bottom:.5rem;color:var(--bs-secondary-color);text-align:left}th{text-align:inherit;text-align:-webkit-match-parent}thead,tbody,tfoot,tr,td,th{border-color:inherit;border-style:solid;border-width:0}label{display:inline-block}button{border-radius:0}button:focus:not(:focus-visible){outline:0}input,button,select,optgroup,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,select{text-transform:none}[role=button]{cursor:pointer}select{word-wrap:normal}select:disabled{opacity:1}[list]:not([type=date]):not([type=datetime-local]):not([type=month]):not([type=week]):not([type=time])::-webkit-calendar-picker-indicator{display:none!important}button,[type=button],[type=reset],[type=submit]{-webkit-appearance:button}button:not(:disabled),[type=button]:not(:disabled),[type=reset]:not(:disabled),[type=submit]:not(:disabled){cursor:pointer}::-moz-focus-inner{padding:0;border-style:none}textarea{resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{float:left;width:100%;padding:0;margin-bottom:.5rem;line-height:inherit;font-size:calc(1.275rem + .3vw)}@media (min-width: 1200px){legend{font-size:1.5rem}}legend+*{clear:left}::-webkit-datetime-edit-fields-wrapper,::-webkit-datetime-edit-text,::-webkit-datetime-edit-minute,::-webkit-datetime-edit-hour-field,::-webkit-datetime-edit-day-field,::-webkit-datetime-edit-month-field,::-webkit-datetime-edit-year-field{padding:0}::-webkit-inner-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-color-swatch-wrapper{padding:0}::file-selector-button{font:inherit;-webkit-appearance:button}output{display:inline-block}iframe{border:0}summary{display:list-item;cursor:pointer}progress{vertical-align:baseline}[hidden]{display:none!important}.lead{font-size:1.25rem;font-weight:300}.display-1{font-weight:300;line-height:1.2;font-size:calc(1.625rem + 4.5vw)}@media (min-width: 1200px){.display-1{font-size:5rem}}.display-2{font-weight:300;line-height:1.2;font-size:calc(1.575rem + 3.9vw)}@media (min-width: 1200px){.display-2{font-size:4.5rem}}.display-3{font-weight:300;line-height:1.2;font-size:calc(1.525rem + 3.3vw)}@media (min-width: 1200px){.display-3{font-size:4rem}}.display-4{font-weight:300;line-height:1.2;font-size:calc(1.475rem + 2.7vw)}@media (min-width: 1200px){.display-4{font-size:3.5rem}}.display-5{font-weight:300;line-height:1.2;font-size:calc(1.425rem + 2.1vw)}@media (min-width: 1200px){.display-5{font-size:3rem}}.display-6{font-weight:300;line-height:1.2;font-size:calc(1.375rem + 1.5vw)}@media (min-width: 1200px){.display-6{font-size:2.5rem}}.list-unstyled,.list-inline{padding-left:0;list-style:none}.list-inline-item{display:inline-block}.list-inline-item:not(:last-child){margin-right:.5rem}.initialism{font-size:.875em;text-transform:uppercase}.blockquote{margin-bottom:1rem;font-size:1.25rem}.blockquote>:last-child{margin-bottom:0}.blockquote-footer{margin-top:-1rem;margin-bottom:1rem;font-size:.875em;color:#6c757d}.blockquote-footer:before{content:"\2014\a0"}.img-fluid{max-width:100%;height:auto}.img-thumbnail{padding:.25rem;background-color:var(--bs-body-bg);border:var(--bs-border-width) solid var(--bs-border-color);border-radius:var(--bs-border-radius);max-width:100%;height:auto}.figure{display:inline-block}.figure-img{margin-bottom:.5rem;line-height:1}.figure-caption{font-size:.875em;color:var(--bs-secondary-color)}.container,.container-fluid,.container-xxl,.container-xl,.container-lg,.container-md,.container-sm{--bs-gutter-x: 1.5rem;--bs-gutter-y: 0;width:100%;padding-right:calc(var(--bs-gutter-x) * .5);padding-left:calc(var(--bs-gutter-x) * .5);margin-right:auto;margin-left:auto}@media (min-width: 576px){.container-sm,.container{max-width:540px}}@media (min-width: 768px){.container-md,.container-sm,.container{max-width:720px}}@media (min-width: 992px){.container-lg,.container-md,.container-sm,.container{max-width:960px}}@media (min-width: 1200px){.container-xl,.container-lg,.container-md,.container-sm,.container{max-width:1140px}}@media (min-width: 1400px){.container-xxl,.container-xl,.container-lg,.container-md,.container-sm,.container{max-width:1320px}}:root{--bs-breakpoint-xs: 0;--bs-breakpoint-sm: 576px;--bs-breakpoint-md: 768px;--bs-breakpoint-lg: 992px;--bs-breakpoint-xl: 1200px;--bs-breakpoint-xxl: 1400px}.row{--bs-gutter-x: 1.5rem;--bs-gutter-y: 0;display:flex;flex-wrap:wrap;margin-top:calc(-1 * var(--bs-gutter-y));margin-right:calc(-.5 * var(--bs-gutter-x));margin-left:calc(-.5 * var(--bs-gutter-x))}.row>*{flex-shrink:0;width:100%;max-width:100%;padding-right:calc(var(--bs-gutter-x) * .5);padding-left:calc(var(--bs-gutter-x) * .5);margin-top:var(--bs-gutter-y)}.col{flex:1 0 0}.row-cols-auto>*{flex:0 0 auto;width:auto}.row-cols-1>*{flex:0 0 auto;width:100%}.row-cols-2>*{flex:0 0 auto;width:50%}.row-cols-3>*{flex:0 0 auto;width:33.33333333%}.row-cols-4>*{flex:0 0 auto;width:25%}.row-cols-5>*{flex:0 0 auto;width:20%}.row-cols-6>*{flex:0 0 auto;width:16.66666667%}.col-auto{flex:0 0 auto;width:auto}.col-1{flex:0 0 auto;width:8.33333333%}.col-2{flex:0 0 auto;width:16.66666667%}.col-3{flex:0 0 auto;width:25%}.col-4{flex:0 0 auto;width:33.33333333%}.col-5{flex:0 0 auto;width:41.66666667%}.col-6{flex:0 0 auto;width:50%}.col-7{flex:0 0 auto;width:58.33333333%}.col-8{flex:0 0 auto;width:66.66666667%}.col-9{flex:0 0 auto;width:75%}.col-10{flex:0 0 auto;width:83.33333333%}.col-11{flex:0 0 auto;width:91.66666667%}.col-12{flex:0 0 auto;width:100%}.offset-1{margin-left:8.33333333%}.offset-2{margin-left:16.66666667%}.offset-3{margin-left:25%}.offset-4{margin-left:33.33333333%}.offset-5{margin-left:41.66666667%}.offset-6{margin-left:50%}.offset-7{margin-left:58.33333333%}.offset-8{margin-left:66.66666667%}.offset-9{margin-left:75%}.offset-10{margin-left:83.33333333%}.offset-11{margin-left:91.66666667%}.g-0,.gx-0{--bs-gutter-x: 0}.g-0,.gy-0{--bs-gutter-y: 0}.g-1,.gx-1{--bs-gutter-x: .25rem}.g-1,.gy-1{--bs-gutter-y: .25rem}.g-2,.gx-2{--bs-gutter-x: .5rem}.g-2,.gy-2{--bs-gutter-y: .5rem}.g-3,.gx-3{--bs-gutter-x: 1rem}.g-3,.gy-3{--bs-gutter-y: 1rem}.g-4,.gx-4{--bs-gutter-x: 1.5rem}.g-4,.gy-4{--bs-gutter-y: 1.5rem}.g-5,.gx-5{--bs-gutter-x: 3rem}.g-5,.gy-5{--bs-gutter-y: 3rem}@media (min-width: 576px){.col-sm{flex:1 0 0}.row-cols-sm-auto>*{flex:0 0 auto;width:auto}.row-cols-sm-1>*{flex:0 0 auto;width:100%}.row-cols-sm-2>*{flex:0 0 auto;width:50%}.row-cols-sm-3>*{flex:0 0 auto;width:33.33333333%}.row-cols-sm-4>*{flex:0 0 auto;width:25%}.row-cols-sm-5>*{flex:0 0 auto;width:20%}.row-cols-sm-6>*{flex:0 0 auto;width:16.66666667%}.col-sm-auto{flex:0 0 auto;width:auto}.col-sm-1{flex:0 0 auto;width:8.33333333%}.col-sm-2{flex:0 0 auto;width:16.66666667%}.col-sm-3{flex:0 0 auto;width:25%}.col-sm-4{flex:0 0 auto;width:33.33333333%}.col-sm-5{flex:0 0 auto;width:41.66666667%}.col-sm-6{flex:0 0 auto;width:50%}.col-sm-7{flex:0 0 auto;width:58.33333333%}.col-sm-8{flex:0 0 auto;width:66.66666667%}.col-sm-9{flex:0 0 auto;width:75%}.col-sm-10{flex:0 0 auto;width:83.33333333%}.col-sm-11{flex:0 0 auto;width:91.66666667%}.col-sm-12{flex:0 0 auto;width:100%}.offset-sm-0{margin-left:0}.offset-sm-1{margin-left:8.33333333%}.offset-sm-2{margin-left:16.66666667%}.offset-sm-3{margin-left:25%}.offset-sm-4{margin-left:33.33333333%}.offset-sm-5{margin-left:41.66666667%}.offset-sm-6{margin-left:50%}.offset-sm-7{margin-left:58.33333333%}.offset-sm-8{margin-left:66.66666667%}.offset-sm-9{margin-left:75%}.offset-sm-10{margin-left:83.33333333%}.offset-sm-11{margin-left:91.66666667%}.g-sm-0,.gx-sm-0{--bs-gutter-x: 0}.g-sm-0,.gy-sm-0{--bs-gutter-y: 0}.g-sm-1,.gx-sm-1{--bs-gutter-x: .25rem}.g-sm-1,.gy-sm-1{--bs-gutter-y: .25rem}.g-sm-2,.gx-sm-2{--bs-gutter-x: .5rem}.g-sm-2,.gy-sm-2{--bs-gutter-y: .5rem}.g-sm-3,.gx-sm-3{--bs-gutter-x: 1rem}.g-sm-3,.gy-sm-3{--bs-gutter-y: 1rem}.g-sm-4,.gx-sm-4{--bs-gutter-x: 1.5rem}.g-sm-4,.gy-sm-4{--bs-gutter-y: 1.5rem}.g-sm-5,.gx-sm-5{--bs-gutter-x: 3rem}.g-sm-5,.gy-sm-5{--bs-gutter-y: 3rem}}@media (min-width: 768px){.col-md{flex:1 0 0}.row-cols-md-auto>*{flex:0 0 auto;width:auto}.row-cols-md-1>*{flex:0 0 auto;width:100%}.row-cols-md-2>*{flex:0 0 auto;width:50%}.row-cols-md-3>*{flex:0 0 auto;width:33.33333333%}.row-cols-md-4>*{flex:0 0 auto;width:25%}.row-cols-md-5>*{flex:0 0 auto;width:20%}.row-cols-md-6>*{flex:0 0 auto;width:16.66666667%}.col-md-auto{flex:0 0 auto;width:auto}.col-md-1{flex:0 0 auto;width:8.33333333%}.col-md-2{flex:0 0 auto;width:16.66666667%}.col-md-3{flex:0 0 auto;width:25%}.col-md-4{flex:0 0 auto;width:33.33333333%}.col-md-5{flex:0 0 auto;width:41.66666667%}.col-md-6{flex:0 0 auto;width:50%}.col-md-7{flex:0 0 auto;width:58.33333333%}.col-md-8{flex:0 0 auto;width:66.66666667%}.col-md-9{flex:0 0 auto;width:75%}.col-md-10{flex:0 0 auto;width:83.33333333%}.col-md-11{flex:0 0 auto;width:91.66666667%}.col-md-12{flex:0 0 auto;width:100%}.offset-md-0{margin-left:0}.offset-md-1{margin-left:8.33333333%}.offset-md-2{margin-left:16.66666667%}.offset-md-3{margin-left:25%}.offset-md-4{margin-left:33.33333333%}.offset-md-5{margin-left:41.66666667%}.offset-md-6{margin-left:50%}.offset-md-7{margin-left:58.33333333%}.offset-md-8{margin-left:66.66666667%}.offset-md-9{margin-left:75%}.offset-md-10{margin-left:83.33333333%}.offset-md-11{margin-left:91.66666667%}.g-md-0,.gx-md-0{--bs-gutter-x: 0}.g-md-0,.gy-md-0{--bs-gutter-y: 0}.g-md-1,.gx-md-1{--bs-gutter-x: .25rem}.g-md-1,.gy-md-1{--bs-gutter-y: .25rem}.g-md-2,.gx-md-2{--bs-gutter-x: .5rem}.g-md-2,.gy-md-2{--bs-gutter-y: .5rem}.g-md-3,.gx-md-3{--bs-gutter-x: 1rem}.g-md-3,.gy-md-3{--bs-gutter-y: 1rem}.g-md-4,.gx-md-4{--bs-gutter-x: 1.5rem}.g-md-4,.gy-md-4{--bs-gutter-y: 1.5rem}.g-md-5,.gx-md-5{--bs-gutter-x: 3rem}.g-md-5,.gy-md-5{--bs-gutter-y: 3rem}}@media (min-width: 992px){.col-lg{flex:1 0 0}.row-cols-lg-auto>*{flex:0 0 auto;width:auto}.row-cols-lg-1>*{flex:0 0 auto;width:100%}.row-cols-lg-2>*{flex:0 0 auto;width:50%}.row-cols-lg-3>*{flex:0 0 auto;width:33.33333333%}.row-cols-lg-4>*{flex:0 0 auto;width:25%}.row-cols-lg-5>*{flex:0 0 auto;width:20%}.row-cols-lg-6>*{flex:0 0 auto;width:16.66666667%}.col-lg-auto{flex:0 0 auto;width:auto}.col-lg-1{flex:0 0 auto;width:8.33333333%}.col-lg-2{flex:0 0 auto;width:16.66666667%}.col-lg-3{flex:0 0 auto;width:25%}.col-lg-4{flex:0 0 auto;width:33.33333333%}.col-lg-5{flex:0 0 auto;width:41.66666667%}.col-lg-6{flex:0 0 auto;width:50%}.col-lg-7{flex:0 0 auto;width:58.33333333%}.col-lg-8{flex:0 0 auto;width:66.66666667%}.col-lg-9{flex:0 0 auto;width:75%}.col-lg-10{flex:0 0 auto;width:83.33333333%}.col-lg-11{flex:0 0 auto;width:91.66666667%}.col-lg-12{flex:0 0 auto;width:100%}.offset-lg-0{margin-left:0}.offset-lg-1{margin-left:8.33333333%}.offset-lg-2{margin-left:16.66666667%}.offset-lg-3{margin-left:25%}.offset-lg-4{margin-left:33.33333333%}.offset-lg-5{margin-left:41.66666667%}.offset-lg-6{margin-left:50%}.offset-lg-7{margin-left:58.33333333%}.offset-lg-8{margin-left:66.66666667%}.offset-lg-9{margin-left:75%}.offset-lg-10{margin-left:83.33333333%}.offset-lg-11{margin-left:91.66666667%}.g-lg-0,.gx-lg-0{--bs-gutter-x: 0}.g-lg-0,.gy-lg-0{--bs-gutter-y: 0}.g-lg-1,.gx-lg-1{--bs-gutter-x: .25rem}.g-lg-1,.gy-lg-1{--bs-gutter-y: .25rem}.g-lg-2,.gx-lg-2{--bs-gutter-x: .5rem}.g-lg-2,.gy-lg-2{--bs-gutter-y: .5rem}.g-lg-3,.gx-lg-3{--bs-gutter-x: 1rem}.g-lg-3,.gy-lg-3{--bs-gutter-y: 1rem}.g-lg-4,.gx-lg-4{--bs-gutter-x: 1.5rem}.g-lg-4,.gy-lg-4{--bs-gutter-y: 1.5rem}.g-lg-5,.gx-lg-5{--bs-gutter-x: 3rem}.g-lg-5,.gy-lg-5{--bs-gutter-y: 3rem}}@media (min-width: 1200px){.col-xl{flex:1 0 0}.row-cols-xl-auto>*{flex:0 0 auto;width:auto}.row-cols-xl-1>*{flex:0 0 auto;width:100%}.row-cols-xl-2>*{flex:0 0 auto;width:50%}.row-cols-xl-3>*{flex:0 0 auto;width:33.33333333%}.row-cols-xl-4>*{flex:0 0 auto;width:25%}.row-cols-xl-5>*{flex:0 0 auto;width:20%}.row-cols-xl-6>*{flex:0 0 auto;width:16.66666667%}.col-xl-auto{flex:0 0 auto;width:auto}.col-xl-1{flex:0 0 auto;width:8.33333333%}.col-xl-2{flex:0 0 auto;width:16.66666667%}.col-xl-3{flex:0 0 auto;width:25%}.col-xl-4{flex:0 0 auto;width:33.33333333%}.col-xl-5{flex:0 0 auto;width:41.66666667%}.col-xl-6{flex:0 0 auto;width:50%}.col-xl-7{flex:0 0 auto;width:58.33333333%}.col-xl-8{flex:0 0 auto;width:66.66666667%}.col-xl-9{flex:0 0 auto;width:75%}.col-xl-10{flex:0 0 auto;width:83.33333333%}.col-xl-11{flex:0 0 auto;width:91.66666667%}.col-xl-12{flex:0 0 auto;width:100%}.offset-xl-0{margin-left:0}.offset-xl-1{margin-left:8.33333333%}.offset-xl-2{margin-left:16.66666667%}.offset-xl-3{margin-left:25%}.offset-xl-4{margin-left:33.33333333%}.offset-xl-5{margin-left:41.66666667%}.offset-xl-6{margin-left:50%}.offset-xl-7{margin-left:58.33333333%}.offset-xl-8{margin-left:66.66666667%}.offset-xl-9{margin-left:75%}.offset-xl-10{margin-left:83.33333333%}.offset-xl-11{margin-left:91.66666667%}.g-xl-0,.gx-xl-0{--bs-gutter-x: 0}.g-xl-0,.gy-xl-0{--bs-gutter-y: 0}.g-xl-1,.gx-xl-1{--bs-gutter-x: .25rem}.g-xl-1,.gy-xl-1{--bs-gutter-y: .25rem}.g-xl-2,.gx-xl-2{--bs-gutter-x: .5rem}.g-xl-2,.gy-xl-2{--bs-gutter-y: .5rem}.g-xl-3,.gx-xl-3{--bs-gutter-x: 1rem}.g-xl-3,.gy-xl-3{--bs-gutter-y: 1rem}.g-xl-4,.gx-xl-4{--bs-gutter-x: 1.5rem}.g-xl-4,.gy-xl-4{--bs-gutter-y: 1.5rem}.g-xl-5,.gx-xl-5{--bs-gutter-x: 3rem}.g-xl-5,.gy-xl-5{--bs-gutter-y: 3rem}}@media (min-width: 1400px){.col-xxl{flex:1 0 0}.row-cols-xxl-auto>*{flex:0 0 auto;width:auto}.row-cols-xxl-1>*{flex:0 0 auto;width:100%}.row-cols-xxl-2>*{flex:0 0 auto;width:50%}.row-cols-xxl-3>*{flex:0 0 auto;width:33.33333333%}.row-cols-xxl-4>*{flex:0 0 auto;width:25%}.row-cols-xxl-5>*{flex:0 0 auto;width:20%}.row-cols-xxl-6>*{flex:0 0 auto;width:16.66666667%}.col-xxl-auto{flex:0 0 auto;width:auto}.col-xxl-1{flex:0 0 auto;width:8.33333333%}.col-xxl-2{flex:0 0 auto;width:16.66666667%}.col-xxl-3{flex:0 0 auto;width:25%}.col-xxl-4{flex:0 0 auto;width:33.33333333%}.col-xxl-5{flex:0 0 auto;width:41.66666667%}.col-xxl-6{flex:0 0 auto;width:50%}.col-xxl-7{flex:0 0 auto;width:58.33333333%}.col-xxl-8{flex:0 0 auto;width:66.66666667%}.col-xxl-9{flex:0 0 auto;width:75%}.col-xxl-10{flex:0 0 auto;width:83.33333333%}.col-xxl-11{flex:0 0 auto;width:91.66666667%}.col-xxl-12{flex:0 0 auto;width:100%}.offset-xxl-0{margin-left:0}.offset-xxl-1{margin-left:8.33333333%}.offset-xxl-2{margin-left:16.66666667%}.offset-xxl-3{margin-left:25%}.offset-xxl-4{margin-left:33.33333333%}.offset-xxl-5{margin-left:41.66666667%}.offset-xxl-6{margin-left:50%}.offset-xxl-7{margin-left:58.33333333%}.offset-xxl-8{margin-left:66.66666667%}.offset-xxl-9{margin-left:75%}.offset-xxl-10{margin-left:83.33333333%}.offset-xxl-11{margin-left:91.66666667%}.g-xxl-0,.gx-xxl-0{--bs-gutter-x: 0}.g-xxl-0,.gy-xxl-0{--bs-gutter-y: 0}.g-xxl-1,.gx-xxl-1{--bs-gutter-x: .25rem}.g-xxl-1,.gy-xxl-1{--bs-gutter-y: .25rem}.g-xxl-2,.gx-xxl-2{--bs-gutter-x: .5rem}.g-xxl-2,.gy-xxl-2{--bs-gutter-y: .5rem}.g-xxl-3,.gx-xxl-3{--bs-gutter-x: 1rem}.g-xxl-3,.gy-xxl-3{--bs-gutter-y: 1rem}.g-xxl-4,.gx-xxl-4{--bs-gutter-x: 1.5rem}.g-xxl-4,.gy-xxl-4{--bs-gutter-y: 1.5rem}.g-xxl-5,.gx-xxl-5{--bs-gutter-x: 3rem}.g-xxl-5,.gy-xxl-5{--bs-gutter-y: 3rem}}.table{--bs-table-color-type: initial;--bs-table-bg-type: initial;--bs-table-color-state: initial;--bs-table-bg-state: initial;--bs-table-color: var(--bs-emphasis-color);--bs-table-bg: var(--bs-body-bg);--bs-table-border-color: var(--bs-border-color);--bs-table-accent-bg: transparent;--bs-table-striped-color: var(--bs-emphasis-color);--bs-table-striped-bg: rgba(var(--bs-emphasis-color-rgb), .05);--bs-table-active-color: var(--bs-emphasis-color);--bs-table-active-bg: rgba(var(--bs-emphasis-color-rgb), .1);--bs-table-hover-color: var(--bs-emphasis-color);--bs-table-hover-bg: rgba(var(--bs-emphasis-color-rgb), .075);width:100%;margin-bottom:1rem;vertical-align:top;border-color:var(--bs-table-border-color)}.table>:not(caption)>*>*{padding:.5rem;color:var(--bs-table-color-state, var(--bs-table-color-type, var(--bs-table-color)));background-color:var(--bs-table-bg);border-bottom-width:var(--bs-border-width);box-shadow:inset 0 0 0 9999px var(--bs-table-bg-state, var(--bs-table-bg-type, var(--bs-table-accent-bg)))}.table>tbody{vertical-align:inherit}.table>thead{vertical-align:bottom}.table-group-divider{border-top:calc(var(--bs-border-width) * 2) solid currentcolor}.caption-top{caption-side:top}.table-sm>:not(caption)>*>*{padding:.25rem}.table-bordered>:not(caption)>*{border-width:var(--bs-border-width) 0}.table-bordered>:not(caption)>*>*{border-width:0 var(--bs-border-width)}.table-borderless>:not(caption)>*>*{border-bottom-width:0}.table-borderless>:not(:first-child){border-top-width:0}.table-striped>tbody>tr:nth-of-type(odd)>*{--bs-table-color-type: var(--bs-table-striped-color);--bs-table-bg-type: var(--bs-table-striped-bg)}.table-striped-columns>:not(caption)>tr>:nth-child(2n){--bs-table-color-type: var(--bs-table-striped-color);--bs-table-bg-type: var(--bs-table-striped-bg)}.table-active{--bs-table-color-state: var(--bs-table-active-color);--bs-table-bg-state: var(--bs-table-active-bg)}.table-hover>tbody>tr:hover>*{--bs-table-color-state: var(--bs-table-hover-color);--bs-table-bg-state: var(--bs-table-hover-bg)}.table-primary{--bs-table-color: #000;--bs-table-bg: #cfe2ff;--bs-table-border-color: #a6b5cc;--bs-table-striped-bg: #c5d7f2;--bs-table-striped-color: #000;--bs-table-active-bg: #bacbe6;--bs-table-active-color: #000;--bs-table-hover-bg: #bfd1ec;--bs-table-hover-color: #000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-secondary{--bs-table-color: #000;--bs-table-bg: #e2e3e5;--bs-table-border-color: #b5b6b7;--bs-table-striped-bg: #d7d8da;--bs-table-striped-color: #000;--bs-table-active-bg: #cbccce;--bs-table-active-color: #000;--bs-table-hover-bg: #d1d2d4;--bs-table-hover-color: #000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-success{--bs-table-color: #000;--bs-table-bg: #d1e7dd;--bs-table-border-color: #a7b9b1;--bs-table-striped-bg: #c7dbd2;--bs-table-striped-color: #000;--bs-table-active-bg: #bcd0c7;--bs-table-active-color: #000;--bs-table-hover-bg: #c1d6cc;--bs-table-hover-color: #000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-info{--bs-table-color: #000;--bs-table-bg: #cff4fc;--bs-table-border-color: #a6c3ca;--bs-table-striped-bg: #c5e8ef;--bs-table-striped-color: #000;--bs-table-active-bg: #badce3;--bs-table-active-color: #000;--bs-table-hover-bg: #bfe2e9;--bs-table-hover-color: #000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-warning{--bs-table-color: #000;--bs-table-bg: #fff3cd;--bs-table-border-color: #ccc2a4;--bs-table-striped-bg: #f2e7c3;--bs-table-striped-color: #000;--bs-table-active-bg: #e6dbb9;--bs-table-active-color: #000;--bs-table-hover-bg: #ece1be;--bs-table-hover-color: #000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-danger{--bs-table-color: #000;--bs-table-bg: #f8d7da;--bs-table-border-color: #c6acae;--bs-table-striped-bg: #eccccf;--bs-table-striped-color: #000;--bs-table-active-bg: #dfc2c4;--bs-table-active-color: #000;--bs-table-hover-bg: #e5c7ca;--bs-table-hover-color: #000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-light{--bs-table-color: #000;--bs-table-bg: #f8f9fa;--bs-table-border-color: #c6c7c8;--bs-table-striped-bg: #ecedee;--bs-table-striped-color: #000;--bs-table-active-bg: #dfe0e1;--bs-table-active-color: #000;--bs-table-hover-bg: #e5e6e7;--bs-table-hover-color: #000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-dark{--bs-table-color: #fff;--bs-table-bg: #212529;--bs-table-border-color: #4d5154;--bs-table-striped-bg: #2c3034;--bs-table-striped-color: #fff;--bs-table-active-bg: #373b3e;--bs-table-active-color: #fff;--bs-table-hover-bg: #323539;--bs-table-hover-color: #fff;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-responsive{overflow-x:auto;-webkit-overflow-scrolling:touch}@media (max-width: 575.98px){.table-responsive-sm{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media (max-width: 767.98px){.table-responsive-md{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media (max-width: 991.98px){.table-responsive-lg{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media (max-width: 1199.98px){.table-responsive-xl{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media (max-width: 1399.98px){.table-responsive-xxl{overflow-x:auto;-webkit-overflow-scrolling:touch}}.form-label{margin-bottom:.5rem}.col-form-label{padding-top:calc(.375rem + var(--bs-border-width));padding-bottom:calc(.375rem + var(--bs-border-width));margin-bottom:0;font-size:inherit;line-height:1.5}.col-form-label-lg{padding-top:calc(.5rem + var(--bs-border-width));padding-bottom:calc(.5rem + var(--bs-border-width));font-size:1.25rem}.col-form-label-sm{padding-top:calc(.25rem + var(--bs-border-width));padding-bottom:calc(.25rem + var(--bs-border-width));font-size:.875rem}.form-text{margin-top:.25rem;font-size:.875em;color:var(--bs-secondary-color)}.form-control{display:block;width:100%;padding:.375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:var(--bs-body-color);appearance:none;background-color:var(--bs-body-bg);background-clip:padding-box;border:var(--bs-border-width) solid var(--bs-border-color);border-radius:var(--bs-border-radius);transition:border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion: reduce){.form-control{transition:none}}.form-control[type=file]{overflow:hidden}.form-control[type=file]:not(:disabled):not([readonly]){cursor:pointer}.form-control:focus{color:var(--bs-body-color);background-color:var(--bs-body-bg);border-color:#86b7fe;outline:0;box-shadow:0 0 0 .25rem #0d6efd40}.form-control::-webkit-date-and-time-value{min-width:85px;height:1.5em;margin:0}.form-control::-webkit-datetime-edit{display:block;padding:0}.form-control::placeholder{color:var(--bs-secondary-color);opacity:1}.form-control:disabled{background-color:var(--bs-secondary-bg);opacity:1}.form-control::file-selector-button{padding:.375rem .75rem;margin:-.375rem -.75rem;margin-inline-end:.75rem;color:var(--bs-body-color);background-color:var(--bs-tertiary-bg);pointer-events:none;border-color:inherit;border-style:solid;border-width:0;border-inline-end-width:var(--bs-border-width);border-radius:0;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion: reduce){.form-control::file-selector-button{transition:none}}.form-control:hover:not(:disabled):not([readonly])::file-selector-button{background-color:var(--bs-secondary-bg)}.form-control-plaintext{display:block;width:100%;padding:.375rem 0;margin-bottom:0;line-height:1.5;color:var(--bs-body-color);background-color:transparent;border:solid transparent;border-width:var(--bs-border-width) 0}.form-control-plaintext:focus{outline:0}.form-control-plaintext.form-control-sm,.form-control-plaintext.form-control-lg{padding-right:0;padding-left:0}.form-control-sm{min-height:calc(1.5em + .5rem + calc(var(--bs-border-width) * 2));padding:.25rem .5rem;font-size:.875rem;border-radius:var(--bs-border-radius-sm)}.form-control-sm::file-selector-button{padding:.25rem .5rem;margin:-.25rem -.5rem;margin-inline-end:.5rem}.form-control-lg{min-height:calc(1.5em + 1rem + calc(var(--bs-border-width) * 2));padding:.5rem 1rem;font-size:1.25rem;border-radius:var(--bs-border-radius-lg)}.form-control-lg::file-selector-button{padding:.5rem 1rem;margin:-.5rem -1rem;margin-inline-end:1rem}textarea.form-control{min-height:calc(1.5em + .75rem + calc(var(--bs-border-width) * 2))}textarea.form-control-sm{min-height:calc(1.5em + .5rem + calc(var(--bs-border-width) * 2))}textarea.form-control-lg{min-height:calc(1.5em + 1rem + calc(var(--bs-border-width) * 2))}.form-control-color{width:3rem;height:calc(1.5em + .75rem + calc(var(--bs-border-width) * 2));padding:.375rem}.form-control-color:not(:disabled):not([readonly]){cursor:pointer}.form-control-color::-moz-color-swatch{border:0!important;border-radius:var(--bs-border-radius)}.form-control-color::-webkit-color-swatch{border:0!important;border-radius:var(--bs-border-radius)}.form-control-color.form-control-sm{height:calc(1.5em + .5rem + calc(var(--bs-border-width) * 2))}.form-control-color.form-control-lg{height:calc(1.5em + 1rem + calc(var(--bs-border-width) * 2))}.form-select{--bs-form-select-bg-img: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3e%3cpath fill='none' stroke='%23343a40' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='m2 5 6 6 6-6'/%3e%3c/svg%3e");display:block;width:100%;padding:.375rem 2.25rem .375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:var(--bs-body-color);appearance:none;background-color:var(--bs-body-bg);background-image:var(--bs-form-select-bg-img),var(--bs-form-select-bg-icon, none);background-repeat:no-repeat;background-position:right .75rem center;background-size:16px 12px;border:var(--bs-border-width) solid var(--bs-border-color);border-radius:var(--bs-border-radius);transition:border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion: reduce){.form-select{transition:none}}.form-select:focus{border-color:#86b7fe;outline:0;box-shadow:0 0 0 .25rem #0d6efd40}.form-select[multiple],.form-select[size]:not([size="1"]){padding-right:.75rem;background-image:none}.form-select:disabled{background-color:var(--bs-secondary-bg)}.form-select:-moz-focusring{color:transparent;text-shadow:0 0 0 var(--bs-body-color)}.form-select-sm{padding-top:.25rem;padding-bottom:.25rem;padding-left:.5rem;font-size:.875rem;border-radius:var(--bs-border-radius-sm)}.form-select-lg{padding-top:.5rem;padding-bottom:.5rem;padding-left:1rem;font-size:1.25rem;border-radius:var(--bs-border-radius-lg)}[data-bs-theme=dark] .form-select{--bs-form-select-bg-img: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3e%3cpath fill='none' stroke='%23dee2e6' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='m2 5 6 6 6-6'/%3e%3c/svg%3e")}.form-check{display:block;min-height:1.5rem;padding-left:1.5em;margin-bottom:.125rem}.form-check .form-check-input{float:left;margin-left:-1.5em}.form-check-reverse{padding-right:1.5em;padding-left:0;text-align:right}.form-check-reverse .form-check-input{float:right;margin-right:-1.5em;margin-left:0}.form-check-input{--bs-form-check-bg: var(--bs-body-bg);flex-shrink:0;width:1em;height:1em;margin-top:.25em;vertical-align:top;appearance:none;background-color:var(--bs-form-check-bg);background-image:var(--bs-form-check-bg-image);background-repeat:no-repeat;background-position:center;background-size:contain;border:var(--bs-border-width) solid var(--bs-border-color);print-color-adjust:exact}.form-check-input[type=checkbox]{border-radius:.25em}.form-check-input[type=radio]{border-radius:50%}.form-check-input:active{filter:brightness(90%)}.form-check-input:focus{border-color:#86b7fe;outline:0;box-shadow:0 0 0 .25rem #0d6efd40}.form-check-input:checked{background-color:#0d6efd;border-color:#0d6efd}.form-check-input:checked[type=checkbox]{--bs-form-check-bg-image: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 20 20'%3e%3cpath fill='none' stroke='%23fff' stroke-linecap='round' stroke-linejoin='round' stroke-width='3' d='m6 10 3 3 6-6'/%3e%3c/svg%3e")}.form-check-input:checked[type=radio]{--bs-form-check-bg-image: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='2' fill='%23fff'/%3e%3c/svg%3e")}.form-check-input[type=checkbox]:indeterminate{background-color:#0d6efd;border-color:#0d6efd;--bs-form-check-bg-image: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 20 20'%3e%3cpath fill='none' stroke='%23fff' stroke-linecap='round' stroke-linejoin='round' stroke-width='3' d='M6 10h8'/%3e%3c/svg%3e")}.form-check-input:disabled{pointer-events:none;filter:none;opacity:.5}.form-check-input[disabled]~.form-check-label,.form-check-input:disabled~.form-check-label{cursor:default;opacity:.5}.form-switch{padding-left:2.5em}.form-switch .form-check-input{--bs-form-switch-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='rgba%280, 0, 0, 0.25%29'/%3e%3c/svg%3e");width:2em;margin-left:-2.5em;background-image:var(--bs-form-switch-bg);background-position:left center;border-radius:2em;transition:background-position .15s ease-in-out}@media (prefers-reduced-motion: reduce){.form-switch .form-check-input{transition:none}}.form-switch .form-check-input:focus{--bs-form-switch-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='%2386b7fe'/%3e%3c/svg%3e")}.form-switch .form-check-input:checked{background-position:right center;--bs-form-switch-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='%23fff'/%3e%3c/svg%3e")}.form-switch.form-check-reverse{padding-right:2.5em;padding-left:0}.form-switch.form-check-reverse .form-check-input{margin-right:-2.5em;margin-left:0}.form-check-inline{display:inline-block;margin-right:1rem}.btn-check{position:absolute;clip:rect(0,0,0,0);pointer-events:none}.btn-check[disabled]+.btn,.btn-check:disabled+.btn{pointer-events:none;filter:none;opacity:.65}[data-bs-theme=dark] .form-switch .form-check-input:not(:checked):not(:focus){--bs-form-switch-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='rgba%28255, 255, 255, 0.25%29'/%3e%3c/svg%3e")}.form-range{width:100%;height:1.5rem;padding:0;appearance:none;background-color:transparent}.form-range:focus{outline:0}.form-range:focus::-webkit-slider-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .25rem #0d6efd40}.form-range:focus::-moz-range-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .25rem #0d6efd40}.form-range::-moz-focus-outer{border:0}.form-range::-webkit-slider-thumb{width:1rem;height:1rem;margin-top:-.25rem;appearance:none;background-color:#0d6efd;border:0;border-radius:1rem;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion: reduce){.form-range::-webkit-slider-thumb{transition:none}}.form-range::-webkit-slider-thumb:active{background-color:#b6d4fe}.form-range::-webkit-slider-runnable-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:var(--bs-secondary-bg);border-color:transparent;border-radius:1rem}.form-range::-moz-range-thumb{width:1rem;height:1rem;appearance:none;background-color:#0d6efd;border:0;border-radius:1rem;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion: reduce){.form-range::-moz-range-thumb{transition:none}}.form-range::-moz-range-thumb:active{background-color:#b6d4fe}.form-range::-moz-range-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:var(--bs-secondary-bg);border-color:transparent;border-radius:1rem}.form-range:disabled{pointer-events:none}.form-range:disabled::-webkit-slider-thumb{background-color:var(--bs-secondary-color)}.form-range:disabled::-moz-range-thumb{background-color:var(--bs-secondary-color)}.form-floating{position:relative}.form-floating>.form-control,.form-floating>.form-control-plaintext,.form-floating>.form-select{height:calc(3.5rem + calc(var(--bs-border-width) * 2));min-height:calc(3.5rem + calc(var(--bs-border-width) * 2));line-height:1.25}.form-floating>label{position:absolute;top:0;left:0;z-index:2;max-width:100%;height:100%;padding:1rem .75rem;overflow:hidden;color:rgba(var(--bs-body-color-rgb),.65);text-align:start;text-overflow:ellipsis;white-space:nowrap;pointer-events:none;border:var(--bs-border-width) solid transparent;transform-origin:0 0;transition:opacity .1s ease-in-out,transform .1s ease-in-out}@media (prefers-reduced-motion: reduce){.form-floating>label{transition:none}}.form-floating>.form-control,.form-floating>.form-control-plaintext{padding:1rem .75rem}.form-floating>.form-control::placeholder,.form-floating>.form-control-plaintext::placeholder{color:transparent}.form-floating>.form-control:focus,.form-floating>.form-control:not(:placeholder-shown),.form-floating>.form-control-plaintext:focus,.form-floating>.form-control-plaintext:not(:placeholder-shown){padding-top:1.625rem;padding-bottom:.625rem}.form-floating>.form-control:-webkit-autofill,.form-floating>.form-control-plaintext:-webkit-autofill{padding-top:1.625rem;padding-bottom:.625rem}.form-floating>.form-select{padding-top:1.625rem;padding-bottom:.625rem;padding-left:.75rem}.form-floating>.form-control:focus~label,.form-floating>.form-control:not(:placeholder-shown)~label,.form-floating>.form-control-plaintext~label,.form-floating>.form-select~label{transform:scale(.85) translateY(-.5rem) translate(.15rem)}.form-floating>.form-control:-webkit-autofill~label{transform:scale(.85) translateY(-.5rem) translate(.15rem)}.form-floating>textarea:focus~label:after,.form-floating>textarea:not(:placeholder-shown)~label:after{position:absolute;inset:1rem .375rem;z-index:-1;height:1.5em;content:"";background-color:var(--bs-body-bg);border-radius:var(--bs-border-radius)}.form-floating>textarea:disabled~label:after{background-color:var(--bs-secondary-bg)}.form-floating>.form-control-plaintext~label{border-width:var(--bs-border-width) 0}.form-floating>:disabled~label,.form-floating>.form-control:disabled~label{color:#6c757d}.input-group{position:relative;display:flex;flex-wrap:wrap;align-items:stretch;width:100%}.input-group>.form-control,.input-group>.form-select,.input-group>.form-floating{position:relative;flex:1 1 auto;width:1%;min-width:0}.input-group>.form-control:focus,.input-group>.form-select:focus,.input-group>.form-floating:focus-within{z-index:5}.input-group .btn{position:relative;z-index:2}.input-group .btn:focus{z-index:5}.input-group-text{display:flex;align-items:center;padding:.375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:var(--bs-body-color);text-align:center;white-space:nowrap;background-color:var(--bs-tertiary-bg);border:var(--bs-border-width) solid var(--bs-border-color);border-radius:var(--bs-border-radius)}.input-group-lg>.form-control,.input-group-lg>.form-select,.input-group-lg>.input-group-text,.input-group-lg>.btn{padding:.5rem 1rem;font-size:1.25rem;border-radius:var(--bs-border-radius-lg)}.input-group-sm>.form-control,.input-group-sm>.form-select,.input-group-sm>.input-group-text,.input-group-sm>.btn{padding:.25rem .5rem;font-size:.875rem;border-radius:var(--bs-border-radius-sm)}.input-group-lg>.form-select,.input-group-sm>.form-select{padding-right:3rem}.input-group:not(.has-validation)>:not(:last-child):not(.dropdown-toggle):not(.dropdown-menu):not(.form-floating),.input-group:not(.has-validation)>.dropdown-toggle:nth-last-child(n+3),.input-group:not(.has-validation)>.form-floating:not(:last-child)>.form-control,.input-group:not(.has-validation)>.form-floating:not(:last-child)>.form-select{border-top-right-radius:0;border-bottom-right-radius:0}.input-group.has-validation>:nth-last-child(n+3):not(.dropdown-toggle):not(.dropdown-menu):not(.form-floating),.input-group.has-validation>.dropdown-toggle:nth-last-child(n+4),.input-group.has-validation>.form-floating:nth-last-child(n+3)>.form-control,.input-group.has-validation>.form-floating:nth-last-child(n+3)>.form-select{border-top-right-radius:0;border-bottom-right-radius:0}.input-group>:not(:first-child):not(.dropdown-menu):not(.valid-tooltip):not(.valid-feedback):not(.invalid-tooltip):not(.invalid-feedback){margin-left:calc(-1 * var(--bs-border-width));border-top-left-radius:0;border-bottom-left-radius:0}.input-group>.form-floating:not(:first-child)>.form-control,.input-group>.form-floating:not(:first-child)>.form-select{border-top-left-radius:0;border-bottom-left-radius:0}.valid-feedback{display:none;width:100%;margin-top:.25rem;font-size:.875em;color:var(--bs-form-valid-color)}.valid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;color:#fff;background-color:var(--bs-success);border-radius:var(--bs-border-radius)}.was-validated :valid~.valid-feedback,.was-validated :valid~.valid-tooltip,.is-valid~.valid-feedback,.is-valid~.valid-tooltip{display:block}.was-validated .form-control:valid,.form-control.is-valid{border-color:var(--bs-form-valid-border-color);padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3e%3cpath fill='%23198754' d='M2.3 6.73.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(.375em + .1875rem) center;background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.was-validated .form-control:valid:focus,.form-control.is-valid:focus{border-color:var(--bs-form-valid-border-color);box-shadow:0 0 0 .25rem rgba(var(--bs-success-rgb),.25)}.was-validated textarea.form-control:valid,textarea.form-control.is-valid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.was-validated .form-select:valid,.form-select.is-valid{border-color:var(--bs-form-valid-border-color)}.was-validated .form-select:valid:not([multiple]):not([size]),.was-validated .form-select:valid:not([multiple])[size="1"],.form-select.is-valid:not([multiple]):not([size]),.form-select.is-valid:not([multiple])[size="1"]{--bs-form-select-bg-icon: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3e%3cpath fill='%23198754' d='M2.3 6.73.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1'/%3e%3c/svg%3e");padding-right:4.125rem;background-position:right .75rem center,center right 2.25rem;background-size:16px 12px,calc(.75em + .375rem) calc(.75em + .375rem)}.was-validated .form-select:valid:focus,.form-select.is-valid:focus{border-color:var(--bs-form-valid-border-color);box-shadow:0 0 0 .25rem rgba(var(--bs-success-rgb),.25)}.was-validated .form-control-color:valid,.form-control-color.is-valid{width:calc(3.75rem + 1.5em)}.was-validated .form-check-input:valid,.form-check-input.is-valid{border-color:var(--bs-form-valid-border-color)}.was-validated .form-check-input:valid:checked,.form-check-input.is-valid:checked{background-color:var(--bs-form-valid-color)}.was-validated .form-check-input:valid:focus,.form-check-input.is-valid:focus{box-shadow:0 0 0 .25rem rgba(var(--bs-success-rgb),.25)}.was-validated .form-check-input:valid~.form-check-label,.form-check-input.is-valid~.form-check-label{color:var(--bs-form-valid-color)}.form-check-inline .form-check-input~.valid-feedback{margin-left:.5em}.was-validated .input-group>.form-control:not(:focus):valid,.input-group>.form-control:not(:focus).is-valid,.was-validated .input-group>.form-select:not(:focus):valid,.input-group>.form-select:not(:focus).is-valid,.was-validated .input-group>.form-floating:not(:focus-within):valid,.input-group>.form-floating:not(:focus-within).is-valid{z-index:3}.invalid-feedback{display:none;width:100%;margin-top:.25rem;font-size:.875em;color:var(--bs-form-invalid-color)}.invalid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;color:#fff;background-color:var(--bs-danger);border-radius:var(--bs-border-radius)}.was-validated :invalid~.invalid-feedback,.was-validated :invalid~.invalid-tooltip,.is-invalid~.invalid-feedback,.is-invalid~.invalid-tooltip{display:block}.was-validated .form-control:invalid,.form-control.is-invalid{border-color:var(--bs-form-invalid-border-color);padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 12 12' width='12' height='12' fill='none' stroke='%23dc3545'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(.375em + .1875rem) center;background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.was-validated .form-control:invalid:focus,.form-control.is-invalid:focus{border-color:var(--bs-form-invalid-border-color);box-shadow:0 0 0 .25rem rgba(var(--bs-danger-rgb),.25)}.was-validated textarea.form-control:invalid,textarea.form-control.is-invalid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.was-validated .form-select:invalid,.form-select.is-invalid{border-color:var(--bs-form-invalid-border-color)}.was-validated .form-select:invalid:not([multiple]):not([size]),.was-validated .form-select:invalid:not([multiple])[size="1"],.form-select.is-invalid:not([multiple]):not([size]),.form-select.is-invalid:not([multiple])[size="1"]{--bs-form-select-bg-icon: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 12 12' width='12' height='12' fill='none' stroke='%23dc3545'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e");padding-right:4.125rem;background-position:right .75rem center,center right 2.25rem;background-size:16px 12px,calc(.75em + .375rem) calc(.75em + .375rem)}.was-validated .form-select:invalid:focus,.form-select.is-invalid:focus{border-color:var(--bs-form-invalid-border-color);box-shadow:0 0 0 .25rem rgba(var(--bs-danger-rgb),.25)}.was-validated .form-control-color:invalid,.form-control-color.is-invalid{width:calc(3.75rem + 1.5em)}.was-validated .form-check-input:invalid,.form-check-input.is-invalid{border-color:var(--bs-form-invalid-border-color)}.was-validated .form-check-input:invalid:checked,.form-check-input.is-invalid:checked{background-color:var(--bs-form-invalid-color)}.was-validated .form-check-input:invalid:focus,.form-check-input.is-invalid:focus{box-shadow:0 0 0 .25rem rgba(var(--bs-danger-rgb),.25)}.was-validated .form-check-input:invalid~.form-check-label,.form-check-input.is-invalid~.form-check-label{color:var(--bs-form-invalid-color)}.form-check-inline .form-check-input~.invalid-feedback{margin-left:.5em}.was-validated .input-group>.form-control:not(:focus):invalid,.input-group>.form-control:not(:focus).is-invalid,.was-validated .input-group>.form-select:not(:focus):invalid,.input-group>.form-select:not(:focus).is-invalid,.was-validated .input-group>.form-floating:not(:focus-within):invalid,.input-group>.form-floating:not(:focus-within).is-invalid{z-index:4}.btn{--bs-btn-padding-x: .75rem;--bs-btn-padding-y: .375rem;--bs-btn-font-family: ;--bs-btn-font-size: 1rem;--bs-btn-font-weight: 400;--bs-btn-line-height: 1.5;--bs-btn-color: var(--bs-body-color);--bs-btn-bg: transparent;--bs-btn-border-width: var(--bs-border-width);--bs-btn-border-color: transparent;--bs-btn-border-radius: var(--bs-border-radius);--bs-btn-hover-border-color: transparent;--bs-btn-box-shadow: inset 0 1px 0 rgba(255, 255, 255, .15), 0 1px 1px rgba(0, 0, 0, .075);--bs-btn-disabled-opacity: .65;--bs-btn-focus-box-shadow: 0 0 0 .25rem rgba(var(--bs-btn-focus-shadow-rgb), .5);display:inline-block;padding:var(--bs-btn-padding-y) var(--bs-btn-padding-x);font-family:var(--bs-btn-font-family);font-size:var(--bs-btn-font-size);font-weight:var(--bs-btn-font-weight);line-height:var(--bs-btn-line-height);color:var(--bs-btn-color);text-align:center;text-decoration:none;vertical-align:middle;cursor:pointer;user-select:none;border:var(--bs-btn-border-width) solid var(--bs-btn-border-color);border-radius:var(--bs-btn-border-radius);background-color:var(--bs-btn-bg);transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion: reduce){.btn{transition:none}}.btn:hover{color:var(--bs-btn-hover-color);background-color:var(--bs-btn-hover-bg);border-color:var(--bs-btn-hover-border-color)}.btn-check+.btn:hover{color:var(--bs-btn-color);background-color:var(--bs-btn-bg);border-color:var(--bs-btn-border-color)}.btn:focus-visible{color:var(--bs-btn-hover-color);background-color:var(--bs-btn-hover-bg);border-color:var(--bs-btn-hover-border-color);outline:0;box-shadow:var(--bs-btn-focus-box-shadow)}.btn-check:focus-visible+.btn{border-color:var(--bs-btn-hover-border-color);outline:0;box-shadow:var(--bs-btn-focus-box-shadow)}.btn-check:checked+.btn,:not(.btn-check)+.btn:active,.btn:first-child:active,.btn.active,.btn.show{color:var(--bs-btn-active-color);background-color:var(--bs-btn-active-bg);border-color:var(--bs-btn-active-border-color)}.btn-check:checked+.btn:focus-visible,:not(.btn-check)+.btn:active:focus-visible,.btn:first-child:active:focus-visible,.btn.active:focus-visible,.btn.show:focus-visible{box-shadow:var(--bs-btn-focus-box-shadow)}.btn-check:checked:focus-visible+.btn{box-shadow:var(--bs-btn-focus-box-shadow)}.btn:disabled,.btn.disabled,fieldset:disabled .btn{color:var(--bs-btn-disabled-color);pointer-events:none;background-color:var(--bs-btn-disabled-bg);border-color:var(--bs-btn-disabled-border-color);opacity:var(--bs-btn-disabled-opacity)}.btn-primary{--bs-btn-color: #fff;--bs-btn-bg: #0d6efd;--bs-btn-border-color: #0d6efd;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #0b5ed7;--bs-btn-hover-border-color: #0a58ca;--bs-btn-focus-shadow-rgb: 49, 132, 253;--bs-btn-active-color: #fff;--bs-btn-active-bg: #0a58ca;--bs-btn-active-border-color: #0a53be;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, .125);--bs-btn-disabled-color: #fff;--bs-btn-disabled-bg: #0d6efd;--bs-btn-disabled-border-color: #0d6efd}.btn-secondary{--bs-btn-color: #fff;--bs-btn-bg: #6c757d;--bs-btn-border-color: #6c757d;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #5c636a;--bs-btn-hover-border-color: #565e64;--bs-btn-focus-shadow-rgb: 130, 138, 145;--bs-btn-active-color: #fff;--bs-btn-active-bg: #565e64;--bs-btn-active-border-color: #51585e;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, .125);--bs-btn-disabled-color: #fff;--bs-btn-disabled-bg: #6c757d;--bs-btn-disabled-border-color: #6c757d}.btn-success{--bs-btn-color: #fff;--bs-btn-bg: #198754;--bs-btn-border-color: #198754;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #157347;--bs-btn-hover-border-color: #146c43;--bs-btn-focus-shadow-rgb: 60, 153, 110;--bs-btn-active-color: #fff;--bs-btn-active-bg: #146c43;--bs-btn-active-border-color: #13653f;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, .125);--bs-btn-disabled-color: #fff;--bs-btn-disabled-bg: #198754;--bs-btn-disabled-border-color: #198754}.btn-info{--bs-btn-color: #000;--bs-btn-bg: #0dcaf0;--bs-btn-border-color: #0dcaf0;--bs-btn-hover-color: #000;--bs-btn-hover-bg: #31d2f2;--bs-btn-hover-border-color: #25cff2;--bs-btn-focus-shadow-rgb: 11, 172, 204;--bs-btn-active-color: #000;--bs-btn-active-bg: #3dd5f3;--bs-btn-active-border-color: #25cff2;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, .125);--bs-btn-disabled-color: #000;--bs-btn-disabled-bg: #0dcaf0;--bs-btn-disabled-border-color: #0dcaf0}.btn-warning{--bs-btn-color: #000;--bs-btn-bg: #ffc107;--bs-btn-border-color: #ffc107;--bs-btn-hover-color: #000;--bs-btn-hover-bg: #ffca2c;--bs-btn-hover-border-color: #ffc720;--bs-btn-focus-shadow-rgb: 217, 164, 6;--bs-btn-active-color: #000;--bs-btn-active-bg: #ffcd39;--bs-btn-active-border-color: #ffc720;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, .125);--bs-btn-disabled-color: #000;--bs-btn-disabled-bg: #ffc107;--bs-btn-disabled-border-color: #ffc107}.btn-danger{--bs-btn-color: #fff;--bs-btn-bg: #dc3545;--bs-btn-border-color: #dc3545;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #bb2d3b;--bs-btn-hover-border-color: #b02a37;--bs-btn-focus-shadow-rgb: 225, 83, 97;--bs-btn-active-color: #fff;--bs-btn-active-bg: #b02a37;--bs-btn-active-border-color: #a52834;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, .125);--bs-btn-disabled-color: #fff;--bs-btn-disabled-bg: #dc3545;--bs-btn-disabled-border-color: #dc3545}.btn-light{--bs-btn-color: #000;--bs-btn-bg: #f8f9fa;--bs-btn-border-color: #f8f9fa;--bs-btn-hover-color: #000;--bs-btn-hover-bg: #d3d4d5;--bs-btn-hover-border-color: #c6c7c8;--bs-btn-focus-shadow-rgb: 211, 212, 213;--bs-btn-active-color: #000;--bs-btn-active-bg: #c6c7c8;--bs-btn-active-border-color: #babbbc;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, .125);--bs-btn-disabled-color: #000;--bs-btn-disabled-bg: #f8f9fa;--bs-btn-disabled-border-color: #f8f9fa}.btn-dark{--bs-btn-color: #fff;--bs-btn-bg: #212529;--bs-btn-border-color: #212529;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #424649;--bs-btn-hover-border-color: #373b3e;--bs-btn-focus-shadow-rgb: 66, 70, 73;--bs-btn-active-color: #fff;--bs-btn-active-bg: #4d5154;--bs-btn-active-border-color: #373b3e;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, .125);--bs-btn-disabled-color: #fff;--bs-btn-disabled-bg: #212529;--bs-btn-disabled-border-color: #212529}.btn-outline-primary{--bs-btn-color: #0d6efd;--bs-btn-border-color: #0d6efd;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #0d6efd;--bs-btn-hover-border-color: #0d6efd;--bs-btn-focus-shadow-rgb: 13, 110, 253;--bs-btn-active-color: #fff;--bs-btn-active-bg: #0d6efd;--bs-btn-active-border-color: #0d6efd;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, .125);--bs-btn-disabled-color: #0d6efd;--bs-btn-disabled-bg: transparent;--bs-btn-disabled-border-color: #0d6efd;--bs-gradient: none}.btn-outline-secondary{--bs-btn-color: #6c757d;--bs-btn-border-color: #6c757d;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #6c757d;--bs-btn-hover-border-color: #6c757d;--bs-btn-focus-shadow-rgb: 108, 117, 125;--bs-btn-active-color: #fff;--bs-btn-active-bg: #6c757d;--bs-btn-active-border-color: #6c757d;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, .125);--bs-btn-disabled-color: #6c757d;--bs-btn-disabled-bg: transparent;--bs-btn-disabled-border-color: #6c757d;--bs-gradient: none}.btn-outline-success{--bs-btn-color: #198754;--bs-btn-border-color: #198754;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #198754;--bs-btn-hover-border-color: #198754;--bs-btn-focus-shadow-rgb: 25, 135, 84;--bs-btn-active-color: #fff;--bs-btn-active-bg: #198754;--bs-btn-active-border-color: #198754;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, .125);--bs-btn-disabled-color: #198754;--bs-btn-disabled-bg: transparent;--bs-btn-disabled-border-color: #198754;--bs-gradient: none}.btn-outline-info{--bs-btn-color: #0dcaf0;--bs-btn-border-color: #0dcaf0;--bs-btn-hover-color: #000;--bs-btn-hover-bg: #0dcaf0;--bs-btn-hover-border-color: #0dcaf0;--bs-btn-focus-shadow-rgb: 13, 202, 240;--bs-btn-active-color: #000;--bs-btn-active-bg: #0dcaf0;--bs-btn-active-border-color: #0dcaf0;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, .125);--bs-btn-disabled-color: #0dcaf0;--bs-btn-disabled-bg: transparent;--bs-btn-disabled-border-color: #0dcaf0;--bs-gradient: none}.btn-outline-warning{--bs-btn-color: #ffc107;--bs-btn-border-color: #ffc107;--bs-btn-hover-color: #000;--bs-btn-hover-bg: #ffc107;--bs-btn-hover-border-color: #ffc107;--bs-btn-focus-shadow-rgb: 255, 193, 7;--bs-btn-active-color: #000;--bs-btn-active-bg: #ffc107;--bs-btn-active-border-color: #ffc107;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, .125);--bs-btn-disabled-color: #ffc107;--bs-btn-disabled-bg: transparent;--bs-btn-disabled-border-color: #ffc107;--bs-gradient: none}.btn-outline-danger{--bs-btn-color: #dc3545;--bs-btn-border-color: #dc3545;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #dc3545;--bs-btn-hover-border-color: #dc3545;--bs-btn-focus-shadow-rgb: 220, 53, 69;--bs-btn-active-color: #fff;--bs-btn-active-bg: #dc3545;--bs-btn-active-border-color: #dc3545;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, .125);--bs-btn-disabled-color: #dc3545;--bs-btn-disabled-bg: transparent;--bs-btn-disabled-border-color: #dc3545;--bs-gradient: none}.btn-outline-light{--bs-btn-color: #f8f9fa;--bs-btn-border-color: #f8f9fa;--bs-btn-hover-color: #000;--bs-btn-hover-bg: #f8f9fa;--bs-btn-hover-border-color: #f8f9fa;--bs-btn-focus-shadow-rgb: 248, 249, 250;--bs-btn-active-color: #000;--bs-btn-active-bg: #f8f9fa;--bs-btn-active-border-color: #f8f9fa;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, .125);--bs-btn-disabled-color: #f8f9fa;--bs-btn-disabled-bg: transparent;--bs-btn-disabled-border-color: #f8f9fa;--bs-gradient: none}.btn-outline-dark{--bs-btn-color: #212529;--bs-btn-border-color: #212529;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #212529;--bs-btn-hover-border-color: #212529;--bs-btn-focus-shadow-rgb: 33, 37, 41;--bs-btn-active-color: #fff;--bs-btn-active-bg: #212529;--bs-btn-active-border-color: #212529;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, .125);--bs-btn-disabled-color: #212529;--bs-btn-disabled-bg: transparent;--bs-btn-disabled-border-color: #212529;--bs-gradient: none}.btn-link{--bs-btn-font-weight: 400;--bs-btn-color: var(--bs-link-color);--bs-btn-bg: transparent;--bs-btn-border-color: transparent;--bs-btn-hover-color: var(--bs-link-hover-color);--bs-btn-hover-border-color: transparent;--bs-btn-active-color: var(--bs-link-hover-color);--bs-btn-active-border-color: transparent;--bs-btn-disabled-color: #6c757d;--bs-btn-disabled-border-color: transparent;--bs-btn-box-shadow: 0 0 0 #000;--bs-btn-focus-shadow-rgb: 49, 132, 253;text-decoration:underline}.btn-link:focus-visible{color:var(--bs-btn-color)}.btn-link:hover{color:var(--bs-btn-hover-color)}.btn-lg,.btn-group-lg>.btn{--bs-btn-padding-y: .5rem;--bs-btn-padding-x: 1rem;--bs-btn-font-size: 1.25rem;--bs-btn-border-radius: var(--bs-border-radius-lg)}.btn-sm,.btn-group-sm>.btn{--bs-btn-padding-y: .25rem;--bs-btn-padding-x: .5rem;--bs-btn-font-size: .875rem;--bs-btn-border-radius: var(--bs-border-radius-sm)}.fade{transition:opacity .15s linear}@media (prefers-reduced-motion: reduce){.fade{transition:none}}.fade:not(.show){opacity:0}.collapse:not(.show){display:none}.collapsing{height:0;overflow:hidden;transition:height .35s ease}@media (prefers-reduced-motion: reduce){.collapsing{transition:none}}.collapsing.collapse-horizontal{width:0;height:auto;transition:width .35s ease}@media (prefers-reduced-motion: reduce){.collapsing.collapse-horizontal{transition:none}}.dropup,.dropend,.dropdown,.dropstart,.dropup-center,.dropdown-center{position:relative}.dropdown-toggle{white-space:nowrap}.dropdown-toggle:after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid;border-right:.3em solid transparent;border-bottom:0;border-left:.3em solid transparent}.dropdown-toggle:empty:after{margin-left:0}.dropdown-menu{--bs-dropdown-zindex: 1000;--bs-dropdown-min-width: 10rem;--bs-dropdown-padding-x: 0;--bs-dropdown-padding-y: .5rem;--bs-dropdown-spacer: .125rem;--bs-dropdown-font-size: 1rem;--bs-dropdown-color: var(--bs-body-color);--bs-dropdown-bg: var(--bs-body-bg);--bs-dropdown-border-color: var(--bs-border-color-translucent);--bs-dropdown-border-radius: var(--bs-border-radius);--bs-dropdown-border-width: var(--bs-border-width);--bs-dropdown-inner-border-radius: calc(var(--bs-border-radius) - var(--bs-border-width));--bs-dropdown-divider-bg: var(--bs-border-color-translucent);--bs-dropdown-divider-margin-y: .5rem;--bs-dropdown-box-shadow: var(--bs-box-shadow);--bs-dropdown-link-color: var(--bs-body-color);--bs-dropdown-link-hover-color: var(--bs-body-color);--bs-dropdown-link-hover-bg: var(--bs-tertiary-bg);--bs-dropdown-link-active-color: #fff;--bs-dropdown-link-active-bg: #0d6efd;--bs-dropdown-link-disabled-color: var(--bs-tertiary-color);--bs-dropdown-item-padding-x: 1rem;--bs-dropdown-item-padding-y: .25rem;--bs-dropdown-header-color: #6c757d;--bs-dropdown-header-padding-x: 1rem;--bs-dropdown-header-padding-y: .5rem;position:absolute;z-index:var(--bs-dropdown-zindex);display:none;min-width:var(--bs-dropdown-min-width);padding:var(--bs-dropdown-padding-y) var(--bs-dropdown-padding-x);margin:0;font-size:var(--bs-dropdown-font-size);color:var(--bs-dropdown-color);text-align:left;list-style:none;background-color:var(--bs-dropdown-bg);background-clip:padding-box;border:var(--bs-dropdown-border-width) solid var(--bs-dropdown-border-color);border-radius:var(--bs-dropdown-border-radius)}.dropdown-menu[data-bs-popper]{top:100%;left:0;margin-top:var(--bs-dropdown-spacer)}.dropdown-menu-start{--bs-position: start}.dropdown-menu-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-end{--bs-position: end}.dropdown-menu-end[data-bs-popper]{right:0;left:auto}@media (min-width: 576px){.dropdown-menu-sm-start{--bs-position: start}.dropdown-menu-sm-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-sm-end{--bs-position: end}.dropdown-menu-sm-end[data-bs-popper]{right:0;left:auto}}@media (min-width: 768px){.dropdown-menu-md-start{--bs-position: start}.dropdown-menu-md-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-md-end{--bs-position: end}.dropdown-menu-md-end[data-bs-popper]{right:0;left:auto}}@media (min-width: 992px){.dropdown-menu-lg-start{--bs-position: start}.dropdown-menu-lg-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-lg-end{--bs-position: end}.dropdown-menu-lg-end[data-bs-popper]{right:0;left:auto}}@media (min-width: 1200px){.dropdown-menu-xl-start{--bs-position: start}.dropdown-menu-xl-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-xl-end{--bs-position: end}.dropdown-menu-xl-end[data-bs-popper]{right:0;left:auto}}@media (min-width: 1400px){.dropdown-menu-xxl-start{--bs-position: start}.dropdown-menu-xxl-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-xxl-end{--bs-position: end}.dropdown-menu-xxl-end[data-bs-popper]{right:0;left:auto}}.dropup .dropdown-menu[data-bs-popper]{top:auto;bottom:100%;margin-top:0;margin-bottom:var(--bs-dropdown-spacer)}.dropup .dropdown-toggle:after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:0;border-right:.3em solid transparent;border-bottom:.3em solid;border-left:.3em solid transparent}.dropup .dropdown-toggle:empty:after{margin-left:0}.dropend .dropdown-menu[data-bs-popper]{top:0;right:auto;left:100%;margin-top:0;margin-left:var(--bs-dropdown-spacer)}.dropend .dropdown-toggle:after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:0;border-bottom:.3em solid transparent;border-left:.3em solid}.dropend .dropdown-toggle:empty:after{margin-left:0}.dropend .dropdown-toggle:after{vertical-align:0}.dropstart .dropdown-menu[data-bs-popper]{top:0;right:100%;left:auto;margin-top:0;margin-right:var(--bs-dropdown-spacer)}.dropstart .dropdown-toggle:after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:""}.dropstart .dropdown-toggle:after{display:none}.dropstart .dropdown-toggle:before{display:inline-block;margin-right:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:.3em solid;border-bottom:.3em solid transparent}.dropstart .dropdown-toggle:empty:after{margin-left:0}.dropstart .dropdown-toggle:before{vertical-align:0}.dropdown-divider{height:0;margin:var(--bs-dropdown-divider-margin-y) 0;overflow:hidden;border-top:1px solid var(--bs-dropdown-divider-bg);opacity:1}.dropdown-item{display:block;width:100%;padding:var(--bs-dropdown-item-padding-y) var(--bs-dropdown-item-padding-x);clear:both;font-weight:400;color:var(--bs-dropdown-link-color);text-align:inherit;text-decoration:none;white-space:nowrap;background-color:transparent;border:0;border-radius:var(--bs-dropdown-item-border-radius, 0)}.dropdown-item:hover,.dropdown-item:focus{color:var(--bs-dropdown-link-hover-color);background-color:var(--bs-dropdown-link-hover-bg)}.dropdown-item.active,.dropdown-item:active{color:var(--bs-dropdown-link-active-color);text-decoration:none;background-color:var(--bs-dropdown-link-active-bg)}.dropdown-item.disabled,.dropdown-item:disabled{color:var(--bs-dropdown-link-disabled-color);pointer-events:none;background-color:transparent}.dropdown-menu.show{display:block}.dropdown-header{display:block;padding:var(--bs-dropdown-header-padding-y) var(--bs-dropdown-header-padding-x);margin-bottom:0;font-size:.875rem;color:var(--bs-dropdown-header-color);white-space:nowrap}.dropdown-item-text{display:block;padding:var(--bs-dropdown-item-padding-y) var(--bs-dropdown-item-padding-x);color:var(--bs-dropdown-link-color)}.dropdown-menu-dark{--bs-dropdown-color: #dee2e6;--bs-dropdown-bg: #343a40;--bs-dropdown-border-color: var(--bs-border-color-translucent);--bs-dropdown-box-shadow: ;--bs-dropdown-link-color: #dee2e6;--bs-dropdown-link-hover-color: #fff;--bs-dropdown-divider-bg: var(--bs-border-color-translucent);--bs-dropdown-link-hover-bg: rgba(255, 255, 255, .15);--bs-dropdown-link-active-color: #fff;--bs-dropdown-link-active-bg: #0d6efd;--bs-dropdown-link-disabled-color: #adb5bd;--bs-dropdown-header-color: #adb5bd}.btn-group,.btn-group-vertical{position:relative;display:inline-flex;vertical-align:middle}.btn-group>.btn,.btn-group-vertical>.btn{position:relative;flex:1 1 auto}.btn-group>.btn-check:checked+.btn,.btn-group>.btn-check:focus+.btn,.btn-group>.btn:hover,.btn-group>.btn:focus,.btn-group>.btn:active,.btn-group>.btn.active,.btn-group-vertical>.btn-check:checked+.btn,.btn-group-vertical>.btn-check:focus+.btn,.btn-group-vertical>.btn:hover,.btn-group-vertical>.btn:focus,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn.active{z-index:1}.btn-toolbar{display:flex;flex-wrap:wrap;justify-content:flex-start}.btn-toolbar .input-group{width:auto}.btn-group{border-radius:var(--bs-border-radius)}.btn-group>:not(.btn-check:first-child)+.btn,.btn-group>.btn-group:not(:first-child){margin-left:calc(-1 * var(--bs-border-width))}.btn-group>.btn:not(:last-child):not(.dropdown-toggle),.btn-group>.btn.dropdown-toggle-split:first-child,.btn-group>.btn-group:not(:last-child)>.btn{border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn:nth-child(n+3),.btn-group>:not(.btn-check)+.btn,.btn-group>.btn-group:not(:first-child)>.btn{border-top-left-radius:0;border-bottom-left-radius:0}.dropdown-toggle-split{padding-right:.5625rem;padding-left:.5625rem}.dropdown-toggle-split:after,.dropup .dropdown-toggle-split:after,.dropend .dropdown-toggle-split:after{margin-left:0}.dropstart .dropdown-toggle-split:before{margin-right:0}.btn-sm+.dropdown-toggle-split,.btn-group-sm>.btn+.dropdown-toggle-split{padding-right:.375rem;padding-left:.375rem}.btn-lg+.dropdown-toggle-split,.btn-group-lg>.btn+.dropdown-toggle-split{padding-right:.75rem;padding-left:.75rem}.btn-group-vertical{flex-direction:column;align-items:flex-start;justify-content:center}.btn-group-vertical>.btn,.btn-group-vertical>.btn-group{width:100%}.btn-group-vertical>.btn:not(:first-child),.btn-group-vertical>.btn-group:not(:first-child){margin-top:calc(-1 * var(--bs-border-width))}.btn-group-vertical>.btn:not(:last-child):not(.dropdown-toggle),.btn-group-vertical>.btn-group:not(:last-child)>.btn{border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn:nth-child(n+3),.btn-group-vertical>:not(.btn-check)+.btn,.btn-group-vertical>.btn-group:not(:first-child)>.btn{border-top-left-radius:0;border-top-right-radius:0}.nav{--bs-nav-link-padding-x: 1rem;--bs-nav-link-padding-y: .5rem;--bs-nav-link-font-weight: ;--bs-nav-link-color: var(--bs-link-color);--bs-nav-link-hover-color: var(--bs-link-hover-color);--bs-nav-link-disabled-color: var(--bs-secondary-color);display:flex;flex-wrap:wrap;padding-left:0;margin-bottom:0;list-style:none}.nav-link{display:block;padding:var(--bs-nav-link-padding-y) var(--bs-nav-link-padding-x);font-size:var(--bs-nav-link-font-size);font-weight:var(--bs-nav-link-font-weight);color:var(--bs-nav-link-color);text-decoration:none;background:none;border:0;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out}@media (prefers-reduced-motion: reduce){.nav-link{transition:none}}.nav-link:hover,.nav-link:focus{color:var(--bs-nav-link-hover-color)}.nav-link:focus-visible{outline:0;box-shadow:0 0 0 .25rem #0d6efd40}.nav-link.disabled,.nav-link:disabled{color:var(--bs-nav-link-disabled-color);pointer-events:none;cursor:default}.nav-tabs{--bs-nav-tabs-border-width: var(--bs-border-width);--bs-nav-tabs-border-color: var(--bs-border-color);--bs-nav-tabs-border-radius: var(--bs-border-radius);--bs-nav-tabs-link-hover-border-color: var(--bs-secondary-bg) var(--bs-secondary-bg) var(--bs-border-color);--bs-nav-tabs-link-active-color: var(--bs-emphasis-color);--bs-nav-tabs-link-active-bg: var(--bs-body-bg);--bs-nav-tabs-link-active-border-color: var(--bs-border-color) var(--bs-border-color) var(--bs-body-bg);border-bottom:var(--bs-nav-tabs-border-width) solid var(--bs-nav-tabs-border-color)}.nav-tabs .nav-link{margin-bottom:calc(-1 * var(--bs-nav-tabs-border-width));border:var(--bs-nav-tabs-border-width) solid transparent;border-top-left-radius:var(--bs-nav-tabs-border-radius);border-top-right-radius:var(--bs-nav-tabs-border-radius)}.nav-tabs .nav-link:hover,.nav-tabs .nav-link:focus{isolation:isolate;border-color:var(--bs-nav-tabs-link-hover-border-color)}.nav-tabs .nav-link.active,.nav-tabs .nav-item.show .nav-link{color:var(--bs-nav-tabs-link-active-color);background-color:var(--bs-nav-tabs-link-active-bg);border-color:var(--bs-nav-tabs-link-active-border-color)}.nav-tabs .dropdown-menu{margin-top:calc(-1 * var(--bs-nav-tabs-border-width));border-top-left-radius:0;border-top-right-radius:0}.nav-pills{--bs-nav-pills-border-radius: var(--bs-border-radius);--bs-nav-pills-link-active-color: #fff;--bs-nav-pills-link-active-bg: #0d6efd}.nav-pills .nav-link{border-radius:var(--bs-nav-pills-border-radius)}.nav-pills .nav-link.active,.nav-pills .show>.nav-link{color:var(--bs-nav-pills-link-active-color);background-color:var(--bs-nav-pills-link-active-bg)}.nav-underline{--bs-nav-underline-gap: 1rem;--bs-nav-underline-border-width: .125rem;--bs-nav-underline-link-active-color: var(--bs-emphasis-color);gap:var(--bs-nav-underline-gap)}.nav-underline .nav-link{padding-right:0;padding-left:0;border-bottom:var(--bs-nav-underline-border-width) solid transparent}.nav-underline .nav-link:hover,.nav-underline .nav-link:focus{border-bottom-color:currentcolor}.nav-underline .nav-link.active,.nav-underline .show>.nav-link{font-weight:700;color:var(--bs-nav-underline-link-active-color);border-bottom-color:currentcolor}.nav-fill>.nav-link,.nav-fill .nav-item{flex:1 1 auto;text-align:center}.nav-justified>.nav-link,.nav-justified .nav-item{flex-grow:1;flex-basis:0;text-align:center}.nav-fill .nav-item .nav-link,.nav-justified .nav-item .nav-link{width:100%}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.navbar{--bs-navbar-padding-x: 0;--bs-navbar-padding-y: .5rem;--bs-navbar-color: rgba(var(--bs-emphasis-color-rgb), .65);--bs-navbar-hover-color: rgba(var(--bs-emphasis-color-rgb), .8);--bs-navbar-disabled-color: rgba(var(--bs-emphasis-color-rgb), .3);--bs-navbar-active-color: rgba(var(--bs-emphasis-color-rgb), 1);--bs-navbar-brand-padding-y: .3125rem;--bs-navbar-brand-margin-end: 1rem;--bs-navbar-brand-font-size: 1.25rem;--bs-navbar-brand-color: rgba(var(--bs-emphasis-color-rgb), 1);--bs-navbar-brand-hover-color: rgba(var(--bs-emphasis-color-rgb), 1);--bs-navbar-nav-link-padding-x: .5rem;--bs-navbar-toggler-padding-y: .25rem;--bs-navbar-toggler-padding-x: .75rem;--bs-navbar-toggler-font-size: 1.25rem;--bs-navbar-toggler-icon-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 30 30'%3e%3cpath stroke='rgba%2833, 37, 41, 0.75%29' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e");--bs-navbar-toggler-border-color: rgba(var(--bs-emphasis-color-rgb), .15);--bs-navbar-toggler-border-radius: var(--bs-border-radius);--bs-navbar-toggler-focus-width: .25rem;--bs-navbar-toggler-transition: box-shadow .15s ease-in-out;position:relative;display:flex;flex-wrap:wrap;align-items:center;justify-content:space-between;padding:var(--bs-navbar-padding-y) var(--bs-navbar-padding-x)}.navbar>.container,.navbar>.container-fluid,.navbar>.container-sm,.navbar>.container-md,.navbar>.container-lg,.navbar>.container-xl,.navbar>.container-xxl{display:flex;flex-wrap:inherit;align-items:center;justify-content:space-between}.navbar-brand{padding-top:var(--bs-navbar-brand-padding-y);padding-bottom:var(--bs-navbar-brand-padding-y);margin-right:var(--bs-navbar-brand-margin-end);font-size:var(--bs-navbar-brand-font-size);color:var(--bs-navbar-brand-color);text-decoration:none;white-space:nowrap}.navbar-brand:hover,.navbar-brand:focus{color:var(--bs-navbar-brand-hover-color)}.navbar-nav{--bs-nav-link-padding-x: 0;--bs-nav-link-padding-y: .5rem;--bs-nav-link-font-weight: ;--bs-nav-link-color: var(--bs-navbar-color);--bs-nav-link-hover-color: var(--bs-navbar-hover-color);--bs-nav-link-disabled-color: var(--bs-navbar-disabled-color);display:flex;flex-direction:column;padding-left:0;margin-bottom:0;list-style:none}.navbar-nav .nav-link.active,.navbar-nav .nav-link.show{color:var(--bs-navbar-active-color)}.navbar-nav .dropdown-menu{position:static}.navbar-text{padding-top:.5rem;padding-bottom:.5rem;color:var(--bs-navbar-color)}.navbar-text a,.navbar-text a:hover,.navbar-text a:focus{color:var(--bs-navbar-active-color)}.navbar-collapse{flex-grow:1;flex-basis:100%;align-items:center}.navbar-toggler{padding:var(--bs-navbar-toggler-padding-y) var(--bs-navbar-toggler-padding-x);font-size:var(--bs-navbar-toggler-font-size);line-height:1;color:var(--bs-navbar-color);background-color:transparent;border:var(--bs-border-width) solid var(--bs-navbar-toggler-border-color);border-radius:var(--bs-navbar-toggler-border-radius);transition:var(--bs-navbar-toggler-transition)}@media (prefers-reduced-motion: reduce){.navbar-toggler{transition:none}}.navbar-toggler:hover{text-decoration:none}.navbar-toggler:focus{text-decoration:none;outline:0;box-shadow:0 0 0 var(--bs-navbar-toggler-focus-width)}.navbar-toggler-icon{display:inline-block;width:1.5em;height:1.5em;vertical-align:middle;background-image:var(--bs-navbar-toggler-icon-bg);background-repeat:no-repeat;background-position:center;background-size:100%}.navbar-nav-scroll{max-height:var(--bs-scroll-height, 75vh);overflow-y:auto}@media (min-width: 576px){.navbar-expand-sm{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-sm .navbar-nav{flex-direction:row}.navbar-expand-sm .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-sm .navbar-nav .nav-link{padding-right:var(--bs-navbar-nav-link-padding-x);padding-left:var(--bs-navbar-nav-link-padding-x)}.navbar-expand-sm .navbar-nav-scroll{overflow:visible}.navbar-expand-sm .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-sm .navbar-toggler{display:none}.navbar-expand-sm .offcanvas{position:static;z-index:auto;flex-grow:1;width:auto!important;height:auto!important;visibility:visible!important;background-color:transparent!important;border:0!important;transform:none!important;transition:none}.navbar-expand-sm .offcanvas .offcanvas-header{display:none}.navbar-expand-sm .offcanvas .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}@media (min-width: 768px){.navbar-expand-md{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-md .navbar-nav{flex-direction:row}.navbar-expand-md .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-md .navbar-nav .nav-link{padding-right:var(--bs-navbar-nav-link-padding-x);padding-left:var(--bs-navbar-nav-link-padding-x)}.navbar-expand-md .navbar-nav-scroll{overflow:visible}.navbar-expand-md .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-md .navbar-toggler{display:none}.navbar-expand-md .offcanvas{position:static;z-index:auto;flex-grow:1;width:auto!important;height:auto!important;visibility:visible!important;background-color:transparent!important;border:0!important;transform:none!important;transition:none}.navbar-expand-md .offcanvas .offcanvas-header{display:none}.navbar-expand-md .offcanvas .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}@media (min-width: 992px){.navbar-expand-lg{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-lg .navbar-nav{flex-direction:row}.navbar-expand-lg .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-lg .navbar-nav .nav-link{padding-right:var(--bs-navbar-nav-link-padding-x);padding-left:var(--bs-navbar-nav-link-padding-x)}.navbar-expand-lg .navbar-nav-scroll{overflow:visible}.navbar-expand-lg .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-lg .navbar-toggler{display:none}.navbar-expand-lg .offcanvas{position:static;z-index:auto;flex-grow:1;width:auto!important;height:auto!important;visibility:visible!important;background-color:transparent!important;border:0!important;transform:none!important;transition:none}.navbar-expand-lg .offcanvas .offcanvas-header{display:none}.navbar-expand-lg .offcanvas .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}@media (min-width: 1200px){.navbar-expand-xl{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-xl .navbar-nav{flex-direction:row}.navbar-expand-xl .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-xl .navbar-nav .nav-link{padding-right:var(--bs-navbar-nav-link-padding-x);padding-left:var(--bs-navbar-nav-link-padding-x)}.navbar-expand-xl .navbar-nav-scroll{overflow:visible}.navbar-expand-xl .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-xl .navbar-toggler{display:none}.navbar-expand-xl .offcanvas{position:static;z-index:auto;flex-grow:1;width:auto!important;height:auto!important;visibility:visible!important;background-color:transparent!important;border:0!important;transform:none!important;transition:none}.navbar-expand-xl .offcanvas .offcanvas-header{display:none}.navbar-expand-xl .offcanvas .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}@media (min-width: 1400px){.navbar-expand-xxl{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-xxl .navbar-nav{flex-direction:row}.navbar-expand-xxl .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-xxl .navbar-nav .nav-link{padding-right:var(--bs-navbar-nav-link-padding-x);padding-left:var(--bs-navbar-nav-link-padding-x)}.navbar-expand-xxl .navbar-nav-scroll{overflow:visible}.navbar-expand-xxl .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-xxl .navbar-toggler{display:none}.navbar-expand-xxl .offcanvas{position:static;z-index:auto;flex-grow:1;width:auto!important;height:auto!important;visibility:visible!important;background-color:transparent!important;border:0!important;transform:none!important;transition:none}.navbar-expand-xxl .offcanvas .offcanvas-header{display:none}.navbar-expand-xxl .offcanvas .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}.navbar-expand{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand .navbar-nav{flex-direction:row}.navbar-expand .navbar-nav .dropdown-menu{position:absolute}.navbar-expand .navbar-nav .nav-link{padding-right:var(--bs-navbar-nav-link-padding-x);padding-left:var(--bs-navbar-nav-link-padding-x)}.navbar-expand .navbar-nav-scroll{overflow:visible}.navbar-expand .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand .navbar-toggler{display:none}.navbar-expand .offcanvas{position:static;z-index:auto;flex-grow:1;width:auto!important;height:auto!important;visibility:visible!important;background-color:transparent!important;border:0!important;transform:none!important;transition:none}.navbar-expand .offcanvas .offcanvas-header{display:none}.navbar-expand .offcanvas .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}.navbar-dark,.navbar[data-bs-theme=dark]{--bs-navbar-color: rgba(255, 255, 255, .55);--bs-navbar-hover-color: rgba(255, 255, 255, .75);--bs-navbar-disabled-color: rgba(255, 255, 255, .25);--bs-navbar-active-color: #fff;--bs-navbar-brand-color: #fff;--bs-navbar-brand-hover-color: #fff;--bs-navbar-toggler-border-color: rgba(255, 255, 255, .1);--bs-navbar-toggler-icon-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 30 30'%3e%3cpath stroke='rgba%28255, 255, 255, 0.55%29' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}[data-bs-theme=dark] .navbar-toggler-icon{--bs-navbar-toggler-icon-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 30 30'%3e%3cpath stroke='rgba%28255, 255, 255, 0.55%29' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.card{--bs-card-spacer-y: 1rem;--bs-card-spacer-x: 1rem;--bs-card-title-spacer-y: .5rem;--bs-card-title-color: ;--bs-card-subtitle-color: ;--bs-card-border-width: var(--bs-border-width);--bs-card-border-color: var(--bs-border-color-translucent);--bs-card-border-radius: var(--bs-border-radius);--bs-card-box-shadow: ;--bs-card-inner-border-radius: calc(var(--bs-border-radius) - (var(--bs-border-width)));--bs-card-cap-padding-y: .5rem;--bs-card-cap-padding-x: 1rem;--bs-card-cap-bg: rgba(var(--bs-body-color-rgb), .03);--bs-card-cap-color: ;--bs-card-height: ;--bs-card-color: ;--bs-card-bg: var(--bs-body-bg);--bs-card-img-overlay-padding: 1rem;--bs-card-group-margin: .75rem;position:relative;display:flex;flex-direction:column;min-width:0;height:var(--bs-card-height);color:var(--bs-body-color);word-wrap:break-word;background-color:var(--bs-card-bg);background-clip:border-box;border:var(--bs-card-border-width) solid var(--bs-card-border-color);border-radius:var(--bs-card-border-radius)}.card>hr{margin-right:0;margin-left:0}.card>.list-group{border-top:inherit;border-bottom:inherit}.card>.list-group:first-child{border-top-width:0;border-top-left-radius:var(--bs-card-inner-border-radius);border-top-right-radius:var(--bs-card-inner-border-radius)}.card>.list-group:last-child{border-bottom-width:0;border-bottom-right-radius:var(--bs-card-inner-border-radius);border-bottom-left-radius:var(--bs-card-inner-border-radius)}.card>.card-header+.list-group,.card>.list-group+.card-footer{border-top:0}.card-body{flex:1 1 auto;padding:var(--bs-card-spacer-y) var(--bs-card-spacer-x);color:var(--bs-card-color)}.card-title{margin-bottom:var(--bs-card-title-spacer-y);color:var(--bs-card-title-color)}.card-subtitle{margin-top:calc(-.5 * var(--bs-card-title-spacer-y));margin-bottom:0;color:var(--bs-card-subtitle-color)}.card-text:last-child{margin-bottom:0}.card-link+.card-link{margin-left:var(--bs-card-spacer-x)}.card-header{padding:var(--bs-card-cap-padding-y) var(--bs-card-cap-padding-x);margin-bottom:0;color:var(--bs-card-cap-color);background-color:var(--bs-card-cap-bg);border-bottom:var(--bs-card-border-width) solid var(--bs-card-border-color)}.card-header:first-child{border-radius:var(--bs-card-inner-border-radius) var(--bs-card-inner-border-radius) 0 0}.card-footer{padding:var(--bs-card-cap-padding-y) var(--bs-card-cap-padding-x);color:var(--bs-card-cap-color);background-color:var(--bs-card-cap-bg);border-top:var(--bs-card-border-width) solid var(--bs-card-border-color)}.card-footer:last-child{border-radius:0 0 var(--bs-card-inner-border-radius) var(--bs-card-inner-border-radius)}.card-header-tabs{margin-right:calc(-.5 * var(--bs-card-cap-padding-x));margin-bottom:calc(-1 * var(--bs-card-cap-padding-y));margin-left:calc(-.5 * var(--bs-card-cap-padding-x));border-bottom:0}.card-header-tabs .nav-link.active{background-color:var(--bs-card-bg);border-bottom-color:var(--bs-card-bg)}.card-header-pills{margin-right:calc(-.5 * var(--bs-card-cap-padding-x));margin-left:calc(-.5 * var(--bs-card-cap-padding-x))}.card-img-overlay{position:absolute;inset:0;padding:var(--bs-card-img-overlay-padding);border-radius:var(--bs-card-inner-border-radius)}.card-img,.card-img-top,.card-img-bottom{width:100%}.card-img,.card-img-top{border-top-left-radius:var(--bs-card-inner-border-radius);border-top-right-radius:var(--bs-card-inner-border-radius)}.card-img,.card-img-bottom{border-bottom-right-radius:var(--bs-card-inner-border-radius);border-bottom-left-radius:var(--bs-card-inner-border-radius)}.card-group>.card{margin-bottom:var(--bs-card-group-margin)}@media (min-width: 576px){.card-group{display:flex;flex-flow:row wrap}.card-group>.card{flex:1 0 0;margin-bottom:0}.card-group>.card+.card{margin-left:0;border-left:0}.card-group>.card:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.card-group>.card:not(:last-child) .card-img-top,.card-group>.card:not(:last-child) .card-header{border-top-right-radius:0}.card-group>.card:not(:last-child) .card-img-bottom,.card-group>.card:not(:last-child) .card-footer{border-bottom-right-radius:0}.card-group>.card:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.card-group>.card:not(:first-child) .card-img-top,.card-group>.card:not(:first-child) .card-header{border-top-left-radius:0}.card-group>.card:not(:first-child) .card-img-bottom,.card-group>.card:not(:first-child) .card-footer{border-bottom-left-radius:0}}.accordion{--bs-accordion-color: var(--bs-body-color);--bs-accordion-bg: var(--bs-body-bg);--bs-accordion-transition: color .15s ease-in-out, background-color .15s ease-in-out, border-color .15s ease-in-out, box-shadow .15s ease-in-out, border-radius .15s ease;--bs-accordion-border-color: var(--bs-border-color);--bs-accordion-border-width: var(--bs-border-width);--bs-accordion-border-radius: var(--bs-border-radius);--bs-accordion-inner-border-radius: calc(var(--bs-border-radius) - (var(--bs-border-width)));--bs-accordion-btn-padding-x: 1.25rem;--bs-accordion-btn-padding-y: 1rem;--bs-accordion-btn-color: var(--bs-body-color);--bs-accordion-btn-bg: var(--bs-accordion-bg);--bs-accordion-btn-icon: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='none' stroke='%23212529' stroke-linecap='round' stroke-linejoin='round'%3e%3cpath d='m2 5 6 6 6-6'/%3e%3c/svg%3e");--bs-accordion-btn-icon-width: 1.25rem;--bs-accordion-btn-icon-transform: rotate(-180deg);--bs-accordion-btn-icon-transition: transform .2s ease-in-out;--bs-accordion-btn-active-icon: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='none' stroke='%23052c65' stroke-linecap='round' stroke-linejoin='round'%3e%3cpath d='m2 5 6 6 6-6'/%3e%3c/svg%3e");--bs-accordion-btn-focus-box-shadow: 0 0 0 .25rem rgba(13, 110, 253, .25);--bs-accordion-body-padding-x: 1.25rem;--bs-accordion-body-padding-y: 1rem;--bs-accordion-active-color: var(--bs-primary-text-emphasis);--bs-accordion-active-bg: var(--bs-primary-bg-subtle)}.accordion-button{position:relative;display:flex;align-items:center;width:100%;padding:var(--bs-accordion-btn-padding-y) var(--bs-accordion-btn-padding-x);font-size:1rem;color:var(--bs-accordion-btn-color);text-align:left;background-color:var(--bs-accordion-btn-bg);border:0;border-radius:0;overflow-anchor:none;transition:var(--bs-accordion-transition)}@media (prefers-reduced-motion: reduce){.accordion-button{transition:none}}.accordion-button:not(.collapsed){color:var(--bs-accordion-active-color);background-color:var(--bs-accordion-active-bg);box-shadow:inset 0 calc(-1 * var(--bs-accordion-border-width)) 0 var(--bs-accordion-border-color)}.accordion-button:not(.collapsed):after{background-image:var(--bs-accordion-btn-active-icon);transform:var(--bs-accordion-btn-icon-transform)}.accordion-button:after{flex-shrink:0;width:var(--bs-accordion-btn-icon-width);height:var(--bs-accordion-btn-icon-width);margin-left:auto;content:"";background-image:var(--bs-accordion-btn-icon);background-repeat:no-repeat;background-size:var(--bs-accordion-btn-icon-width);transition:var(--bs-accordion-btn-icon-transition)}@media (prefers-reduced-motion: reduce){.accordion-button:after{transition:none}}.accordion-button:hover{z-index:2}.accordion-button:focus{z-index:3;outline:0;box-shadow:var(--bs-accordion-btn-focus-box-shadow)}.accordion-header{margin-bottom:0}.accordion-item{color:var(--bs-accordion-color);background-color:var(--bs-accordion-bg);border:var(--bs-accordion-border-width) solid var(--bs-accordion-border-color)}.accordion-item:first-of-type{border-top-left-radius:var(--bs-accordion-border-radius);border-top-right-radius:var(--bs-accordion-border-radius)}.accordion-item:first-of-type>.accordion-header .accordion-button{border-top-left-radius:var(--bs-accordion-inner-border-radius);border-top-right-radius:var(--bs-accordion-inner-border-radius)}.accordion-item:not(:first-of-type){border-top:0}.accordion-item:last-of-type{border-bottom-right-radius:var(--bs-accordion-border-radius);border-bottom-left-radius:var(--bs-accordion-border-radius)}.accordion-item:last-of-type>.accordion-header .accordion-button.collapsed{border-bottom-right-radius:var(--bs-accordion-inner-border-radius);border-bottom-left-radius:var(--bs-accordion-inner-border-radius)}.accordion-item:last-of-type>.accordion-collapse{border-bottom-right-radius:var(--bs-accordion-border-radius);border-bottom-left-radius:var(--bs-accordion-border-radius)}.accordion-body{padding:var(--bs-accordion-body-padding-y) var(--bs-accordion-body-padding-x)}.accordion-flush>.accordion-item{border-right:0;border-left:0;border-radius:0}.accordion-flush>.accordion-item:first-child{border-top:0}.accordion-flush>.accordion-item:last-child{border-bottom:0}.accordion-flush>.accordion-item>.accordion-collapse,.accordion-flush>.accordion-item>.accordion-header .accordion-button,.accordion-flush>.accordion-item>.accordion-header .accordion-button.collapsed{border-radius:0}[data-bs-theme=dark] .accordion-button:after{--bs-accordion-btn-icon: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%236ea8fe'%3e%3cpath fill-rule='evenodd' d='M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708'/%3e%3c/svg%3e");--bs-accordion-btn-active-icon: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%236ea8fe'%3e%3cpath fill-rule='evenodd' d='M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708'/%3e%3c/svg%3e")}.breadcrumb{--bs-breadcrumb-padding-x: 0;--bs-breadcrumb-padding-y: 0;--bs-breadcrumb-margin-bottom: 1rem;--bs-breadcrumb-bg: ;--bs-breadcrumb-border-radius: ;--bs-breadcrumb-divider-color: var(--bs-secondary-color);--bs-breadcrumb-item-padding-x: .5rem;--bs-breadcrumb-item-active-color: var(--bs-secondary-color);display:flex;flex-wrap:wrap;padding:var(--bs-breadcrumb-padding-y) var(--bs-breadcrumb-padding-x);margin-bottom:var(--bs-breadcrumb-margin-bottom);font-size:var(--bs-breadcrumb-font-size);list-style:none;background-color:var(--bs-breadcrumb-bg);border-radius:var(--bs-breadcrumb-border-radius)}.breadcrumb-item+.breadcrumb-item{padding-left:var(--bs-breadcrumb-item-padding-x)}.breadcrumb-item+.breadcrumb-item:before{float:left;padding-right:var(--bs-breadcrumb-item-padding-x);color:var(--bs-breadcrumb-divider-color);content:var(--bs-breadcrumb-divider, "/")}.breadcrumb-item.active{color:var(--bs-breadcrumb-item-active-color)}.pagination{--bs-pagination-padding-x: .75rem;--bs-pagination-padding-y: .375rem;--bs-pagination-font-size: 1rem;--bs-pagination-color: var(--bs-link-color);--bs-pagination-bg: var(--bs-body-bg);--bs-pagination-border-width: var(--bs-border-width);--bs-pagination-border-color: var(--bs-border-color);--bs-pagination-border-radius: var(--bs-border-radius);--bs-pagination-hover-color: var(--bs-link-hover-color);--bs-pagination-hover-bg: var(--bs-tertiary-bg);--bs-pagination-hover-border-color: var(--bs-border-color);--bs-pagination-focus-color: var(--bs-link-hover-color);--bs-pagination-focus-bg: var(--bs-secondary-bg);--bs-pagination-focus-box-shadow: 0 0 0 .25rem rgba(13, 110, 253, .25);--bs-pagination-active-color: #fff;--bs-pagination-active-bg: #0d6efd;--bs-pagination-active-border-color: #0d6efd;--bs-pagination-disabled-color: var(--bs-secondary-color);--bs-pagination-disabled-bg: var(--bs-secondary-bg);--bs-pagination-disabled-border-color: var(--bs-border-color);display:flex;padding-left:0;list-style:none}.page-link{position:relative;display:block;padding:var(--bs-pagination-padding-y) var(--bs-pagination-padding-x);font-size:var(--bs-pagination-font-size);color:var(--bs-pagination-color);text-decoration:none;background-color:var(--bs-pagination-bg);border:var(--bs-pagination-border-width) solid var(--bs-pagination-border-color);transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion: reduce){.page-link{transition:none}}.page-link:hover{z-index:2;color:var(--bs-pagination-hover-color);background-color:var(--bs-pagination-hover-bg);border-color:var(--bs-pagination-hover-border-color)}.page-link:focus{z-index:3;color:var(--bs-pagination-focus-color);background-color:var(--bs-pagination-focus-bg);outline:0;box-shadow:var(--bs-pagination-focus-box-shadow)}.page-link.active,.active>.page-link{z-index:3;color:var(--bs-pagination-active-color);background-color:var(--bs-pagination-active-bg);border-color:var(--bs-pagination-active-border-color)}.page-link.disabled,.disabled>.page-link{color:var(--bs-pagination-disabled-color);pointer-events:none;background-color:var(--bs-pagination-disabled-bg);border-color:var(--bs-pagination-disabled-border-color)}.page-item:not(:first-child) .page-link{margin-left:calc(-1 * var(--bs-border-width))}.page-item:first-child .page-link{border-top-left-radius:var(--bs-pagination-border-radius);border-bottom-left-radius:var(--bs-pagination-border-radius)}.page-item:last-child .page-link{border-top-right-radius:var(--bs-pagination-border-radius);border-bottom-right-radius:var(--bs-pagination-border-radius)}.pagination-lg{--bs-pagination-padding-x: 1.5rem;--bs-pagination-padding-y: .75rem;--bs-pagination-font-size: 1.25rem;--bs-pagination-border-radius: var(--bs-border-radius-lg)}.pagination-sm{--bs-pagination-padding-x: .5rem;--bs-pagination-padding-y: .25rem;--bs-pagination-font-size: .875rem;--bs-pagination-border-radius: var(--bs-border-radius-sm)}.badge{--bs-badge-padding-x: .65em;--bs-badge-padding-y: .35em;--bs-badge-font-size: .75em;--bs-badge-font-weight: 700;--bs-badge-color: #fff;--bs-badge-border-radius: var(--bs-border-radius);display:inline-block;padding:var(--bs-badge-padding-y) var(--bs-badge-padding-x);font-size:var(--bs-badge-font-size);font-weight:var(--bs-badge-font-weight);line-height:1;color:var(--bs-badge-color);text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:var(--bs-badge-border-radius)}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.alert{--bs-alert-bg: transparent;--bs-alert-padding-x: 1rem;--bs-alert-padding-y: 1rem;--bs-alert-margin-bottom: 1rem;--bs-alert-color: inherit;--bs-alert-border-color: transparent;--bs-alert-border: var(--bs-border-width) solid var(--bs-alert-border-color);--bs-alert-border-radius: var(--bs-border-radius);--bs-alert-link-color: inherit;position:relative;padding:var(--bs-alert-padding-y) var(--bs-alert-padding-x);margin-bottom:var(--bs-alert-margin-bottom);color:var(--bs-alert-color);background-color:var(--bs-alert-bg);border:var(--bs-alert-border);border-radius:var(--bs-alert-border-radius)}.alert-heading{color:inherit}.alert-link{font-weight:700;color:var(--bs-alert-link-color)}.alert-dismissible{padding-right:3rem}.alert-dismissible .btn-close{position:absolute;top:0;right:0;z-index:2;padding:1.25rem 1rem}.alert-primary{--bs-alert-color: var(--bs-primary-text-emphasis);--bs-alert-bg: var(--bs-primary-bg-subtle);--bs-alert-border-color: var(--bs-primary-border-subtle);--bs-alert-link-color: var(--bs-primary-text-emphasis)}.alert-secondary{--bs-alert-color: var(--bs-secondary-text-emphasis);--bs-alert-bg: var(--bs-secondary-bg-subtle);--bs-alert-border-color: var(--bs-secondary-border-subtle);--bs-alert-link-color: var(--bs-secondary-text-emphasis)}.alert-success{--bs-alert-color: var(--bs-success-text-emphasis);--bs-alert-bg: var(--bs-success-bg-subtle);--bs-alert-border-color: var(--bs-success-border-subtle);--bs-alert-link-color: var(--bs-success-text-emphasis)}.alert-info{--bs-alert-color: var(--bs-info-text-emphasis);--bs-alert-bg: var(--bs-info-bg-subtle);--bs-alert-border-color: var(--bs-info-border-subtle);--bs-alert-link-color: var(--bs-info-text-emphasis)}.alert-warning{--bs-alert-color: var(--bs-warning-text-emphasis);--bs-alert-bg: var(--bs-warning-bg-subtle);--bs-alert-border-color: var(--bs-warning-border-subtle);--bs-alert-link-color: var(--bs-warning-text-emphasis)}.alert-danger{--bs-alert-color: var(--bs-danger-text-emphasis);--bs-alert-bg: var(--bs-danger-bg-subtle);--bs-alert-border-color: var(--bs-danger-border-subtle);--bs-alert-link-color: var(--bs-danger-text-emphasis)}.alert-light{--bs-alert-color: var(--bs-light-text-emphasis);--bs-alert-bg: var(--bs-light-bg-subtle);--bs-alert-border-color: var(--bs-light-border-subtle);--bs-alert-link-color: var(--bs-light-text-emphasis)}.alert-dark{--bs-alert-color: var(--bs-dark-text-emphasis);--bs-alert-bg: var(--bs-dark-bg-subtle);--bs-alert-border-color: var(--bs-dark-border-subtle);--bs-alert-link-color: var(--bs-dark-text-emphasis)}@keyframes progress-bar-stripes{0%{background-position-x:var(--bs-progress-height)}}.progress,.progress-stacked{--bs-progress-height: 1rem;--bs-progress-font-size: .75rem;--bs-progress-bg: var(--bs-secondary-bg);--bs-progress-border-radius: var(--bs-border-radius);--bs-progress-box-shadow: var(--bs-box-shadow-inset);--bs-progress-bar-color: #fff;--bs-progress-bar-bg: #0d6efd;--bs-progress-bar-transition: width .6s ease;display:flex;height:var(--bs-progress-height);overflow:hidden;font-size:var(--bs-progress-font-size);background-color:var(--bs-progress-bg);border-radius:var(--bs-progress-border-radius)}.progress-bar{display:flex;flex-direction:column;justify-content:center;overflow:hidden;color:var(--bs-progress-bar-color);text-align:center;white-space:nowrap;background-color:var(--bs-progress-bar-bg);transition:var(--bs-progress-bar-transition)}@media (prefers-reduced-motion: reduce){.progress-bar{transition:none}}.progress-bar-striped{background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-size:var(--bs-progress-height) var(--bs-progress-height)}.progress-stacked>.progress{overflow:visible}.progress-stacked>.progress>.progress-bar{width:100%}.progress-bar-animated{animation:1s linear infinite progress-bar-stripes}@media (prefers-reduced-motion: reduce){.progress-bar-animated{animation:none}}.list-group{--bs-list-group-color: var(--bs-body-color);--bs-list-group-bg: var(--bs-body-bg);--bs-list-group-border-color: var(--bs-border-color);--bs-list-group-border-width: var(--bs-border-width);--bs-list-group-border-radius: var(--bs-border-radius);--bs-list-group-item-padding-x: 1rem;--bs-list-group-item-padding-y: .5rem;--bs-list-group-action-color: var(--bs-secondary-color);--bs-list-group-action-hover-color: var(--bs-emphasis-color);--bs-list-group-action-hover-bg: var(--bs-tertiary-bg);--bs-list-group-action-active-color: var(--bs-body-color);--bs-list-group-action-active-bg: var(--bs-secondary-bg);--bs-list-group-disabled-color: var(--bs-secondary-color);--bs-list-group-disabled-bg: var(--bs-body-bg);--bs-list-group-active-color: #fff;--bs-list-group-active-bg: #0d6efd;--bs-list-group-active-border-color: #0d6efd;display:flex;flex-direction:column;padding-left:0;margin-bottom:0;border-radius:var(--bs-list-group-border-radius)}.list-group-numbered{list-style-type:none;counter-reset:section}.list-group-numbered>.list-group-item:before{content:counters(section,".") ". ";counter-increment:section}.list-group-item{position:relative;display:block;padding:var(--bs-list-group-item-padding-y) var(--bs-list-group-item-padding-x);color:var(--bs-list-group-color);text-decoration:none;background-color:var(--bs-list-group-bg);border:var(--bs-list-group-border-width) solid var(--bs-list-group-border-color)}.list-group-item:first-child{border-top-left-radius:inherit;border-top-right-radius:inherit}.list-group-item:last-child{border-bottom-right-radius:inherit;border-bottom-left-radius:inherit}.list-group-item.disabled,.list-group-item:disabled{color:var(--bs-list-group-disabled-color);pointer-events:none;background-color:var(--bs-list-group-disabled-bg)}.list-group-item.active{z-index:2;color:var(--bs-list-group-active-color);background-color:var(--bs-list-group-active-bg);border-color:var(--bs-list-group-active-border-color)}.list-group-item+.list-group-item{border-top-width:0}.list-group-item+.list-group-item.active{margin-top:calc(-1 * var(--bs-list-group-border-width));border-top-width:var(--bs-list-group-border-width)}.list-group-item-action{width:100%;color:var(--bs-list-group-action-color);text-align:inherit}.list-group-item-action:not(.active):hover,.list-group-item-action:not(.active):focus{z-index:1;color:var(--bs-list-group-action-hover-color);text-decoration:none;background-color:var(--bs-list-group-action-hover-bg)}.list-group-item-action:not(.active):active{color:var(--bs-list-group-action-active-color);background-color:var(--bs-list-group-action-active-bg)}.list-group-horizontal{flex-direction:row}.list-group-horizontal>.list-group-item:first-child:not(:last-child){border-bottom-left-radius:var(--bs-list-group-border-radius);border-top-right-radius:0}.list-group-horizontal>.list-group-item:last-child:not(:first-child){border-top-right-radius:var(--bs-list-group-border-radius);border-bottom-left-radius:0}.list-group-horizontal>.list-group-item.active{margin-top:0}.list-group-horizontal>.list-group-item+.list-group-item{border-top-width:var(--bs-list-group-border-width);border-left-width:0}.list-group-horizontal>.list-group-item+.list-group-item.active{margin-left:calc(-1 * var(--bs-list-group-border-width));border-left-width:var(--bs-list-group-border-width)}@media (min-width: 576px){.list-group-horizontal-sm{flex-direction:row}.list-group-horizontal-sm>.list-group-item:first-child:not(:last-child){border-bottom-left-radius:var(--bs-list-group-border-radius);border-top-right-radius:0}.list-group-horizontal-sm>.list-group-item:last-child:not(:first-child){border-top-right-radius:var(--bs-list-group-border-radius);border-bottom-left-radius:0}.list-group-horizontal-sm>.list-group-item.active{margin-top:0}.list-group-horizontal-sm>.list-group-item+.list-group-item{border-top-width:var(--bs-list-group-border-width);border-left-width:0}.list-group-horizontal-sm>.list-group-item+.list-group-item.active{margin-left:calc(-1 * var(--bs-list-group-border-width));border-left-width:var(--bs-list-group-border-width)}}@media (min-width: 768px){.list-group-horizontal-md{flex-direction:row}.list-group-horizontal-md>.list-group-item:first-child:not(:last-child){border-bottom-left-radius:var(--bs-list-group-border-radius);border-top-right-radius:0}.list-group-horizontal-md>.list-group-item:last-child:not(:first-child){border-top-right-radius:var(--bs-list-group-border-radius);border-bottom-left-radius:0}.list-group-horizontal-md>.list-group-item.active{margin-top:0}.list-group-horizontal-md>.list-group-item+.list-group-item{border-top-width:var(--bs-list-group-border-width);border-left-width:0}.list-group-horizontal-md>.list-group-item+.list-group-item.active{margin-left:calc(-1 * var(--bs-list-group-border-width));border-left-width:var(--bs-list-group-border-width)}}@media (min-width: 992px){.list-group-horizontal-lg{flex-direction:row}.list-group-horizontal-lg>.list-group-item:first-child:not(:last-child){border-bottom-left-radius:var(--bs-list-group-border-radius);border-top-right-radius:0}.list-group-horizontal-lg>.list-group-item:last-child:not(:first-child){border-top-right-radius:var(--bs-list-group-border-radius);border-bottom-left-radius:0}.list-group-horizontal-lg>.list-group-item.active{margin-top:0}.list-group-horizontal-lg>.list-group-item+.list-group-item{border-top-width:var(--bs-list-group-border-width);border-left-width:0}.list-group-horizontal-lg>.list-group-item+.list-group-item.active{margin-left:calc(-1 * var(--bs-list-group-border-width));border-left-width:var(--bs-list-group-border-width)}}@media (min-width: 1200px){.list-group-horizontal-xl{flex-direction:row}.list-group-horizontal-xl>.list-group-item:first-child:not(:last-child){border-bottom-left-radius:var(--bs-list-group-border-radius);border-top-right-radius:0}.list-group-horizontal-xl>.list-group-item:last-child:not(:first-child){border-top-right-radius:var(--bs-list-group-border-radius);border-bottom-left-radius:0}.list-group-horizontal-xl>.list-group-item.active{margin-top:0}.list-group-horizontal-xl>.list-group-item+.list-group-item{border-top-width:var(--bs-list-group-border-width);border-left-width:0}.list-group-horizontal-xl>.list-group-item+.list-group-item.active{margin-left:calc(-1 * var(--bs-list-group-border-width));border-left-width:var(--bs-list-group-border-width)}}@media (min-width: 1400px){.list-group-horizontal-xxl{flex-direction:row}.list-group-horizontal-xxl>.list-group-item:first-child:not(:last-child){border-bottom-left-radius:var(--bs-list-group-border-radius);border-top-right-radius:0}.list-group-horizontal-xxl>.list-group-item:last-child:not(:first-child){border-top-right-radius:var(--bs-list-group-border-radius);border-bottom-left-radius:0}.list-group-horizontal-xxl>.list-group-item.active{margin-top:0}.list-group-horizontal-xxl>.list-group-item+.list-group-item{border-top-width:var(--bs-list-group-border-width);border-left-width:0}.list-group-horizontal-xxl>.list-group-item+.list-group-item.active{margin-left:calc(-1 * var(--bs-list-group-border-width));border-left-width:var(--bs-list-group-border-width)}}.list-group-flush{border-radius:0}.list-group-flush>.list-group-item{border-width:0 0 var(--bs-list-group-border-width)}.list-group-flush>.list-group-item:last-child{border-bottom-width:0}.list-group-item-primary{--bs-list-group-color: var(--bs-primary-text-emphasis);--bs-list-group-bg: var(--bs-primary-bg-subtle);--bs-list-group-border-color: var(--bs-primary-border-subtle);--bs-list-group-action-hover-color: var(--bs-emphasis-color);--bs-list-group-action-hover-bg: var(--bs-primary-border-subtle);--bs-list-group-action-active-color: var(--bs-emphasis-color);--bs-list-group-action-active-bg: var(--bs-primary-border-subtle);--bs-list-group-active-color: var(--bs-primary-bg-subtle);--bs-list-group-active-bg: var(--bs-primary-text-emphasis);--bs-list-group-active-border-color: var(--bs-primary-text-emphasis)}.list-group-item-secondary{--bs-list-group-color: var(--bs-secondary-text-emphasis);--bs-list-group-bg: var(--bs-secondary-bg-subtle);--bs-list-group-border-color: var(--bs-secondary-border-subtle);--bs-list-group-action-hover-color: var(--bs-emphasis-color);--bs-list-group-action-hover-bg: var(--bs-secondary-border-subtle);--bs-list-group-action-active-color: var(--bs-emphasis-color);--bs-list-group-action-active-bg: var(--bs-secondary-border-subtle);--bs-list-group-active-color: var(--bs-secondary-bg-subtle);--bs-list-group-active-bg: var(--bs-secondary-text-emphasis);--bs-list-group-active-border-color: var(--bs-secondary-text-emphasis)}.list-group-item-success{--bs-list-group-color: var(--bs-success-text-emphasis);--bs-list-group-bg: var(--bs-success-bg-subtle);--bs-list-group-border-color: var(--bs-success-border-subtle);--bs-list-group-action-hover-color: var(--bs-emphasis-color);--bs-list-group-action-hover-bg: var(--bs-success-border-subtle);--bs-list-group-action-active-color: var(--bs-emphasis-color);--bs-list-group-action-active-bg: var(--bs-success-border-subtle);--bs-list-group-active-color: var(--bs-success-bg-subtle);--bs-list-group-active-bg: var(--bs-success-text-emphasis);--bs-list-group-active-border-color: var(--bs-success-text-emphasis)}.list-group-item-info{--bs-list-group-color: var(--bs-info-text-emphasis);--bs-list-group-bg: var(--bs-info-bg-subtle);--bs-list-group-border-color: var(--bs-info-border-subtle);--bs-list-group-action-hover-color: var(--bs-emphasis-color);--bs-list-group-action-hover-bg: var(--bs-info-border-subtle);--bs-list-group-action-active-color: var(--bs-emphasis-color);--bs-list-group-action-active-bg: var(--bs-info-border-subtle);--bs-list-group-active-color: var(--bs-info-bg-subtle);--bs-list-group-active-bg: var(--bs-info-text-emphasis);--bs-list-group-active-border-color: var(--bs-info-text-emphasis)}.list-group-item-warning{--bs-list-group-color: var(--bs-warning-text-emphasis);--bs-list-group-bg: var(--bs-warning-bg-subtle);--bs-list-group-border-color: var(--bs-warning-border-subtle);--bs-list-group-action-hover-color: var(--bs-emphasis-color);--bs-list-group-action-hover-bg: var(--bs-warning-border-subtle);--bs-list-group-action-active-color: var(--bs-emphasis-color);--bs-list-group-action-active-bg: var(--bs-warning-border-subtle);--bs-list-group-active-color: var(--bs-warning-bg-subtle);--bs-list-group-active-bg: var(--bs-warning-text-emphasis);--bs-list-group-active-border-color: var(--bs-warning-text-emphasis)}.list-group-item-danger{--bs-list-group-color: var(--bs-danger-text-emphasis);--bs-list-group-bg: var(--bs-danger-bg-subtle);--bs-list-group-border-color: var(--bs-danger-border-subtle);--bs-list-group-action-hover-color: var(--bs-emphasis-color);--bs-list-group-action-hover-bg: var(--bs-danger-border-subtle);--bs-list-group-action-active-color: var(--bs-emphasis-color);--bs-list-group-action-active-bg: var(--bs-danger-border-subtle);--bs-list-group-active-color: var(--bs-danger-bg-subtle);--bs-list-group-active-bg: var(--bs-danger-text-emphasis);--bs-list-group-active-border-color: var(--bs-danger-text-emphasis)}.list-group-item-light{--bs-list-group-color: var(--bs-light-text-emphasis);--bs-list-group-bg: var(--bs-light-bg-subtle);--bs-list-group-border-color: var(--bs-light-border-subtle);--bs-list-group-action-hover-color: var(--bs-emphasis-color);--bs-list-group-action-hover-bg: var(--bs-light-border-subtle);--bs-list-group-action-active-color: var(--bs-emphasis-color);--bs-list-group-action-active-bg: var(--bs-light-border-subtle);--bs-list-group-active-color: var(--bs-light-bg-subtle);--bs-list-group-active-bg: var(--bs-light-text-emphasis);--bs-list-group-active-border-color: var(--bs-light-text-emphasis)}.list-group-item-dark{--bs-list-group-color: var(--bs-dark-text-emphasis);--bs-list-group-bg: var(--bs-dark-bg-subtle);--bs-list-group-border-color: var(--bs-dark-border-subtle);--bs-list-group-action-hover-color: var(--bs-emphasis-color);--bs-list-group-action-hover-bg: var(--bs-dark-border-subtle);--bs-list-group-action-active-color: var(--bs-emphasis-color);--bs-list-group-action-active-bg: var(--bs-dark-border-subtle);--bs-list-group-active-color: var(--bs-dark-bg-subtle);--bs-list-group-active-bg: var(--bs-dark-text-emphasis);--bs-list-group-active-border-color: var(--bs-dark-text-emphasis)}.btn-close{--bs-btn-close-color: #000;--bs-btn-close-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23000'%3e%3cpath d='M.293.293a1 1 0 0 1 1.414 0L8 6.586 14.293.293a1 1 0 1 1 1.414 1.414L9.414 8l6.293 6.293a1 1 0 0 1-1.414 1.414L8 9.414l-6.293 6.293a1 1 0 0 1-1.414-1.414L6.586 8 .293 1.707a1 1 0 0 1 0-1.414'/%3e%3c/svg%3e");--bs-btn-close-opacity: .5;--bs-btn-close-hover-opacity: .75;--bs-btn-close-focus-shadow: 0 0 0 .25rem rgba(13, 110, 253, .25);--bs-btn-close-focus-opacity: 1;--bs-btn-close-disabled-opacity: .25;box-sizing:content-box;width:1em;height:1em;padding:.25em;color:var(--bs-btn-close-color);background:transparent var(--bs-btn-close-bg) center/1em auto no-repeat;filter:var(--bs-btn-close-filter);border:0;border-radius:.375rem;opacity:var(--bs-btn-close-opacity)}.btn-close:hover{color:var(--bs-btn-close-color);text-decoration:none;opacity:var(--bs-btn-close-hover-opacity)}.btn-close:focus{outline:0;box-shadow:var(--bs-btn-close-focus-shadow);opacity:var(--bs-btn-close-focus-opacity)}.btn-close:disabled,.btn-close.disabled{pointer-events:none;user-select:none;opacity:var(--bs-btn-close-disabled-opacity)}.btn-close-white{--bs-btn-close-filter: invert(1) grayscale(100%) brightness(200%)}:root,[data-bs-theme=light]{--bs-btn-close-filter: }[data-bs-theme=dark]{--bs-btn-close-filter: invert(1) grayscale(100%) brightness(200%)}.toast{--bs-toast-zindex: 1090;--bs-toast-padding-x: .75rem;--bs-toast-padding-y: .5rem;--bs-toast-spacing: 1.5rem;--bs-toast-max-width: 350px;--bs-toast-font-size: .875rem;--bs-toast-color: ;--bs-toast-bg: rgba(var(--bs-body-bg-rgb), .85);--bs-toast-border-width: var(--bs-border-width);--bs-toast-border-color: var(--bs-border-color-translucent);--bs-toast-border-radius: var(--bs-border-radius);--bs-toast-box-shadow: var(--bs-box-shadow);--bs-toast-header-color: var(--bs-secondary-color);--bs-toast-header-bg: rgba(var(--bs-body-bg-rgb), .85);--bs-toast-header-border-color: var(--bs-border-color-translucent);width:var(--bs-toast-max-width);max-width:100%;font-size:var(--bs-toast-font-size);color:var(--bs-toast-color);pointer-events:auto;background-color:var(--bs-toast-bg);background-clip:padding-box;border:var(--bs-toast-border-width) solid var(--bs-toast-border-color);box-shadow:var(--bs-toast-box-shadow);border-radius:var(--bs-toast-border-radius)}.toast.showing{opacity:0}.toast:not(.show){display:none}.toast-container{--bs-toast-zindex: 1090;position:absolute;z-index:var(--bs-toast-zindex);width:max-content;max-width:100%;pointer-events:none}.toast-container>:not(:last-child){margin-bottom:var(--bs-toast-spacing)}.toast-header{display:flex;align-items:center;padding:var(--bs-toast-padding-y) var(--bs-toast-padding-x);color:var(--bs-toast-header-color);background-color:var(--bs-toast-header-bg);background-clip:padding-box;border-bottom:var(--bs-toast-border-width) solid var(--bs-toast-header-border-color);border-top-left-radius:calc(var(--bs-toast-border-radius) - var(--bs-toast-border-width));border-top-right-radius:calc(var(--bs-toast-border-radius) - var(--bs-toast-border-width))}.toast-header .btn-close{margin-right:calc(-.5 * var(--bs-toast-padding-x));margin-left:var(--bs-toast-padding-x)}.toast-body{padding:var(--bs-toast-padding-x);word-wrap:break-word}.modal{--bs-modal-zindex: 1055;--bs-modal-width: 500px;--bs-modal-padding: 1rem;--bs-modal-margin: .5rem;--bs-modal-color: var(--bs-body-color);--bs-modal-bg: var(--bs-body-bg);--bs-modal-border-color: var(--bs-border-color-translucent);--bs-modal-border-width: var(--bs-border-width);--bs-modal-border-radius: var(--bs-border-radius-lg);--bs-modal-box-shadow: var(--bs-box-shadow-sm);--bs-modal-inner-border-radius: calc(var(--bs-border-radius-lg) - (var(--bs-border-width)));--bs-modal-header-padding-x: 1rem;--bs-modal-header-padding-y: 1rem;--bs-modal-header-padding: 1rem 1rem;--bs-modal-header-border-color: var(--bs-border-color);--bs-modal-header-border-width: var(--bs-border-width);--bs-modal-title-line-height: 1.5;--bs-modal-footer-gap: .5rem;--bs-modal-footer-bg: ;--bs-modal-footer-border-color: var(--bs-border-color);--bs-modal-footer-border-width: var(--bs-border-width);position:fixed;top:0;left:0;z-index:var(--bs-modal-zindex);display:none;width:100%;height:100%;overflow-x:hidden;overflow-y:auto;outline:0}.modal-dialog{position:relative;width:auto;margin:var(--bs-modal-margin);pointer-events:none}.modal.fade .modal-dialog{transform:translateY(-50px);transition:transform .3s ease-out}@media (prefers-reduced-motion: reduce){.modal.fade .modal-dialog{transition:none}}.modal.show .modal-dialog{transform:none}.modal.modal-static .modal-dialog{transform:scale(1.02)}.modal-dialog-scrollable{height:calc(100% - var(--bs-modal-margin) * 2)}.modal-dialog-scrollable .modal-content{max-height:100%;overflow:hidden}.modal-dialog-scrollable .modal-body{overflow-y:auto}.modal-dialog-centered{display:flex;align-items:center;min-height:calc(100% - var(--bs-modal-margin) * 2)}.modal-content{position:relative;display:flex;flex-direction:column;width:100%;color:var(--bs-modal-color);pointer-events:auto;background-color:var(--bs-modal-bg);background-clip:padding-box;border:var(--bs-modal-border-width) solid var(--bs-modal-border-color);border-radius:var(--bs-modal-border-radius);outline:0}.modal-backdrop{--bs-backdrop-zindex: 1050;--bs-backdrop-bg: #000;--bs-backdrop-opacity: .5;position:fixed;top:0;left:0;z-index:var(--bs-backdrop-zindex);width:100vw;height:100vh;background-color:var(--bs-backdrop-bg)}.modal-backdrop.fade{opacity:0}.modal-backdrop.show{opacity:var(--bs-backdrop-opacity)}.modal-header{display:flex;flex-shrink:0;align-items:center;padding:var(--bs-modal-header-padding);border-bottom:var(--bs-modal-header-border-width) solid var(--bs-modal-header-border-color);border-top-left-radius:var(--bs-modal-inner-border-radius);border-top-right-radius:var(--bs-modal-inner-border-radius)}.modal-header .btn-close{padding:calc(var(--bs-modal-header-padding-y) * .5) calc(var(--bs-modal-header-padding-x) * .5);margin-top:calc(-.5 * var(--bs-modal-header-padding-y));margin-right:calc(-.5 * var(--bs-modal-header-padding-x));margin-bottom:calc(-.5 * var(--bs-modal-header-padding-y));margin-left:auto}.modal-title{margin-bottom:0;line-height:var(--bs-modal-title-line-height)}.modal-body{position:relative;flex:1 1 auto;padding:var(--bs-modal-padding)}.modal-footer{display:flex;flex-shrink:0;flex-wrap:wrap;align-items:center;justify-content:flex-end;padding:calc(var(--bs-modal-padding) - var(--bs-modal-footer-gap) * .5);background-color:var(--bs-modal-footer-bg);border-top:var(--bs-modal-footer-border-width) solid var(--bs-modal-footer-border-color);border-bottom-right-radius:var(--bs-modal-inner-border-radius);border-bottom-left-radius:var(--bs-modal-inner-border-radius)}.modal-footer>*{margin:calc(var(--bs-modal-footer-gap) * .5)}@media (min-width: 576px){.modal{--bs-modal-margin: 1.75rem;--bs-modal-box-shadow: var(--bs-box-shadow)}.modal-dialog{max-width:var(--bs-modal-width);margin-right:auto;margin-left:auto}.modal-sm{--bs-modal-width: 300px}}@media (min-width: 992px){.modal-lg,.modal-xl{--bs-modal-width: 800px}}@media (min-width: 1200px){.modal-xl{--bs-modal-width: 1140px}}.modal-fullscreen{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen .modal-header,.modal-fullscreen .modal-footer{border-radius:0}.modal-fullscreen .modal-body{overflow-y:auto}@media (max-width: 575.98px){.modal-fullscreen-sm-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-sm-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-sm-down .modal-header,.modal-fullscreen-sm-down .modal-footer{border-radius:0}.modal-fullscreen-sm-down .modal-body{overflow-y:auto}}@media (max-width: 767.98px){.modal-fullscreen-md-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-md-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-md-down .modal-header,.modal-fullscreen-md-down .modal-footer{border-radius:0}.modal-fullscreen-md-down .modal-body{overflow-y:auto}}@media (max-width: 991.98px){.modal-fullscreen-lg-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-lg-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-lg-down .modal-header,.modal-fullscreen-lg-down .modal-footer{border-radius:0}.modal-fullscreen-lg-down .modal-body{overflow-y:auto}}@media (max-width: 1199.98px){.modal-fullscreen-xl-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-xl-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-xl-down .modal-header,.modal-fullscreen-xl-down .modal-footer{border-radius:0}.modal-fullscreen-xl-down .modal-body{overflow-y:auto}}@media (max-width: 1399.98px){.modal-fullscreen-xxl-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-xxl-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-xxl-down .modal-header,.modal-fullscreen-xxl-down .modal-footer{border-radius:0}.modal-fullscreen-xxl-down .modal-body{overflow-y:auto}}.tooltip{--bs-tooltip-zindex: 1080;--bs-tooltip-max-width: 200px;--bs-tooltip-padding-x: .5rem;--bs-tooltip-padding-y: .25rem;--bs-tooltip-margin: ;--bs-tooltip-font-size: .875rem;--bs-tooltip-color: var(--bs-body-bg);--bs-tooltip-bg: var(--bs-emphasis-color);--bs-tooltip-border-radius: var(--bs-border-radius);--bs-tooltip-opacity: .9;--bs-tooltip-arrow-width: .8rem;--bs-tooltip-arrow-height: .4rem;z-index:var(--bs-tooltip-zindex);display:block;margin:var(--bs-tooltip-margin);font-family:var(--bs-font-sans-serif);font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;white-space:normal;word-spacing:normal;line-break:auto;font-size:var(--bs-tooltip-font-size);word-wrap:break-word;opacity:0}.tooltip.show{opacity:var(--bs-tooltip-opacity)}.tooltip .tooltip-arrow{display:block;width:var(--bs-tooltip-arrow-width);height:var(--bs-tooltip-arrow-height)}.tooltip .tooltip-arrow:before{position:absolute;content:"";border-color:transparent;border-style:solid}.bs-tooltip-top .tooltip-arrow,.bs-tooltip-auto[data-popper-placement^=top] .tooltip-arrow{bottom:calc(-1 * var(--bs-tooltip-arrow-height))}.bs-tooltip-top .tooltip-arrow:before,.bs-tooltip-auto[data-popper-placement^=top] .tooltip-arrow:before{top:-1px;border-width:var(--bs-tooltip-arrow-height) calc(var(--bs-tooltip-arrow-width) * .5) 0;border-top-color:var(--bs-tooltip-bg)}.bs-tooltip-end .tooltip-arrow,.bs-tooltip-auto[data-popper-placement^=right] .tooltip-arrow{left:calc(-1 * var(--bs-tooltip-arrow-height));width:var(--bs-tooltip-arrow-height);height:var(--bs-tooltip-arrow-width)}.bs-tooltip-end .tooltip-arrow:before,.bs-tooltip-auto[data-popper-placement^=right] .tooltip-arrow:before{right:-1px;border-width:calc(var(--bs-tooltip-arrow-width) * .5) var(--bs-tooltip-arrow-height) calc(var(--bs-tooltip-arrow-width) * .5) 0;border-right-color:var(--bs-tooltip-bg)}.bs-tooltip-bottom .tooltip-arrow,.bs-tooltip-auto[data-popper-placement^=bottom] .tooltip-arrow{top:calc(-1 * var(--bs-tooltip-arrow-height))}.bs-tooltip-bottom .tooltip-arrow:before,.bs-tooltip-auto[data-popper-placement^=bottom] .tooltip-arrow:before{bottom:-1px;border-width:0 calc(var(--bs-tooltip-arrow-width) * .5) var(--bs-tooltip-arrow-height);border-bottom-color:var(--bs-tooltip-bg)}.bs-tooltip-start .tooltip-arrow,.bs-tooltip-auto[data-popper-placement^=left] .tooltip-arrow{right:calc(-1 * var(--bs-tooltip-arrow-height));width:var(--bs-tooltip-arrow-height);height:var(--bs-tooltip-arrow-width)}.bs-tooltip-start .tooltip-arrow:before,.bs-tooltip-auto[data-popper-placement^=left] .tooltip-arrow:before{left:-1px;border-width:calc(var(--bs-tooltip-arrow-width) * .5) 0 calc(var(--bs-tooltip-arrow-width) * .5) var(--bs-tooltip-arrow-height);border-left-color:var(--bs-tooltip-bg)}.tooltip-inner{max-width:var(--bs-tooltip-max-width);padding:var(--bs-tooltip-padding-y) var(--bs-tooltip-padding-x);color:var(--bs-tooltip-color);text-align:center;background-color:var(--bs-tooltip-bg);border-radius:var(--bs-tooltip-border-radius)}.popover{--bs-popover-zindex: 1070;--bs-popover-max-width: 276px;--bs-popover-font-size: .875rem;--bs-popover-bg: var(--bs-body-bg);--bs-popover-border-width: var(--bs-border-width);--bs-popover-border-color: var(--bs-border-color-translucent);--bs-popover-border-radius: var(--bs-border-radius-lg);--bs-popover-inner-border-radius: calc(var(--bs-border-radius-lg) - var(--bs-border-width));--bs-popover-box-shadow: var(--bs-box-shadow);--bs-popover-header-padding-x: 1rem;--bs-popover-header-padding-y: .5rem;--bs-popover-header-font-size: 1rem;--bs-popover-header-color: inherit;--bs-popover-header-bg: var(--bs-secondary-bg);--bs-popover-body-padding-x: 1rem;--bs-popover-body-padding-y: 1rem;--bs-popover-body-color: var(--bs-body-color);--bs-popover-arrow-width: 1rem;--bs-popover-arrow-height: .5rem;--bs-popover-arrow-border: var(--bs-popover-border-color);z-index:var(--bs-popover-zindex);display:block;max-width:var(--bs-popover-max-width);font-family:var(--bs-font-sans-serif);font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;white-space:normal;word-spacing:normal;line-break:auto;font-size:var(--bs-popover-font-size);word-wrap:break-word;background-color:var(--bs-popover-bg);background-clip:padding-box;border:var(--bs-popover-border-width) solid var(--bs-popover-border-color);border-radius:var(--bs-popover-border-radius)}.popover .popover-arrow{display:block;width:var(--bs-popover-arrow-width);height:var(--bs-popover-arrow-height)}.popover .popover-arrow:before,.popover .popover-arrow:after{position:absolute;display:block;content:"";border-color:transparent;border-style:solid;border-width:0}.bs-popover-top>.popover-arrow,.bs-popover-auto[data-popper-placement^=top]>.popover-arrow{bottom:calc(-1 * (var(--bs-popover-arrow-height)) - var(--bs-popover-border-width))}.bs-popover-top>.popover-arrow:before,.bs-popover-auto[data-popper-placement^=top]>.popover-arrow:before,.bs-popover-top>.popover-arrow:after,.bs-popover-auto[data-popper-placement^=top]>.popover-arrow:after{border-width:var(--bs-popover-arrow-height) calc(var(--bs-popover-arrow-width) * .5) 0}.bs-popover-top>.popover-arrow:before,.bs-popover-auto[data-popper-placement^=top]>.popover-arrow:before{bottom:0;border-top-color:var(--bs-popover-arrow-border)}.bs-popover-top>.popover-arrow:after,.bs-popover-auto[data-popper-placement^=top]>.popover-arrow:after{bottom:var(--bs-popover-border-width);border-top-color:var(--bs-popover-bg)}.bs-popover-end>.popover-arrow,.bs-popover-auto[data-popper-placement^=right]>.popover-arrow{left:calc(-1 * (var(--bs-popover-arrow-height)) - var(--bs-popover-border-width));width:var(--bs-popover-arrow-height);height:var(--bs-popover-arrow-width)}.bs-popover-end>.popover-arrow:before,.bs-popover-auto[data-popper-placement^=right]>.popover-arrow:before,.bs-popover-end>.popover-arrow:after,.bs-popover-auto[data-popper-placement^=right]>.popover-arrow:after{border-width:calc(var(--bs-popover-arrow-width) * .5) var(--bs-popover-arrow-height) calc(var(--bs-popover-arrow-width) * .5) 0}.bs-popover-end>.popover-arrow:before,.bs-popover-auto[data-popper-placement^=right]>.popover-arrow:before{left:0;border-right-color:var(--bs-popover-arrow-border)}.bs-popover-end>.popover-arrow:after,.bs-popover-auto[data-popper-placement^=right]>.popover-arrow:after{left:var(--bs-popover-border-width);border-right-color:var(--bs-popover-bg)}.bs-popover-bottom>.popover-arrow,.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow{top:calc(-1 * (var(--bs-popover-arrow-height)) - var(--bs-popover-border-width))}.bs-popover-bottom>.popover-arrow:before,.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow:before,.bs-popover-bottom>.popover-arrow:after,.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow:after{border-width:0 calc(var(--bs-popover-arrow-width) * .5) var(--bs-popover-arrow-height)}.bs-popover-bottom>.popover-arrow:before,.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow:before{top:0;border-bottom-color:var(--bs-popover-arrow-border)}.bs-popover-bottom>.popover-arrow:after,.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow:after{top:var(--bs-popover-border-width);border-bottom-color:var(--bs-popover-bg)}.bs-popover-bottom .popover-header:before,.bs-popover-auto[data-popper-placement^=bottom] .popover-header:before{position:absolute;top:0;left:50%;display:block;width:var(--bs-popover-arrow-width);margin-left:calc(-.5 * var(--bs-popover-arrow-width));content:"";border-bottom:var(--bs-popover-border-width) solid var(--bs-popover-header-bg)}.bs-popover-start>.popover-arrow,.bs-popover-auto[data-popper-placement^=left]>.popover-arrow{right:calc(-1 * (var(--bs-popover-arrow-height)) - var(--bs-popover-border-width));width:var(--bs-popover-arrow-height);height:var(--bs-popover-arrow-width)}.bs-popover-start>.popover-arrow:before,.bs-popover-auto[data-popper-placement^=left]>.popover-arrow:before,.bs-popover-start>.popover-arrow:after,.bs-popover-auto[data-popper-placement^=left]>.popover-arrow:after{border-width:calc(var(--bs-popover-arrow-width) * .5) 0 calc(var(--bs-popover-arrow-width) * .5) var(--bs-popover-arrow-height)}.bs-popover-start>.popover-arrow:before,.bs-popover-auto[data-popper-placement^=left]>.popover-arrow:before{right:0;border-left-color:var(--bs-popover-arrow-border)}.bs-popover-start>.popover-arrow:after,.bs-popover-auto[data-popper-placement^=left]>.popover-arrow:after{right:var(--bs-popover-border-width);border-left-color:var(--bs-popover-bg)}.popover-header{padding:var(--bs-popover-header-padding-y) var(--bs-popover-header-padding-x);margin-bottom:0;font-size:var(--bs-popover-header-font-size);color:var(--bs-popover-header-color);background-color:var(--bs-popover-header-bg);border-bottom:var(--bs-popover-border-width) solid var(--bs-popover-border-color);border-top-left-radius:var(--bs-popover-inner-border-radius);border-top-right-radius:var(--bs-popover-inner-border-radius)}.popover-header:empty{display:none}.popover-body{padding:var(--bs-popover-body-padding-y) var(--bs-popover-body-padding-x);color:var(--bs-popover-body-color)}.carousel{position:relative}.carousel.pointer-event{touch-action:pan-y}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel-inner:after{display:block;clear:both;content:""}.carousel-item{position:relative;display:none;float:left;width:100%;margin-right:-100%;backface-visibility:hidden;transition:transform .6s ease-in-out}@media (prefers-reduced-motion: reduce){.carousel-item{transition:none}}.carousel-item.active,.carousel-item-next,.carousel-item-prev{display:block}.carousel-item-next:not(.carousel-item-start),.active.carousel-item-end{transform:translate(100%)}.carousel-item-prev:not(.carousel-item-end),.active.carousel-item-start{transform:translate(-100%)}.carousel-fade .carousel-item{opacity:0;transition-property:opacity;transform:none}.carousel-fade .carousel-item.active,.carousel-fade .carousel-item-next.carousel-item-start,.carousel-fade .carousel-item-prev.carousel-item-end{z-index:1;opacity:1}.carousel-fade .active.carousel-item-start,.carousel-fade .active.carousel-item-end{z-index:0;opacity:0;transition:opacity 0s .6s}@media (prefers-reduced-motion: reduce){.carousel-fade .active.carousel-item-start,.carousel-fade .active.carousel-item-end{transition:none}}.carousel-control-prev,.carousel-control-next{position:absolute;top:0;bottom:0;z-index:1;display:flex;align-items:center;justify-content:center;width:15%;padding:0;color:#fff;text-align:center;background:none;filter:var(--bs-carousel-control-icon-filter);border:0;opacity:.5;transition:opacity .15s ease}@media (prefers-reduced-motion: reduce){.carousel-control-prev,.carousel-control-next{transition:none}}.carousel-control-prev:hover,.carousel-control-prev:focus,.carousel-control-next:hover,.carousel-control-next:focus{color:#fff;text-decoration:none;outline:0;opacity:.9}.carousel-control-prev{left:0}.carousel-control-next{right:0}.carousel-control-prev-icon,.carousel-control-next-icon{display:inline-block;width:2rem;height:2rem;background-repeat:no-repeat;background-position:50%;background-size:100% 100%}.carousel-control-prev-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23fff'%3e%3cpath d='M11.354 1.646a.5.5 0 0 1 0 .708L5.707 8l5.647 5.646a.5.5 0 0 1-.708.708l-6-6a.5.5 0 0 1 0-.708l6-6a.5.5 0 0 1 .708 0'/%3e%3c/svg%3e")}.carousel-control-next-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23fff'%3e%3cpath d='M4.646 1.646a.5.5 0 0 1 .708 0l6 6a.5.5 0 0 1 0 .708l-6 6a.5.5 0 0 1-.708-.708L10.293 8 4.646 2.354a.5.5 0 0 1 0-.708'/%3e%3c/svg%3e")}.carousel-indicators{position:absolute;right:0;bottom:0;left:0;z-index:2;display:flex;justify-content:center;padding:0;margin-right:15%;margin-bottom:1rem;margin-left:15%}.carousel-indicators [data-bs-target]{box-sizing:content-box;flex:0 1 auto;width:30px;height:3px;padding:0;margin-right:3px;margin-left:3px;text-indent:-999px;cursor:pointer;background-color:var(--bs-carousel-indicator-active-bg);background-clip:padding-box;border:0;border-top:10px solid transparent;border-bottom:10px solid transparent;opacity:.5;transition:opacity .6s ease}@media (prefers-reduced-motion: reduce){.carousel-indicators [data-bs-target]{transition:none}}.carousel-indicators .active{opacity:1}.carousel-caption{position:absolute;right:15%;bottom:1.25rem;left:15%;padding-top:1.25rem;padding-bottom:1.25rem;color:var(--bs-carousel-caption-color);text-align:center}.carousel-dark{--bs-carousel-indicator-active-bg: #000;--bs-carousel-caption-color: #000;--bs-carousel-control-icon-filter: invert(1) grayscale(100)}:root,[data-bs-theme=light]{--bs-carousel-indicator-active-bg: #fff;--bs-carousel-caption-color: #fff;--bs-carousel-control-icon-filter: }[data-bs-theme=dark]{--bs-carousel-indicator-active-bg: #000;--bs-carousel-caption-color: #000;--bs-carousel-control-icon-filter: invert(1) grayscale(100)}.spinner-grow,.spinner-border{display:inline-block;width:var(--bs-spinner-width);height:var(--bs-spinner-height);vertical-align:var(--bs-spinner-vertical-align);border-radius:50%;animation:var(--bs-spinner-animation-speed) linear infinite var(--bs-spinner-animation-name)}@keyframes spinner-border{to{transform:rotate(360deg)}}.spinner-border{--bs-spinner-width: 2rem;--bs-spinner-height: 2rem;--bs-spinner-vertical-align: -.125em;--bs-spinner-border-width: .25em;--bs-spinner-animation-speed: .75s;--bs-spinner-animation-name: spinner-border;border:var(--bs-spinner-border-width) solid currentcolor;border-right-color:transparent}.spinner-border-sm{--bs-spinner-width: 1rem;--bs-spinner-height: 1rem;--bs-spinner-border-width: .2em}@keyframes spinner-grow{0%{transform:scale(0)}50%{opacity:1;transform:none}}.spinner-grow{--bs-spinner-width: 2rem;--bs-spinner-height: 2rem;--bs-spinner-vertical-align: -.125em;--bs-spinner-animation-speed: .75s;--bs-spinner-animation-name: spinner-grow;background-color:currentcolor;opacity:0}.spinner-grow-sm{--bs-spinner-width: 1rem;--bs-spinner-height: 1rem}@media (prefers-reduced-motion: reduce){.spinner-border,.spinner-grow{--bs-spinner-animation-speed: 1.5s}}.offcanvas,.offcanvas-xxl,.offcanvas-xl,.offcanvas-lg,.offcanvas-md,.offcanvas-sm{--bs-offcanvas-zindex: 1045;--bs-offcanvas-width: 400px;--bs-offcanvas-height: 30vh;--bs-offcanvas-padding-x: 1rem;--bs-offcanvas-padding-y: 1rem;--bs-offcanvas-color: var(--bs-body-color);--bs-offcanvas-bg: var(--bs-body-bg);--bs-offcanvas-border-width: var(--bs-border-width);--bs-offcanvas-border-color: var(--bs-border-color-translucent);--bs-offcanvas-box-shadow: var(--bs-box-shadow-sm);--bs-offcanvas-transition: transform .3s ease-in-out;--bs-offcanvas-title-line-height: 1.5}@media (max-width: 575.98px){.offcanvas-sm{position:fixed;bottom:0;z-index:var(--bs-offcanvas-zindex);display:flex;flex-direction:column;max-width:100%;color:var(--bs-offcanvas-color);visibility:hidden;background-color:var(--bs-offcanvas-bg);background-clip:padding-box;outline:0;transition:var(--bs-offcanvas-transition)}}@media (max-width: 575.98px) and (prefers-reduced-motion: reduce){.offcanvas-sm{transition:none}}@media (max-width: 575.98px){.offcanvas-sm.offcanvas-start{top:0;left:0;width:var(--bs-offcanvas-width);border-right:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translate(-100%)}.offcanvas-sm.offcanvas-end{top:0;right:0;width:var(--bs-offcanvas-width);border-left:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translate(100%)}.offcanvas-sm.offcanvas-top{top:0;right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-bottom:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(-100%)}.offcanvas-sm.offcanvas-bottom{right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-top:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(100%)}.offcanvas-sm.showing,.offcanvas-sm.show:not(.hiding){transform:none}.offcanvas-sm.showing,.offcanvas-sm.hiding,.offcanvas-sm.show{visibility:visible}}@media (min-width: 576px){.offcanvas-sm{--bs-offcanvas-height: auto;--bs-offcanvas-border-width: 0;background-color:transparent!important}.offcanvas-sm .offcanvas-header{display:none}.offcanvas-sm .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible;background-color:transparent!important}}@media (max-width: 767.98px){.offcanvas-md{position:fixed;bottom:0;z-index:var(--bs-offcanvas-zindex);display:flex;flex-direction:column;max-width:100%;color:var(--bs-offcanvas-color);visibility:hidden;background-color:var(--bs-offcanvas-bg);background-clip:padding-box;outline:0;transition:var(--bs-offcanvas-transition)}}@media (max-width: 767.98px) and (prefers-reduced-motion: reduce){.offcanvas-md{transition:none}}@media (max-width: 767.98px){.offcanvas-md.offcanvas-start{top:0;left:0;width:var(--bs-offcanvas-width);border-right:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translate(-100%)}.offcanvas-md.offcanvas-end{top:0;right:0;width:var(--bs-offcanvas-width);border-left:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translate(100%)}.offcanvas-md.offcanvas-top{top:0;right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-bottom:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(-100%)}.offcanvas-md.offcanvas-bottom{right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-top:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(100%)}.offcanvas-md.showing,.offcanvas-md.show:not(.hiding){transform:none}.offcanvas-md.showing,.offcanvas-md.hiding,.offcanvas-md.show{visibility:visible}}@media (min-width: 768px){.offcanvas-md{--bs-offcanvas-height: auto;--bs-offcanvas-border-width: 0;background-color:transparent!important}.offcanvas-md .offcanvas-header{display:none}.offcanvas-md .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible;background-color:transparent!important}}@media (max-width: 991.98px){.offcanvas-lg{position:fixed;bottom:0;z-index:var(--bs-offcanvas-zindex);display:flex;flex-direction:column;max-width:100%;color:var(--bs-offcanvas-color);visibility:hidden;background-color:var(--bs-offcanvas-bg);background-clip:padding-box;outline:0;transition:var(--bs-offcanvas-transition)}}@media (max-width: 991.98px) and (prefers-reduced-motion: reduce){.offcanvas-lg{transition:none}}@media (max-width: 991.98px){.offcanvas-lg.offcanvas-start{top:0;left:0;width:var(--bs-offcanvas-width);border-right:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translate(-100%)}.offcanvas-lg.offcanvas-end{top:0;right:0;width:var(--bs-offcanvas-width);border-left:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translate(100%)}.offcanvas-lg.offcanvas-top{top:0;right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-bottom:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(-100%)}.offcanvas-lg.offcanvas-bottom{right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-top:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(100%)}.offcanvas-lg.showing,.offcanvas-lg.show:not(.hiding){transform:none}.offcanvas-lg.showing,.offcanvas-lg.hiding,.offcanvas-lg.show{visibility:visible}}@media (min-width: 992px){.offcanvas-lg{--bs-offcanvas-height: auto;--bs-offcanvas-border-width: 0;background-color:transparent!important}.offcanvas-lg .offcanvas-header{display:none}.offcanvas-lg .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible;background-color:transparent!important}}@media (max-width: 1199.98px){.offcanvas-xl{position:fixed;bottom:0;z-index:var(--bs-offcanvas-zindex);display:flex;flex-direction:column;max-width:100%;color:var(--bs-offcanvas-color);visibility:hidden;background-color:var(--bs-offcanvas-bg);background-clip:padding-box;outline:0;transition:var(--bs-offcanvas-transition)}}@media (max-width: 1199.98px) and (prefers-reduced-motion: reduce){.offcanvas-xl{transition:none}}@media (max-width: 1199.98px){.offcanvas-xl.offcanvas-start{top:0;left:0;width:var(--bs-offcanvas-width);border-right:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translate(-100%)}.offcanvas-xl.offcanvas-end{top:0;right:0;width:var(--bs-offcanvas-width);border-left:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translate(100%)}.offcanvas-xl.offcanvas-top{top:0;right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-bottom:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(-100%)}.offcanvas-xl.offcanvas-bottom{right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-top:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(100%)}.offcanvas-xl.showing,.offcanvas-xl.show:not(.hiding){transform:none}.offcanvas-xl.showing,.offcanvas-xl.hiding,.offcanvas-xl.show{visibility:visible}}@media (min-width: 1200px){.offcanvas-xl{--bs-offcanvas-height: auto;--bs-offcanvas-border-width: 0;background-color:transparent!important}.offcanvas-xl .offcanvas-header{display:none}.offcanvas-xl .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible;background-color:transparent!important}}@media (max-width: 1399.98px){.offcanvas-xxl{position:fixed;bottom:0;z-index:var(--bs-offcanvas-zindex);display:flex;flex-direction:column;max-width:100%;color:var(--bs-offcanvas-color);visibility:hidden;background-color:var(--bs-offcanvas-bg);background-clip:padding-box;outline:0;transition:var(--bs-offcanvas-transition)}}@media (max-width: 1399.98px) and (prefers-reduced-motion: reduce){.offcanvas-xxl{transition:none}}@media (max-width: 1399.98px){.offcanvas-xxl.offcanvas-start{top:0;left:0;width:var(--bs-offcanvas-width);border-right:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translate(-100%)}.offcanvas-xxl.offcanvas-end{top:0;right:0;width:var(--bs-offcanvas-width);border-left:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translate(100%)}.offcanvas-xxl.offcanvas-top{top:0;right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-bottom:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(-100%)}.offcanvas-xxl.offcanvas-bottom{right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-top:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(100%)}.offcanvas-xxl.showing,.offcanvas-xxl.show:not(.hiding){transform:none}.offcanvas-xxl.showing,.offcanvas-xxl.hiding,.offcanvas-xxl.show{visibility:visible}}@media (min-width: 1400px){.offcanvas-xxl{--bs-offcanvas-height: auto;--bs-offcanvas-border-width: 0;background-color:transparent!important}.offcanvas-xxl .offcanvas-header{display:none}.offcanvas-xxl .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible;background-color:transparent!important}}.offcanvas{position:fixed;bottom:0;z-index:var(--bs-offcanvas-zindex);display:flex;flex-direction:column;max-width:100%;color:var(--bs-offcanvas-color);visibility:hidden;background-color:var(--bs-offcanvas-bg);background-clip:padding-box;outline:0;transition:var(--bs-offcanvas-transition)}@media (prefers-reduced-motion: reduce){.offcanvas{transition:none}}.offcanvas.offcanvas-start{top:0;left:0;width:var(--bs-offcanvas-width);border-right:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translate(-100%)}.offcanvas.offcanvas-end{top:0;right:0;width:var(--bs-offcanvas-width);border-left:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translate(100%)}.offcanvas.offcanvas-top{top:0;right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-bottom:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(-100%)}.offcanvas.offcanvas-bottom{right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-top:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(100%)}.offcanvas.showing,.offcanvas.show:not(.hiding){transform:none}.offcanvas.showing,.offcanvas.hiding,.offcanvas.show{visibility:visible}.offcanvas-backdrop{position:fixed;top:0;left:0;z-index:1040;width:100vw;height:100vh;background-color:#000}.offcanvas-backdrop.fade{opacity:0}.offcanvas-backdrop.show{opacity:.5}.offcanvas-header{display:flex;align-items:center;padding:var(--bs-offcanvas-padding-y) var(--bs-offcanvas-padding-x)}.offcanvas-header .btn-close{padding:calc(var(--bs-offcanvas-padding-y) * .5) calc(var(--bs-offcanvas-padding-x) * .5);margin-top:calc(-.5 * var(--bs-offcanvas-padding-y));margin-right:calc(-.5 * var(--bs-offcanvas-padding-x));margin-bottom:calc(-.5 * var(--bs-offcanvas-padding-y));margin-left:auto}.offcanvas-title{margin-bottom:0;line-height:var(--bs-offcanvas-title-line-height)}.offcanvas-body{flex-grow:1;padding:var(--bs-offcanvas-padding-y) var(--bs-offcanvas-padding-x);overflow-y:auto}.placeholder{display:inline-block;min-height:1em;vertical-align:middle;cursor:wait;background-color:currentcolor;opacity:.5}.placeholder.btn:before{display:inline-block;content:""}.placeholder-xs{min-height:.6em}.placeholder-sm{min-height:.8em}.placeholder-lg{min-height:1.2em}.placeholder-glow .placeholder{animation:placeholder-glow 2s ease-in-out infinite}@keyframes placeholder-glow{50%{opacity:.2}}.placeholder-wave{mask-image:linear-gradient(130deg,#000 55%,#000c,#000 95%);mask-size:200% 100%;animation:placeholder-wave 2s linear infinite}@keyframes placeholder-wave{to{mask-position:-200% 0%}}.clearfix:after{display:block;clear:both;content:""}.text-bg-primary{color:#fff!important;background-color:RGBA(var(--bs-primary-rgb),var(--bs-bg-opacity, 1))!important}.text-bg-secondary{color:#fff!important;background-color:RGBA(var(--bs-secondary-rgb),var(--bs-bg-opacity, 1))!important}.text-bg-success{color:#fff!important;background-color:RGBA(var(--bs-success-rgb),var(--bs-bg-opacity, 1))!important}.text-bg-info{color:#000!important;background-color:RGBA(var(--bs-info-rgb),var(--bs-bg-opacity, 1))!important}.text-bg-warning{color:#000!important;background-color:RGBA(var(--bs-warning-rgb),var(--bs-bg-opacity, 1))!important}.text-bg-danger{color:#fff!important;background-color:RGBA(var(--bs-danger-rgb),var(--bs-bg-opacity, 1))!important}.text-bg-light{color:#000!important;background-color:RGBA(var(--bs-light-rgb),var(--bs-bg-opacity, 1))!important}.text-bg-dark{color:#fff!important;background-color:RGBA(var(--bs-dark-rgb),var(--bs-bg-opacity, 1))!important}.link-primary{color:RGBA(var(--bs-primary-rgb),var(--bs-link-opacity, 1))!important;text-decoration-color:RGBA(var(--bs-primary-rgb),var(--bs-link-underline-opacity, 1))!important}.link-primary:hover,.link-primary:focus{color:RGBA(10,88,202,var(--bs-link-opacity, 1))!important;text-decoration-color:RGBA(10,88,202,var(--bs-link-underline-opacity, 1))!important}.link-secondary{color:RGBA(var(--bs-secondary-rgb),var(--bs-link-opacity, 1))!important;text-decoration-color:RGBA(var(--bs-secondary-rgb),var(--bs-link-underline-opacity, 1))!important}.link-secondary:hover,.link-secondary:focus{color:RGBA(86,94,100,var(--bs-link-opacity, 1))!important;text-decoration-color:RGBA(86,94,100,var(--bs-link-underline-opacity, 1))!important}.link-success{color:RGBA(var(--bs-success-rgb),var(--bs-link-opacity, 1))!important;text-decoration-color:RGBA(var(--bs-success-rgb),var(--bs-link-underline-opacity, 1))!important}.link-success:hover,.link-success:focus{color:RGBA(20,108,67,var(--bs-link-opacity, 1))!important;text-decoration-color:RGBA(20,108,67,var(--bs-link-underline-opacity, 1))!important}.link-info{color:RGBA(var(--bs-info-rgb),var(--bs-link-opacity, 1))!important;text-decoration-color:RGBA(var(--bs-info-rgb),var(--bs-link-underline-opacity, 1))!important}.link-info:hover,.link-info:focus{color:RGBA(61,213,243,var(--bs-link-opacity, 1))!important;text-decoration-color:RGBA(61,213,243,var(--bs-link-underline-opacity, 1))!important}.link-warning{color:RGBA(var(--bs-warning-rgb),var(--bs-link-opacity, 1))!important;text-decoration-color:RGBA(var(--bs-warning-rgb),var(--bs-link-underline-opacity, 1))!important}.link-warning:hover,.link-warning:focus{color:RGBA(255,205,57,var(--bs-link-opacity, 1))!important;text-decoration-color:RGBA(255,205,57,var(--bs-link-underline-opacity, 1))!important}.link-danger{color:RGBA(var(--bs-danger-rgb),var(--bs-link-opacity, 1))!important;text-decoration-color:RGBA(var(--bs-danger-rgb),var(--bs-link-underline-opacity, 1))!important}.link-danger:hover,.link-danger:focus{color:RGBA(176,42,55,var(--bs-link-opacity, 1))!important;text-decoration-color:RGBA(176,42,55,var(--bs-link-underline-opacity, 1))!important}.link-light{color:RGBA(var(--bs-light-rgb),var(--bs-link-opacity, 1))!important;text-decoration-color:RGBA(var(--bs-light-rgb),var(--bs-link-underline-opacity, 1))!important}.link-light:hover,.link-light:focus{color:RGBA(249,250,251,var(--bs-link-opacity, 1))!important;text-decoration-color:RGBA(249,250,251,var(--bs-link-underline-opacity, 1))!important}.link-dark{color:RGBA(var(--bs-dark-rgb),var(--bs-link-opacity, 1))!important;text-decoration-color:RGBA(var(--bs-dark-rgb),var(--bs-link-underline-opacity, 1))!important}.link-dark:hover,.link-dark:focus{color:RGBA(26,30,33,var(--bs-link-opacity, 1))!important;text-decoration-color:RGBA(26,30,33,var(--bs-link-underline-opacity, 1))!important}.link-body-emphasis{color:RGBA(var(--bs-emphasis-color-rgb),var(--bs-link-opacity, 1))!important;text-decoration-color:RGBA(var(--bs-emphasis-color-rgb),var(--bs-link-underline-opacity, 1))!important}.link-body-emphasis:hover,.link-body-emphasis:focus{color:RGBA(var(--bs-emphasis-color-rgb),var(--bs-link-opacity, .75))!important;text-decoration-color:RGBA(var(--bs-emphasis-color-rgb),var(--bs-link-underline-opacity, .75))!important}.focus-ring:focus{outline:0;box-shadow:var(--bs-focus-ring-x, 0) var(--bs-focus-ring-y, 0) var(--bs-focus-ring-blur, 0) var(--bs-focus-ring-width) var(--bs-focus-ring-color)}.icon-link{display:inline-flex;gap:.375rem;align-items:center;text-decoration-color:rgba(var(--bs-link-color-rgb),var(--bs-link-opacity, .5));text-underline-offset:.25em;backface-visibility:hidden}.icon-link>.bi{flex-shrink:0;width:1em;height:1em;fill:currentcolor;transition:.2s ease-in-out transform}@media (prefers-reduced-motion: reduce){.icon-link>.bi{transition:none}}.icon-link-hover:hover>.bi,.icon-link-hover:focus-visible>.bi{transform:var(--bs-icon-link-transform, translate3d(.25em, 0, 0))}.ratio{position:relative;width:100%}.ratio:before{display:block;padding-top:var(--bs-aspect-ratio);content:""}.ratio>*{position:absolute;top:0;left:0;width:100%;height:100%}.ratio-1x1{--bs-aspect-ratio: 100%}.ratio-4x3{--bs-aspect-ratio: 75% }.ratio-16x9{--bs-aspect-ratio: 56.25% }.ratio-21x9{--bs-aspect-ratio: calc(9 / 21 * 100%)}.fixed-top{position:fixed;top:0;right:0;left:0;z-index:1030}.fixed-bottom{position:fixed;right:0;bottom:0;left:0;z-index:1030}.sticky-top{position:sticky;top:0;z-index:1020}.sticky-bottom{position:sticky;bottom:0;z-index:1020}@media (min-width: 576px){.sticky-sm-top{position:sticky;top:0;z-index:1020}.sticky-sm-bottom{position:sticky;bottom:0;z-index:1020}}@media (min-width: 768px){.sticky-md-top{position:sticky;top:0;z-index:1020}.sticky-md-bottom{position:sticky;bottom:0;z-index:1020}}@media (min-width: 992px){.sticky-lg-top{position:sticky;top:0;z-index:1020}.sticky-lg-bottom{position:sticky;bottom:0;z-index:1020}}@media (min-width: 1200px){.sticky-xl-top{position:sticky;top:0;z-index:1020}.sticky-xl-bottom{position:sticky;bottom:0;z-index:1020}}@media (min-width: 1400px){.sticky-xxl-top{position:sticky;top:0;z-index:1020}.sticky-xxl-bottom{position:sticky;bottom:0;z-index:1020}}.hstack{display:flex;flex-direction:row;align-items:center;align-self:stretch}.vstack{display:flex;flex:1 1 auto;flex-direction:column;align-self:stretch}.visually-hidden,.visually-hidden-focusable:not(:focus):not(:focus-within){width:1px!important;height:1px!important;padding:0!important;margin:-1px!important;overflow:hidden!important;clip:rect(0,0,0,0)!important;white-space:nowrap!important;border:0!important}.visually-hidden:not(caption),.visually-hidden-focusable:not(:focus):not(:focus-within):not(caption){position:absolute!important}.stretched-link:after{position:absolute;inset:0;z-index:1;content:""}.text-truncate{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.vr{display:inline-block;align-self:stretch;width:var(--bs-border-width);min-height:1em;background-color:currentcolor;opacity:.25}.align-baseline{vertical-align:baseline!important}.align-top{vertical-align:top!important}.align-middle{vertical-align:middle!important}.align-bottom{vertical-align:bottom!important}.align-text-bottom{vertical-align:text-bottom!important}.align-text-top{vertical-align:text-top!important}.float-start{float:left!important}.float-end{float:right!important}.float-none{float:none!important}.object-fit-contain{object-fit:contain!important}.object-fit-cover{object-fit:cover!important}.object-fit-fill{object-fit:fill!important}.object-fit-scale{object-fit:scale-down!important}.object-fit-none{object-fit:none!important}.opacity-0{opacity:0!important}.opacity-25{opacity:.25!important}.opacity-50{opacity:.5!important}.opacity-75{opacity:.75!important}.opacity-100{opacity:1!important}.overflow-auto{overflow:auto!important}.overflow-hidden{overflow:hidden!important}.overflow-visible{overflow:visible!important}.overflow-scroll{overflow:scroll!important}.overflow-x-auto{overflow-x:auto!important}.overflow-x-hidden{overflow-x:hidden!important}.overflow-x-visible{overflow-x:visible!important}.overflow-x-scroll{overflow-x:scroll!important}.overflow-y-auto{overflow-y:auto!important}.overflow-y-hidden{overflow-y:hidden!important}.overflow-y-visible{overflow-y:visible!important}.overflow-y-scroll{overflow-y:scroll!important}.d-inline{display:inline!important}.d-inline-block{display:inline-block!important}.d-block{display:block!important}.d-grid{display:grid!important}.d-inline-grid{display:inline-grid!important}.d-table{display:table!important}.d-table-row{display:table-row!important}.d-table-cell{display:table-cell!important}.d-flex{display:flex!important}.d-inline-flex{display:inline-flex!important}.d-none{display:none!important}.shadow{box-shadow:var(--bs-box-shadow)!important}.shadow-sm{box-shadow:var(--bs-box-shadow-sm)!important}.shadow-lg{box-shadow:var(--bs-box-shadow-lg)!important}.shadow-none{box-shadow:none!important}.focus-ring-primary{--bs-focus-ring-color: rgba(var(--bs-primary-rgb), var(--bs-focus-ring-opacity))}.focus-ring-secondary{--bs-focus-ring-color: rgba(var(--bs-secondary-rgb), var(--bs-focus-ring-opacity))}.focus-ring-success{--bs-focus-ring-color: rgba(var(--bs-success-rgb), var(--bs-focus-ring-opacity))}.focus-ring-info{--bs-focus-ring-color: rgba(var(--bs-info-rgb), var(--bs-focus-ring-opacity))}.focus-ring-warning{--bs-focus-ring-color: rgba(var(--bs-warning-rgb), var(--bs-focus-ring-opacity))}.focus-ring-danger{--bs-focus-ring-color: rgba(var(--bs-danger-rgb), var(--bs-focus-ring-opacity))}.focus-ring-light{--bs-focus-ring-color: rgba(var(--bs-light-rgb), var(--bs-focus-ring-opacity))}.focus-ring-dark{--bs-focus-ring-color: rgba(var(--bs-dark-rgb), var(--bs-focus-ring-opacity))}.position-static{position:static!important}.position-relative{position:relative!important}.position-absolute{position:absolute!important}.position-fixed{position:fixed!important}.position-sticky{position:sticky!important}.top-0{top:0!important}.top-50{top:50%!important}.top-100{top:100%!important}.bottom-0{bottom:0!important}.bottom-50{bottom:50%!important}.bottom-100{bottom:100%!important}.start-0{left:0!important}.start-50{left:50%!important}.start-100{left:100%!important}.end-0{right:0!important}.end-50{right:50%!important}.end-100{right:100%!important}.translate-middle{transform:translate(-50%,-50%)!important}.translate-middle-x{transform:translate(-50%)!important}.translate-middle-y{transform:translateY(-50%)!important}.border{border:var(--bs-border-width) var(--bs-border-style) var(--bs-border-color)!important}.border-0{border:0!important}.border-top{border-top:var(--bs-border-width) var(--bs-border-style) var(--bs-border-color)!important}.border-top-0{border-top:0!important}.border-end{border-right:var(--bs-border-width) var(--bs-border-style) var(--bs-border-color)!important}.border-end-0{border-right:0!important}.border-bottom{border-bottom:var(--bs-border-width) var(--bs-border-style) var(--bs-border-color)!important}.border-bottom-0{border-bottom:0!important}.border-start{border-left:var(--bs-border-width) var(--bs-border-style) var(--bs-border-color)!important}.border-start-0{border-left:0!important}.border-primary{--bs-border-opacity: 1;border-color:rgba(var(--bs-primary-rgb),var(--bs-border-opacity))!important}.border-secondary{--bs-border-opacity: 1;border-color:rgba(var(--bs-secondary-rgb),var(--bs-border-opacity))!important}.border-success{--bs-border-opacity: 1;border-color:rgba(var(--bs-success-rgb),var(--bs-border-opacity))!important}.border-info{--bs-border-opacity: 1;border-color:rgba(var(--bs-info-rgb),var(--bs-border-opacity))!important}.border-warning{--bs-border-opacity: 1;border-color:rgba(var(--bs-warning-rgb),var(--bs-border-opacity))!important}.border-danger{--bs-border-opacity: 1;border-color:rgba(var(--bs-danger-rgb),var(--bs-border-opacity))!important}.border-light{--bs-border-opacity: 1;border-color:rgba(var(--bs-light-rgb),var(--bs-border-opacity))!important}.border-dark{--bs-border-opacity: 1;border-color:rgba(var(--bs-dark-rgb),var(--bs-border-opacity))!important}.border-black{--bs-border-opacity: 1;border-color:rgba(var(--bs-black-rgb),var(--bs-border-opacity))!important}.border-white{--bs-border-opacity: 1;border-color:rgba(var(--bs-white-rgb),var(--bs-border-opacity))!important}.border-primary-subtle{border-color:var(--bs-primary-border-subtle)!important}.border-secondary-subtle{border-color:var(--bs-secondary-border-subtle)!important}.border-success-subtle{border-color:var(--bs-success-border-subtle)!important}.border-info-subtle{border-color:var(--bs-info-border-subtle)!important}.border-warning-subtle{border-color:var(--bs-warning-border-subtle)!important}.border-danger-subtle{border-color:var(--bs-danger-border-subtle)!important}.border-light-subtle{border-color:var(--bs-light-border-subtle)!important}.border-dark-subtle{border-color:var(--bs-dark-border-subtle)!important}.border-1{border-width:1px!important}.border-2{border-width:2px!important}.border-3{border-width:3px!important}.border-4{border-width:4px!important}.border-5{border-width:5px!important}.border-opacity-10{--bs-border-opacity: .1}.border-opacity-25{--bs-border-opacity: .25}.border-opacity-50{--bs-border-opacity: .5}.border-opacity-75{--bs-border-opacity: .75}.border-opacity-100{--bs-border-opacity: 1}.w-25{width:25%!important}.w-50{width:50%!important}.w-75{width:75%!important}.w-100{width:100%!important}.w-auto{width:auto!important}.mw-100{max-width:100%!important}.vw-100{width:100vw!important}.min-vw-100{min-width:100vw!important}.h-25{height:25%!important}.h-50{height:50%!important}.h-75{height:75%!important}.h-100{height:100%!important}.h-auto{height:auto!important}.mh-100{max-height:100%!important}.vh-100{height:100vh!important}.min-vh-100{min-height:100vh!important}.flex-fill{flex:1 1 auto!important}.flex-row{flex-direction:row!important}.flex-column{flex-direction:column!important}.flex-row-reverse{flex-direction:row-reverse!important}.flex-column-reverse{flex-direction:column-reverse!important}.flex-grow-0{flex-grow:0!important}.flex-grow-1{flex-grow:1!important}.flex-shrink-0{flex-shrink:0!important}.flex-shrink-1{flex-shrink:1!important}.flex-wrap{flex-wrap:wrap!important}.flex-nowrap{flex-wrap:nowrap!important}.flex-wrap-reverse{flex-wrap:wrap-reverse!important}.justify-content-start{justify-content:flex-start!important}.justify-content-end{justify-content:flex-end!important}.justify-content-center{justify-content:center!important}.justify-content-between{justify-content:space-between!important}.justify-content-around{justify-content:space-around!important}.justify-content-evenly{justify-content:space-evenly!important}.align-items-start{align-items:flex-start!important}.align-items-end{align-items:flex-end!important}.align-items-center{align-items:center!important}.align-items-baseline{align-items:baseline!important}.align-items-stretch{align-items:stretch!important}.align-content-start{align-content:flex-start!important}.align-content-end{align-content:flex-end!important}.align-content-center{align-content:center!important}.align-content-between{align-content:space-between!important}.align-content-around{align-content:space-around!important}.align-content-stretch{align-content:stretch!important}.align-self-auto{align-self:auto!important}.align-self-start{align-self:flex-start!important}.align-self-end{align-self:flex-end!important}.align-self-center{align-self:center!important}.align-self-baseline{align-self:baseline!important}.align-self-stretch{align-self:stretch!important}.order-first{order:-1!important}.order-0{order:0!important}.order-1{order:1!important}.order-2{order:2!important}.order-3{order:3!important}.order-4{order:4!important}.order-5{order:5!important}.order-last{order:6!important}.m-0{margin:0!important}.m-1{margin:.25rem!important}.m-2{margin:.5rem!important}.m-3{margin:1rem!important}.m-4{margin:1.5rem!important}.m-5{margin:3rem!important}.m-auto{margin:auto!important}.mx-0{margin-right:0!important;margin-left:0!important}.mx-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-3{margin-right:1rem!important;margin-left:1rem!important}.mx-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-5{margin-right:3rem!important;margin-left:3rem!important}.mx-auto{margin-right:auto!important;margin-left:auto!important}.my-0{margin-top:0!important;margin-bottom:0!important}.my-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-0{margin-top:0!important}.mt-1{margin-top:.25rem!important}.mt-2{margin-top:.5rem!important}.mt-3{margin-top:1rem!important}.mt-4{margin-top:1.5rem!important}.mt-5{margin-top:3rem!important}.mt-auto{margin-top:auto!important}.me-0{margin-right:0!important}.me-1{margin-right:.25rem!important}.me-2{margin-right:.5rem!important}.me-3{margin-right:1rem!important}.me-4{margin-right:1.5rem!important}.me-5{margin-right:3rem!important}.me-auto{margin-right:auto!important}.mb-0{margin-bottom:0!important}.mb-1{margin-bottom:.25rem!important}.mb-2{margin-bottom:.5rem!important}.mb-3{margin-bottom:1rem!important}.mb-4{margin-bottom:1.5rem!important}.mb-5{margin-bottom:3rem!important}.mb-auto{margin-bottom:auto!important}.ms-0{margin-left:0!important}.ms-1{margin-left:.25rem!important}.ms-2{margin-left:.5rem!important}.ms-3{margin-left:1rem!important}.ms-4{margin-left:1.5rem!important}.ms-5{margin-left:3rem!important}.ms-auto{margin-left:auto!important}.p-0{padding:0!important}.p-1{padding:.25rem!important}.p-2{padding:.5rem!important}.p-3{padding:1rem!important}.p-4{padding:1.5rem!important}.p-5{padding:3rem!important}.px-0{padding-right:0!important;padding-left:0!important}.px-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-3{padding-right:1rem!important;padding-left:1rem!important}.px-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-5{padding-right:3rem!important;padding-left:3rem!important}.py-0{padding-top:0!important;padding-bottom:0!important}.py-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-0{padding-top:0!important}.pt-1{padding-top:.25rem!important}.pt-2{padding-top:.5rem!important}.pt-3{padding-top:1rem!important}.pt-4{padding-top:1.5rem!important}.pt-5{padding-top:3rem!important}.pe-0{padding-right:0!important}.pe-1{padding-right:.25rem!important}.pe-2{padding-right:.5rem!important}.pe-3{padding-right:1rem!important}.pe-4{padding-right:1.5rem!important}.pe-5{padding-right:3rem!important}.pb-0{padding-bottom:0!important}.pb-1{padding-bottom:.25rem!important}.pb-2{padding-bottom:.5rem!important}.pb-3{padding-bottom:1rem!important}.pb-4{padding-bottom:1.5rem!important}.pb-5{padding-bottom:3rem!important}.ps-0{padding-left:0!important}.ps-1{padding-left:.25rem!important}.ps-2{padding-left:.5rem!important}.ps-3{padding-left:1rem!important}.ps-4{padding-left:1.5rem!important}.ps-5{padding-left:3rem!important}.gap-0{gap:0!important}.gap-1{gap:.25rem!important}.gap-2{gap:.5rem!important}.gap-3{gap:1rem!important}.gap-4{gap:1.5rem!important}.gap-5{gap:3rem!important}.row-gap-0{row-gap:0!important}.row-gap-1{row-gap:.25rem!important}.row-gap-2{row-gap:.5rem!important}.row-gap-3{row-gap:1rem!important}.row-gap-4{row-gap:1.5rem!important}.row-gap-5{row-gap:3rem!important}.column-gap-0{column-gap:0!important}.column-gap-1{column-gap:.25rem!important}.column-gap-2{column-gap:.5rem!important}.column-gap-3{column-gap:1rem!important}.column-gap-4{column-gap:1.5rem!important}.column-gap-5{column-gap:3rem!important}.font-monospace{font-family:var(--bs-font-monospace)!important}.fs-1{font-size:calc(1.375rem + 1.5vw)!important}.fs-2{font-size:calc(1.325rem + .9vw)!important}.fs-3{font-size:calc(1.3rem + .6vw)!important}.fs-4{font-size:calc(1.275rem + .3vw)!important}.fs-5{font-size:1.25rem!important}.fs-6{font-size:1rem!important}.fst-italic{font-style:italic!important}.fst-normal{font-style:normal!important}.fw-lighter{font-weight:lighter!important}.fw-light{font-weight:300!important}.fw-normal{font-weight:400!important}.fw-medium{font-weight:500!important}.fw-semibold{font-weight:600!important}.fw-bold{font-weight:700!important}.fw-bolder{font-weight:bolder!important}.lh-1{line-height:1!important}.lh-sm{line-height:1.25!important}.lh-base{line-height:1.5!important}.lh-lg{line-height:2!important}.text-start{text-align:left!important}.text-end{text-align:right!important}.text-center{text-align:center!important}.text-decoration-none{text-decoration:none!important}.text-decoration-underline{text-decoration:underline!important}.text-decoration-line-through{text-decoration:line-through!important}.text-lowercase{text-transform:lowercase!important}.text-uppercase{text-transform:uppercase!important}.text-capitalize{text-transform:capitalize!important}.text-wrap{white-space:normal!important}.text-nowrap{white-space:nowrap!important}.text-break{word-wrap:break-word!important;word-break:break-word!important}.text-primary{--bs-text-opacity: 1;color:rgba(var(--bs-primary-rgb),var(--bs-text-opacity))!important}.text-secondary{--bs-text-opacity: 1;color:rgba(var(--bs-secondary-rgb),var(--bs-text-opacity))!important}.text-success{--bs-text-opacity: 1;color:rgba(var(--bs-success-rgb),var(--bs-text-opacity))!important}.text-info{--bs-text-opacity: 1;color:rgba(var(--bs-info-rgb),var(--bs-text-opacity))!important}.text-warning{--bs-text-opacity: 1;color:rgba(var(--bs-warning-rgb),var(--bs-text-opacity))!important}.text-danger{--bs-text-opacity: 1;color:rgba(var(--bs-danger-rgb),var(--bs-text-opacity))!important}.text-light{--bs-text-opacity: 1;color:rgba(var(--bs-light-rgb),var(--bs-text-opacity))!important}.text-dark{--bs-text-opacity: 1;color:rgba(var(--bs-dark-rgb),var(--bs-text-opacity))!important}.text-black{--bs-text-opacity: 1;color:rgba(var(--bs-black-rgb),var(--bs-text-opacity))!important}.text-white{--bs-text-opacity: 1;color:rgba(var(--bs-white-rgb),var(--bs-text-opacity))!important}.text-body{--bs-text-opacity: 1;color:rgba(var(--bs-body-color-rgb),var(--bs-text-opacity))!important}.text-muted{--bs-text-opacity: 1;color:var(--bs-secondary-color)!important}.text-black-50{--bs-text-opacity: 1;color:#00000080!important}.text-white-50{--bs-text-opacity: 1;color:#ffffff80!important}.text-body-secondary{--bs-text-opacity: 1;color:var(--bs-secondary-color)!important}.text-body-tertiary{--bs-text-opacity: 1;color:var(--bs-tertiary-color)!important}.text-body-emphasis{--bs-text-opacity: 1;color:var(--bs-emphasis-color)!important}.text-reset{--bs-text-opacity: 1;color:inherit!important}.text-opacity-25{--bs-text-opacity: .25}.text-opacity-50{--bs-text-opacity: .5}.text-opacity-75{--bs-text-opacity: .75}.text-opacity-100{--bs-text-opacity: 1}.text-primary-emphasis{color:var(--bs-primary-text-emphasis)!important}.text-secondary-emphasis{color:var(--bs-secondary-text-emphasis)!important}.text-success-emphasis{color:var(--bs-success-text-emphasis)!important}.text-info-emphasis{color:var(--bs-info-text-emphasis)!important}.text-warning-emphasis{color:var(--bs-warning-text-emphasis)!important}.text-danger-emphasis{color:var(--bs-danger-text-emphasis)!important}.text-light-emphasis{color:var(--bs-light-text-emphasis)!important}.text-dark-emphasis{color:var(--bs-dark-text-emphasis)!important}.link-opacity-10,.link-opacity-10-hover:hover{--bs-link-opacity: .1}.link-opacity-25,.link-opacity-25-hover:hover{--bs-link-opacity: .25}.link-opacity-50,.link-opacity-50-hover:hover{--bs-link-opacity: .5}.link-opacity-75,.link-opacity-75-hover:hover{--bs-link-opacity: .75}.link-opacity-100,.link-opacity-100-hover:hover{--bs-link-opacity: 1}.link-offset-1,.link-offset-1-hover:hover{text-underline-offset:.125em!important}.link-offset-2,.link-offset-2-hover:hover{text-underline-offset:.25em!important}.link-offset-3,.link-offset-3-hover:hover{text-underline-offset:.375em!important}.link-underline-primary{--bs-link-underline-opacity: 1;text-decoration-color:rgba(var(--bs-primary-rgb),var(--bs-link-underline-opacity))!important}.link-underline-secondary{--bs-link-underline-opacity: 1;text-decoration-color:rgba(var(--bs-secondary-rgb),var(--bs-link-underline-opacity))!important}.link-underline-success{--bs-link-underline-opacity: 1;text-decoration-color:rgba(var(--bs-success-rgb),var(--bs-link-underline-opacity))!important}.link-underline-info{--bs-link-underline-opacity: 1;text-decoration-color:rgba(var(--bs-info-rgb),var(--bs-link-underline-opacity))!important}.link-underline-warning{--bs-link-underline-opacity: 1;text-decoration-color:rgba(var(--bs-warning-rgb),var(--bs-link-underline-opacity))!important}.link-underline-danger{--bs-link-underline-opacity: 1;text-decoration-color:rgba(var(--bs-danger-rgb),var(--bs-link-underline-opacity))!important}.link-underline-light{--bs-link-underline-opacity: 1;text-decoration-color:rgba(var(--bs-light-rgb),var(--bs-link-underline-opacity))!important}.link-underline-dark{--bs-link-underline-opacity: 1;text-decoration-color:rgba(var(--bs-dark-rgb),var(--bs-link-underline-opacity))!important}.link-underline{--bs-link-underline-opacity: 1;text-decoration-color:rgba(var(--bs-link-color-rgb),var(--bs-link-underline-opacity, 1))!important}.link-underline-opacity-0,.link-underline-opacity-0-hover:hover{--bs-link-underline-opacity: 0}.link-underline-opacity-10,.link-underline-opacity-10-hover:hover{--bs-link-underline-opacity: .1}.link-underline-opacity-25,.link-underline-opacity-25-hover:hover{--bs-link-underline-opacity: .25}.link-underline-opacity-50,.link-underline-opacity-50-hover:hover{--bs-link-underline-opacity: .5}.link-underline-opacity-75,.link-underline-opacity-75-hover:hover{--bs-link-underline-opacity: .75}.link-underline-opacity-100,.link-underline-opacity-100-hover:hover{--bs-link-underline-opacity: 1}.bg-primary{--bs-bg-opacity: 1;background-color:rgba(var(--bs-primary-rgb),var(--bs-bg-opacity))!important}.bg-secondary{--bs-bg-opacity: 1;background-color:rgba(var(--bs-secondary-rgb),var(--bs-bg-opacity))!important}.bg-success{--bs-bg-opacity: 1;background-color:rgba(var(--bs-success-rgb),var(--bs-bg-opacity))!important}.bg-info{--bs-bg-opacity: 1;background-color:rgba(var(--bs-info-rgb),var(--bs-bg-opacity))!important}.bg-warning{--bs-bg-opacity: 1;background-color:rgba(var(--bs-warning-rgb),var(--bs-bg-opacity))!important}.bg-danger{--bs-bg-opacity: 1;background-color:rgba(var(--bs-danger-rgb),var(--bs-bg-opacity))!important}.bg-light{--bs-bg-opacity: 1;background-color:rgba(var(--bs-light-rgb),var(--bs-bg-opacity))!important}.bg-dark{--bs-bg-opacity: 1;background-color:rgba(var(--bs-dark-rgb),var(--bs-bg-opacity))!important}.bg-black{--bs-bg-opacity: 1;background-color:rgba(var(--bs-black-rgb),var(--bs-bg-opacity))!important}.bg-white{--bs-bg-opacity: 1;background-color:rgba(var(--bs-white-rgb),var(--bs-bg-opacity))!important}.bg-body{--bs-bg-opacity: 1;background-color:rgba(var(--bs-body-bg-rgb),var(--bs-bg-opacity))!important}.bg-transparent{--bs-bg-opacity: 1;background-color:transparent!important}.bg-body-secondary{--bs-bg-opacity: 1;background-color:rgba(var(--bs-secondary-bg-rgb),var(--bs-bg-opacity))!important}.bg-body-tertiary{--bs-bg-opacity: 1;background-color:rgba(var(--bs-tertiary-bg-rgb),var(--bs-bg-opacity))!important}.bg-opacity-10{--bs-bg-opacity: .1}.bg-opacity-25{--bs-bg-opacity: .25}.bg-opacity-50{--bs-bg-opacity: .5}.bg-opacity-75{--bs-bg-opacity: .75}.bg-opacity-100{--bs-bg-opacity: 1}.bg-primary-subtle{background-color:var(--bs-primary-bg-subtle)!important}.bg-secondary-subtle{background-color:var(--bs-secondary-bg-subtle)!important}.bg-success-subtle{background-color:var(--bs-success-bg-subtle)!important}.bg-info-subtle{background-color:var(--bs-info-bg-subtle)!important}.bg-warning-subtle{background-color:var(--bs-warning-bg-subtle)!important}.bg-danger-subtle{background-color:var(--bs-danger-bg-subtle)!important}.bg-light-subtle{background-color:var(--bs-light-bg-subtle)!important}.bg-dark-subtle{background-color:var(--bs-dark-bg-subtle)!important}.bg-gradient{background-image:var(--bs-gradient)!important}.user-select-all{user-select:all!important}.user-select-auto{user-select:auto!important}.user-select-none{user-select:none!important}.pe-none{pointer-events:none!important}.pe-auto{pointer-events:auto!important}.rounded{border-radius:var(--bs-border-radius)!important}.rounded-0{border-radius:0!important}.rounded-1{border-radius:var(--bs-border-radius-sm)!important}.rounded-2{border-radius:var(--bs-border-radius)!important}.rounded-3{border-radius:var(--bs-border-radius-lg)!important}.rounded-4{border-radius:var(--bs-border-radius-xl)!important}.rounded-5{border-radius:var(--bs-border-radius-xxl)!important}.rounded-circle{border-radius:50%!important}.rounded-pill{border-radius:var(--bs-border-radius-pill)!important}.rounded-top{border-top-left-radius:var(--bs-border-radius)!important;border-top-right-radius:var(--bs-border-radius)!important}.rounded-top-0{border-top-left-radius:0!important;border-top-right-radius:0!important}.rounded-top-1{border-top-left-radius:var(--bs-border-radius-sm)!important;border-top-right-radius:var(--bs-border-radius-sm)!important}.rounded-top-2{border-top-left-radius:var(--bs-border-radius)!important;border-top-right-radius:var(--bs-border-radius)!important}.rounded-top-3{border-top-left-radius:var(--bs-border-radius-lg)!important;border-top-right-radius:var(--bs-border-radius-lg)!important}.rounded-top-4{border-top-left-radius:var(--bs-border-radius-xl)!important;border-top-right-radius:var(--bs-border-radius-xl)!important}.rounded-top-5{border-top-left-radius:var(--bs-border-radius-xxl)!important;border-top-right-radius:var(--bs-border-radius-xxl)!important}.rounded-top-circle{border-top-left-radius:50%!important;border-top-right-radius:50%!important}.rounded-top-pill{border-top-left-radius:var(--bs-border-radius-pill)!important;border-top-right-radius:var(--bs-border-radius-pill)!important}.rounded-end{border-top-right-radius:var(--bs-border-radius)!important;border-bottom-right-radius:var(--bs-border-radius)!important}.rounded-end-0{border-top-right-radius:0!important;border-bottom-right-radius:0!important}.rounded-end-1{border-top-right-radius:var(--bs-border-radius-sm)!important;border-bottom-right-radius:var(--bs-border-radius-sm)!important}.rounded-end-2{border-top-right-radius:var(--bs-border-radius)!important;border-bottom-right-radius:var(--bs-border-radius)!important}.rounded-end-3{border-top-right-radius:var(--bs-border-radius-lg)!important;border-bottom-right-radius:var(--bs-border-radius-lg)!important}.rounded-end-4{border-top-right-radius:var(--bs-border-radius-xl)!important;border-bottom-right-radius:var(--bs-border-radius-xl)!important}.rounded-end-5{border-top-right-radius:var(--bs-border-radius-xxl)!important;border-bottom-right-radius:var(--bs-border-radius-xxl)!important}.rounded-end-circle{border-top-right-radius:50%!important;border-bottom-right-radius:50%!important}.rounded-end-pill{border-top-right-radius:var(--bs-border-radius-pill)!important;border-bottom-right-radius:var(--bs-border-radius-pill)!important}.rounded-bottom{border-bottom-right-radius:var(--bs-border-radius)!important;border-bottom-left-radius:var(--bs-border-radius)!important}.rounded-bottom-0{border-bottom-right-radius:0!important;border-bottom-left-radius:0!important}.rounded-bottom-1{border-bottom-right-radius:var(--bs-border-radius-sm)!important;border-bottom-left-radius:var(--bs-border-radius-sm)!important}.rounded-bottom-2{border-bottom-right-radius:var(--bs-border-radius)!important;border-bottom-left-radius:var(--bs-border-radius)!important}.rounded-bottom-3{border-bottom-right-radius:var(--bs-border-radius-lg)!important;border-bottom-left-radius:var(--bs-border-radius-lg)!important}.rounded-bottom-4{border-bottom-right-radius:var(--bs-border-radius-xl)!important;border-bottom-left-radius:var(--bs-border-radius-xl)!important}.rounded-bottom-5{border-bottom-right-radius:var(--bs-border-radius-xxl)!important;border-bottom-left-radius:var(--bs-border-radius-xxl)!important}.rounded-bottom-circle{border-bottom-right-radius:50%!important;border-bottom-left-radius:50%!important}.rounded-bottom-pill{border-bottom-right-radius:var(--bs-border-radius-pill)!important;border-bottom-left-radius:var(--bs-border-radius-pill)!important}.rounded-start{border-bottom-left-radius:var(--bs-border-radius)!important;border-top-left-radius:var(--bs-border-radius)!important}.rounded-start-0{border-bottom-left-radius:0!important;border-top-left-radius:0!important}.rounded-start-1{border-bottom-left-radius:var(--bs-border-radius-sm)!important;border-top-left-radius:var(--bs-border-radius-sm)!important}.rounded-start-2{border-bottom-left-radius:var(--bs-border-radius)!important;border-top-left-radius:var(--bs-border-radius)!important}.rounded-start-3{border-bottom-left-radius:var(--bs-border-radius-lg)!important;border-top-left-radius:var(--bs-border-radius-lg)!important}.rounded-start-4{border-bottom-left-radius:var(--bs-border-radius-xl)!important;border-top-left-radius:var(--bs-border-radius-xl)!important}.rounded-start-5{border-bottom-left-radius:var(--bs-border-radius-xxl)!important;border-top-left-radius:var(--bs-border-radius-xxl)!important}.rounded-start-circle{border-bottom-left-radius:50%!important;border-top-left-radius:50%!important}.rounded-start-pill{border-bottom-left-radius:var(--bs-border-radius-pill)!important;border-top-left-radius:var(--bs-border-radius-pill)!important}.visible{visibility:visible!important}.invisible{visibility:hidden!important}.z-n1{z-index:-1!important}.z-0{z-index:0!important}.z-1{z-index:1!important}.z-2{z-index:2!important}.z-3{z-index:3!important}@media (min-width: 576px){.float-sm-start{float:left!important}.float-sm-end{float:right!important}.float-sm-none{float:none!important}.object-fit-sm-contain{object-fit:contain!important}.object-fit-sm-cover{object-fit:cover!important}.object-fit-sm-fill{object-fit:fill!important}.object-fit-sm-scale{object-fit:scale-down!important}.object-fit-sm-none{object-fit:none!important}.d-sm-inline{display:inline!important}.d-sm-inline-block{display:inline-block!important}.d-sm-block{display:block!important}.d-sm-grid{display:grid!important}.d-sm-inline-grid{display:inline-grid!important}.d-sm-table{display:table!important}.d-sm-table-row{display:table-row!important}.d-sm-table-cell{display:table-cell!important}.d-sm-flex{display:flex!important}.d-sm-inline-flex{display:inline-flex!important}.d-sm-none{display:none!important}.flex-sm-fill{flex:1 1 auto!important}.flex-sm-row{flex-direction:row!important}.flex-sm-column{flex-direction:column!important}.flex-sm-row-reverse{flex-direction:row-reverse!important}.flex-sm-column-reverse{flex-direction:column-reverse!important}.flex-sm-grow-0{flex-grow:0!important}.flex-sm-grow-1{flex-grow:1!important}.flex-sm-shrink-0{flex-shrink:0!important}.flex-sm-shrink-1{flex-shrink:1!important}.flex-sm-wrap{flex-wrap:wrap!important}.flex-sm-nowrap{flex-wrap:nowrap!important}.flex-sm-wrap-reverse{flex-wrap:wrap-reverse!important}.justify-content-sm-start{justify-content:flex-start!important}.justify-content-sm-end{justify-content:flex-end!important}.justify-content-sm-center{justify-content:center!important}.justify-content-sm-between{justify-content:space-between!important}.justify-content-sm-around{justify-content:space-around!important}.justify-content-sm-evenly{justify-content:space-evenly!important}.align-items-sm-start{align-items:flex-start!important}.align-items-sm-end{align-items:flex-end!important}.align-items-sm-center{align-items:center!important}.align-items-sm-baseline{align-items:baseline!important}.align-items-sm-stretch{align-items:stretch!important}.align-content-sm-start{align-content:flex-start!important}.align-content-sm-end{align-content:flex-end!important}.align-content-sm-center{align-content:center!important}.align-content-sm-between{align-content:space-between!important}.align-content-sm-around{align-content:space-around!important}.align-content-sm-stretch{align-content:stretch!important}.align-self-sm-auto{align-self:auto!important}.align-self-sm-start{align-self:flex-start!important}.align-self-sm-end{align-self:flex-end!important}.align-self-sm-center{align-self:center!important}.align-self-sm-baseline{align-self:baseline!important}.align-self-sm-stretch{align-self:stretch!important}.order-sm-first{order:-1!important}.order-sm-0{order:0!important}.order-sm-1{order:1!important}.order-sm-2{order:2!important}.order-sm-3{order:3!important}.order-sm-4{order:4!important}.order-sm-5{order:5!important}.order-sm-last{order:6!important}.m-sm-0{margin:0!important}.m-sm-1{margin:.25rem!important}.m-sm-2{margin:.5rem!important}.m-sm-3{margin:1rem!important}.m-sm-4{margin:1.5rem!important}.m-sm-5{margin:3rem!important}.m-sm-auto{margin:auto!important}.mx-sm-0{margin-right:0!important;margin-left:0!important}.mx-sm-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-sm-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-sm-3{margin-right:1rem!important;margin-left:1rem!important}.mx-sm-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-sm-5{margin-right:3rem!important;margin-left:3rem!important}.mx-sm-auto{margin-right:auto!important;margin-left:auto!important}.my-sm-0{margin-top:0!important;margin-bottom:0!important}.my-sm-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-sm-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-sm-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-sm-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-sm-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-sm-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-sm-0{margin-top:0!important}.mt-sm-1{margin-top:.25rem!important}.mt-sm-2{margin-top:.5rem!important}.mt-sm-3{margin-top:1rem!important}.mt-sm-4{margin-top:1.5rem!important}.mt-sm-5{margin-top:3rem!important}.mt-sm-auto{margin-top:auto!important}.me-sm-0{margin-right:0!important}.me-sm-1{margin-right:.25rem!important}.me-sm-2{margin-right:.5rem!important}.me-sm-3{margin-right:1rem!important}.me-sm-4{margin-right:1.5rem!important}.me-sm-5{margin-right:3rem!important}.me-sm-auto{margin-right:auto!important}.mb-sm-0{margin-bottom:0!important}.mb-sm-1{margin-bottom:.25rem!important}.mb-sm-2{margin-bottom:.5rem!important}.mb-sm-3{margin-bottom:1rem!important}.mb-sm-4{margin-bottom:1.5rem!important}.mb-sm-5{margin-bottom:3rem!important}.mb-sm-auto{margin-bottom:auto!important}.ms-sm-0{margin-left:0!important}.ms-sm-1{margin-left:.25rem!important}.ms-sm-2{margin-left:.5rem!important}.ms-sm-3{margin-left:1rem!important}.ms-sm-4{margin-left:1.5rem!important}.ms-sm-5{margin-left:3rem!important}.ms-sm-auto{margin-left:auto!important}.p-sm-0{padding:0!important}.p-sm-1{padding:.25rem!important}.p-sm-2{padding:.5rem!important}.p-sm-3{padding:1rem!important}.p-sm-4{padding:1.5rem!important}.p-sm-5{padding:3rem!important}.px-sm-0{padding-right:0!important;padding-left:0!important}.px-sm-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-sm-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-sm-3{padding-right:1rem!important;padding-left:1rem!important}.px-sm-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-sm-5{padding-right:3rem!important;padding-left:3rem!important}.py-sm-0{padding-top:0!important;padding-bottom:0!important}.py-sm-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-sm-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-sm-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-sm-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-sm-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-sm-0{padding-top:0!important}.pt-sm-1{padding-top:.25rem!important}.pt-sm-2{padding-top:.5rem!important}.pt-sm-3{padding-top:1rem!important}.pt-sm-4{padding-top:1.5rem!important}.pt-sm-5{padding-top:3rem!important}.pe-sm-0{padding-right:0!important}.pe-sm-1{padding-right:.25rem!important}.pe-sm-2{padding-right:.5rem!important}.pe-sm-3{padding-right:1rem!important}.pe-sm-4{padding-right:1.5rem!important}.pe-sm-5{padding-right:3rem!important}.pb-sm-0{padding-bottom:0!important}.pb-sm-1{padding-bottom:.25rem!important}.pb-sm-2{padding-bottom:.5rem!important}.pb-sm-3{padding-bottom:1rem!important}.pb-sm-4{padding-bottom:1.5rem!important}.pb-sm-5{padding-bottom:3rem!important}.ps-sm-0{padding-left:0!important}.ps-sm-1{padding-left:.25rem!important}.ps-sm-2{padding-left:.5rem!important}.ps-sm-3{padding-left:1rem!important}.ps-sm-4{padding-left:1.5rem!important}.ps-sm-5{padding-left:3rem!important}.gap-sm-0{gap:0!important}.gap-sm-1{gap:.25rem!important}.gap-sm-2{gap:.5rem!important}.gap-sm-3{gap:1rem!important}.gap-sm-4{gap:1.5rem!important}.gap-sm-5{gap:3rem!important}.row-gap-sm-0{row-gap:0!important}.row-gap-sm-1{row-gap:.25rem!important}.row-gap-sm-2{row-gap:.5rem!important}.row-gap-sm-3{row-gap:1rem!important}.row-gap-sm-4{row-gap:1.5rem!important}.row-gap-sm-5{row-gap:3rem!important}.column-gap-sm-0{column-gap:0!important}.column-gap-sm-1{column-gap:.25rem!important}.column-gap-sm-2{column-gap:.5rem!important}.column-gap-sm-3{column-gap:1rem!important}.column-gap-sm-4{column-gap:1.5rem!important}.column-gap-sm-5{column-gap:3rem!important}.text-sm-start{text-align:left!important}.text-sm-end{text-align:right!important}.text-sm-center{text-align:center!important}}@media (min-width: 768px){.float-md-start{float:left!important}.float-md-end{float:right!important}.float-md-none{float:none!important}.object-fit-md-contain{object-fit:contain!important}.object-fit-md-cover{object-fit:cover!important}.object-fit-md-fill{object-fit:fill!important}.object-fit-md-scale{object-fit:scale-down!important}.object-fit-md-none{object-fit:none!important}.d-md-inline{display:inline!important}.d-md-inline-block{display:inline-block!important}.d-md-block{display:block!important}.d-md-grid{display:grid!important}.d-md-inline-grid{display:inline-grid!important}.d-md-table{display:table!important}.d-md-table-row{display:table-row!important}.d-md-table-cell{display:table-cell!important}.d-md-flex{display:flex!important}.d-md-inline-flex{display:inline-flex!important}.d-md-none{display:none!important}.flex-md-fill{flex:1 1 auto!important}.flex-md-row{flex-direction:row!important}.flex-md-column{flex-direction:column!important}.flex-md-row-reverse{flex-direction:row-reverse!important}.flex-md-column-reverse{flex-direction:column-reverse!important}.flex-md-grow-0{flex-grow:0!important}.flex-md-grow-1{flex-grow:1!important}.flex-md-shrink-0{flex-shrink:0!important}.flex-md-shrink-1{flex-shrink:1!important}.flex-md-wrap{flex-wrap:wrap!important}.flex-md-nowrap{flex-wrap:nowrap!important}.flex-md-wrap-reverse{flex-wrap:wrap-reverse!important}.justify-content-md-start{justify-content:flex-start!important}.justify-content-md-end{justify-content:flex-end!important}.justify-content-md-center{justify-content:center!important}.justify-content-md-between{justify-content:space-between!important}.justify-content-md-around{justify-content:space-around!important}.justify-content-md-evenly{justify-content:space-evenly!important}.align-items-md-start{align-items:flex-start!important}.align-items-md-end{align-items:flex-end!important}.align-items-md-center{align-items:center!important}.align-items-md-baseline{align-items:baseline!important}.align-items-md-stretch{align-items:stretch!important}.align-content-md-start{align-content:flex-start!important}.align-content-md-end{align-content:flex-end!important}.align-content-md-center{align-content:center!important}.align-content-md-between{align-content:space-between!important}.align-content-md-around{align-content:space-around!important}.align-content-md-stretch{align-content:stretch!important}.align-self-md-auto{align-self:auto!important}.align-self-md-start{align-self:flex-start!important}.align-self-md-end{align-self:flex-end!important}.align-self-md-center{align-self:center!important}.align-self-md-baseline{align-self:baseline!important}.align-self-md-stretch{align-self:stretch!important}.order-md-first{order:-1!important}.order-md-0{order:0!important}.order-md-1{order:1!important}.order-md-2{order:2!important}.order-md-3{order:3!important}.order-md-4{order:4!important}.order-md-5{order:5!important}.order-md-last{order:6!important}.m-md-0{margin:0!important}.m-md-1{margin:.25rem!important}.m-md-2{margin:.5rem!important}.m-md-3{margin:1rem!important}.m-md-4{margin:1.5rem!important}.m-md-5{margin:3rem!important}.m-md-auto{margin:auto!important}.mx-md-0{margin-right:0!important;margin-left:0!important}.mx-md-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-md-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-md-3{margin-right:1rem!important;margin-left:1rem!important}.mx-md-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-md-5{margin-right:3rem!important;margin-left:3rem!important}.mx-md-auto{margin-right:auto!important;margin-left:auto!important}.my-md-0{margin-top:0!important;margin-bottom:0!important}.my-md-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-md-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-md-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-md-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-md-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-md-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-md-0{margin-top:0!important}.mt-md-1{margin-top:.25rem!important}.mt-md-2{margin-top:.5rem!important}.mt-md-3{margin-top:1rem!important}.mt-md-4{margin-top:1.5rem!important}.mt-md-5{margin-top:3rem!important}.mt-md-auto{margin-top:auto!important}.me-md-0{margin-right:0!important}.me-md-1{margin-right:.25rem!important}.me-md-2{margin-right:.5rem!important}.me-md-3{margin-right:1rem!important}.me-md-4{margin-right:1.5rem!important}.me-md-5{margin-right:3rem!important}.me-md-auto{margin-right:auto!important}.mb-md-0{margin-bottom:0!important}.mb-md-1{margin-bottom:.25rem!important}.mb-md-2{margin-bottom:.5rem!important}.mb-md-3{margin-bottom:1rem!important}.mb-md-4{margin-bottom:1.5rem!important}.mb-md-5{margin-bottom:3rem!important}.mb-md-auto{margin-bottom:auto!important}.ms-md-0{margin-left:0!important}.ms-md-1{margin-left:.25rem!important}.ms-md-2{margin-left:.5rem!important}.ms-md-3{margin-left:1rem!important}.ms-md-4{margin-left:1.5rem!important}.ms-md-5{margin-left:3rem!important}.ms-md-auto{margin-left:auto!important}.p-md-0{padding:0!important}.p-md-1{padding:.25rem!important}.p-md-2{padding:.5rem!important}.p-md-3{padding:1rem!important}.p-md-4{padding:1.5rem!important}.p-md-5{padding:3rem!important}.px-md-0{padding-right:0!important;padding-left:0!important}.px-md-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-md-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-md-3{padding-right:1rem!important;padding-left:1rem!important}.px-md-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-md-5{padding-right:3rem!important;padding-left:3rem!important}.py-md-0{padding-top:0!important;padding-bottom:0!important}.py-md-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-md-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-md-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-md-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-md-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-md-0{padding-top:0!important}.pt-md-1{padding-top:.25rem!important}.pt-md-2{padding-top:.5rem!important}.pt-md-3{padding-top:1rem!important}.pt-md-4{padding-top:1.5rem!important}.pt-md-5{padding-top:3rem!important}.pe-md-0{padding-right:0!important}.pe-md-1{padding-right:.25rem!important}.pe-md-2{padding-right:.5rem!important}.pe-md-3{padding-right:1rem!important}.pe-md-4{padding-right:1.5rem!important}.pe-md-5{padding-right:3rem!important}.pb-md-0{padding-bottom:0!important}.pb-md-1{padding-bottom:.25rem!important}.pb-md-2{padding-bottom:.5rem!important}.pb-md-3{padding-bottom:1rem!important}.pb-md-4{padding-bottom:1.5rem!important}.pb-md-5{padding-bottom:3rem!important}.ps-md-0{padding-left:0!important}.ps-md-1{padding-left:.25rem!important}.ps-md-2{padding-left:.5rem!important}.ps-md-3{padding-left:1rem!important}.ps-md-4{padding-left:1.5rem!important}.ps-md-5{padding-left:3rem!important}.gap-md-0{gap:0!important}.gap-md-1{gap:.25rem!important}.gap-md-2{gap:.5rem!important}.gap-md-3{gap:1rem!important}.gap-md-4{gap:1.5rem!important}.gap-md-5{gap:3rem!important}.row-gap-md-0{row-gap:0!important}.row-gap-md-1{row-gap:.25rem!important}.row-gap-md-2{row-gap:.5rem!important}.row-gap-md-3{row-gap:1rem!important}.row-gap-md-4{row-gap:1.5rem!important}.row-gap-md-5{row-gap:3rem!important}.column-gap-md-0{column-gap:0!important}.column-gap-md-1{column-gap:.25rem!important}.column-gap-md-2{column-gap:.5rem!important}.column-gap-md-3{column-gap:1rem!important}.column-gap-md-4{column-gap:1.5rem!important}.column-gap-md-5{column-gap:3rem!important}.text-md-start{text-align:left!important}.text-md-end{text-align:right!important}.text-md-center{text-align:center!important}}@media (min-width: 992px){.float-lg-start{float:left!important}.float-lg-end{float:right!important}.float-lg-none{float:none!important}.object-fit-lg-contain{object-fit:contain!important}.object-fit-lg-cover{object-fit:cover!important}.object-fit-lg-fill{object-fit:fill!important}.object-fit-lg-scale{object-fit:scale-down!important}.object-fit-lg-none{object-fit:none!important}.d-lg-inline{display:inline!important}.d-lg-inline-block{display:inline-block!important}.d-lg-block{display:block!important}.d-lg-grid{display:grid!important}.d-lg-inline-grid{display:inline-grid!important}.d-lg-table{display:table!important}.d-lg-table-row{display:table-row!important}.d-lg-table-cell{display:table-cell!important}.d-lg-flex{display:flex!important}.d-lg-inline-flex{display:inline-flex!important}.d-lg-none{display:none!important}.flex-lg-fill{flex:1 1 auto!important}.flex-lg-row{flex-direction:row!important}.flex-lg-column{flex-direction:column!important}.flex-lg-row-reverse{flex-direction:row-reverse!important}.flex-lg-column-reverse{flex-direction:column-reverse!important}.flex-lg-grow-0{flex-grow:0!important}.flex-lg-grow-1{flex-grow:1!important}.flex-lg-shrink-0{flex-shrink:0!important}.flex-lg-shrink-1{flex-shrink:1!important}.flex-lg-wrap{flex-wrap:wrap!important}.flex-lg-nowrap{flex-wrap:nowrap!important}.flex-lg-wrap-reverse{flex-wrap:wrap-reverse!important}.justify-content-lg-start{justify-content:flex-start!important}.justify-content-lg-end{justify-content:flex-end!important}.justify-content-lg-center{justify-content:center!important}.justify-content-lg-between{justify-content:space-between!important}.justify-content-lg-around{justify-content:space-around!important}.justify-content-lg-evenly{justify-content:space-evenly!important}.align-items-lg-start{align-items:flex-start!important}.align-items-lg-end{align-items:flex-end!important}.align-items-lg-center{align-items:center!important}.align-items-lg-baseline{align-items:baseline!important}.align-items-lg-stretch{align-items:stretch!important}.align-content-lg-start{align-content:flex-start!important}.align-content-lg-end{align-content:flex-end!important}.align-content-lg-center{align-content:center!important}.align-content-lg-between{align-content:space-between!important}.align-content-lg-around{align-content:space-around!important}.align-content-lg-stretch{align-content:stretch!important}.align-self-lg-auto{align-self:auto!important}.align-self-lg-start{align-self:flex-start!important}.align-self-lg-end{align-self:flex-end!important}.align-self-lg-center{align-self:center!important}.align-self-lg-baseline{align-self:baseline!important}.align-self-lg-stretch{align-self:stretch!important}.order-lg-first{order:-1!important}.order-lg-0{order:0!important}.order-lg-1{order:1!important}.order-lg-2{order:2!important}.order-lg-3{order:3!important}.order-lg-4{order:4!important}.order-lg-5{order:5!important}.order-lg-last{order:6!important}.m-lg-0{margin:0!important}.m-lg-1{margin:.25rem!important}.m-lg-2{margin:.5rem!important}.m-lg-3{margin:1rem!important}.m-lg-4{margin:1.5rem!important}.m-lg-5{margin:3rem!important}.m-lg-auto{margin:auto!important}.mx-lg-0{margin-right:0!important;margin-left:0!important}.mx-lg-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-lg-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-lg-3{margin-right:1rem!important;margin-left:1rem!important}.mx-lg-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-lg-5{margin-right:3rem!important;margin-left:3rem!important}.mx-lg-auto{margin-right:auto!important;margin-left:auto!important}.my-lg-0{margin-top:0!important;margin-bottom:0!important}.my-lg-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-lg-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-lg-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-lg-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-lg-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-lg-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-lg-0{margin-top:0!important}.mt-lg-1{margin-top:.25rem!important}.mt-lg-2{margin-top:.5rem!important}.mt-lg-3{margin-top:1rem!important}.mt-lg-4{margin-top:1.5rem!important}.mt-lg-5{margin-top:3rem!important}.mt-lg-auto{margin-top:auto!important}.me-lg-0{margin-right:0!important}.me-lg-1{margin-right:.25rem!important}.me-lg-2{margin-right:.5rem!important}.me-lg-3{margin-right:1rem!important}.me-lg-4{margin-right:1.5rem!important}.me-lg-5{margin-right:3rem!important}.me-lg-auto{margin-right:auto!important}.mb-lg-0{margin-bottom:0!important}.mb-lg-1{margin-bottom:.25rem!important}.mb-lg-2{margin-bottom:.5rem!important}.mb-lg-3{margin-bottom:1rem!important}.mb-lg-4{margin-bottom:1.5rem!important}.mb-lg-5{margin-bottom:3rem!important}.mb-lg-auto{margin-bottom:auto!important}.ms-lg-0{margin-left:0!important}.ms-lg-1{margin-left:.25rem!important}.ms-lg-2{margin-left:.5rem!important}.ms-lg-3{margin-left:1rem!important}.ms-lg-4{margin-left:1.5rem!important}.ms-lg-5{margin-left:3rem!important}.ms-lg-auto{margin-left:auto!important}.p-lg-0{padding:0!important}.p-lg-1{padding:.25rem!important}.p-lg-2{padding:.5rem!important}.p-lg-3{padding:1rem!important}.p-lg-4{padding:1.5rem!important}.p-lg-5{padding:3rem!important}.px-lg-0{padding-right:0!important;padding-left:0!important}.px-lg-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-lg-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-lg-3{padding-right:1rem!important;padding-left:1rem!important}.px-lg-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-lg-5{padding-right:3rem!important;padding-left:3rem!important}.py-lg-0{padding-top:0!important;padding-bottom:0!important}.py-lg-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-lg-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-lg-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-lg-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-lg-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-lg-0{padding-top:0!important}.pt-lg-1{padding-top:.25rem!important}.pt-lg-2{padding-top:.5rem!important}.pt-lg-3{padding-top:1rem!important}.pt-lg-4{padding-top:1.5rem!important}.pt-lg-5{padding-top:3rem!important}.pe-lg-0{padding-right:0!important}.pe-lg-1{padding-right:.25rem!important}.pe-lg-2{padding-right:.5rem!important}.pe-lg-3{padding-right:1rem!important}.pe-lg-4{padding-right:1.5rem!important}.pe-lg-5{padding-right:3rem!important}.pb-lg-0{padding-bottom:0!important}.pb-lg-1{padding-bottom:.25rem!important}.pb-lg-2{padding-bottom:.5rem!important}.pb-lg-3{padding-bottom:1rem!important}.pb-lg-4{padding-bottom:1.5rem!important}.pb-lg-5{padding-bottom:3rem!important}.ps-lg-0{padding-left:0!important}.ps-lg-1{padding-left:.25rem!important}.ps-lg-2{padding-left:.5rem!important}.ps-lg-3{padding-left:1rem!important}.ps-lg-4{padding-left:1.5rem!important}.ps-lg-5{padding-left:3rem!important}.gap-lg-0{gap:0!important}.gap-lg-1{gap:.25rem!important}.gap-lg-2{gap:.5rem!important}.gap-lg-3{gap:1rem!important}.gap-lg-4{gap:1.5rem!important}.gap-lg-5{gap:3rem!important}.row-gap-lg-0{row-gap:0!important}.row-gap-lg-1{row-gap:.25rem!important}.row-gap-lg-2{row-gap:.5rem!important}.row-gap-lg-3{row-gap:1rem!important}.row-gap-lg-4{row-gap:1.5rem!important}.row-gap-lg-5{row-gap:3rem!important}.column-gap-lg-0{column-gap:0!important}.column-gap-lg-1{column-gap:.25rem!important}.column-gap-lg-2{column-gap:.5rem!important}.column-gap-lg-3{column-gap:1rem!important}.column-gap-lg-4{column-gap:1.5rem!important}.column-gap-lg-5{column-gap:3rem!important}.text-lg-start{text-align:left!important}.text-lg-end{text-align:right!important}.text-lg-center{text-align:center!important}}@media (min-width: 1200px){.float-xl-start{float:left!important}.float-xl-end{float:right!important}.float-xl-none{float:none!important}.object-fit-xl-contain{object-fit:contain!important}.object-fit-xl-cover{object-fit:cover!important}.object-fit-xl-fill{object-fit:fill!important}.object-fit-xl-scale{object-fit:scale-down!important}.object-fit-xl-none{object-fit:none!important}.d-xl-inline{display:inline!important}.d-xl-inline-block{display:inline-block!important}.d-xl-block{display:block!important}.d-xl-grid{display:grid!important}.d-xl-inline-grid{display:inline-grid!important}.d-xl-table{display:table!important}.d-xl-table-row{display:table-row!important}.d-xl-table-cell{display:table-cell!important}.d-xl-flex{display:flex!important}.d-xl-inline-flex{display:inline-flex!important}.d-xl-none{display:none!important}.flex-xl-fill{flex:1 1 auto!important}.flex-xl-row{flex-direction:row!important}.flex-xl-column{flex-direction:column!important}.flex-xl-row-reverse{flex-direction:row-reverse!important}.flex-xl-column-reverse{flex-direction:column-reverse!important}.flex-xl-grow-0{flex-grow:0!important}.flex-xl-grow-1{flex-grow:1!important}.flex-xl-shrink-0{flex-shrink:0!important}.flex-xl-shrink-1{flex-shrink:1!important}.flex-xl-wrap{flex-wrap:wrap!important}.flex-xl-nowrap{flex-wrap:nowrap!important}.flex-xl-wrap-reverse{flex-wrap:wrap-reverse!important}.justify-content-xl-start{justify-content:flex-start!important}.justify-content-xl-end{justify-content:flex-end!important}.justify-content-xl-center{justify-content:center!important}.justify-content-xl-between{justify-content:space-between!important}.justify-content-xl-around{justify-content:space-around!important}.justify-content-xl-evenly{justify-content:space-evenly!important}.align-items-xl-start{align-items:flex-start!important}.align-items-xl-end{align-items:flex-end!important}.align-items-xl-center{align-items:center!important}.align-items-xl-baseline{align-items:baseline!important}.align-items-xl-stretch{align-items:stretch!important}.align-content-xl-start{align-content:flex-start!important}.align-content-xl-end{align-content:flex-end!important}.align-content-xl-center{align-content:center!important}.align-content-xl-between{align-content:space-between!important}.align-content-xl-around{align-content:space-around!important}.align-content-xl-stretch{align-content:stretch!important}.align-self-xl-auto{align-self:auto!important}.align-self-xl-start{align-self:flex-start!important}.align-self-xl-end{align-self:flex-end!important}.align-self-xl-center{align-self:center!important}.align-self-xl-baseline{align-self:baseline!important}.align-self-xl-stretch{align-self:stretch!important}.order-xl-first{order:-1!important}.order-xl-0{order:0!important}.order-xl-1{order:1!important}.order-xl-2{order:2!important}.order-xl-3{order:3!important}.order-xl-4{order:4!important}.order-xl-5{order:5!important}.order-xl-last{order:6!important}.m-xl-0{margin:0!important}.m-xl-1{margin:.25rem!important}.m-xl-2{margin:.5rem!important}.m-xl-3{margin:1rem!important}.m-xl-4{margin:1.5rem!important}.m-xl-5{margin:3rem!important}.m-xl-auto{margin:auto!important}.mx-xl-0{margin-right:0!important;margin-left:0!important}.mx-xl-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-xl-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-xl-3{margin-right:1rem!important;margin-left:1rem!important}.mx-xl-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-xl-5{margin-right:3rem!important;margin-left:3rem!important}.mx-xl-auto{margin-right:auto!important;margin-left:auto!important}.my-xl-0{margin-top:0!important;margin-bottom:0!important}.my-xl-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-xl-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-xl-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-xl-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-xl-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-xl-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-xl-0{margin-top:0!important}.mt-xl-1{margin-top:.25rem!important}.mt-xl-2{margin-top:.5rem!important}.mt-xl-3{margin-top:1rem!important}.mt-xl-4{margin-top:1.5rem!important}.mt-xl-5{margin-top:3rem!important}.mt-xl-auto{margin-top:auto!important}.me-xl-0{margin-right:0!important}.me-xl-1{margin-right:.25rem!important}.me-xl-2{margin-right:.5rem!important}.me-xl-3{margin-right:1rem!important}.me-xl-4{margin-right:1.5rem!important}.me-xl-5{margin-right:3rem!important}.me-xl-auto{margin-right:auto!important}.mb-xl-0{margin-bottom:0!important}.mb-xl-1{margin-bottom:.25rem!important}.mb-xl-2{margin-bottom:.5rem!important}.mb-xl-3{margin-bottom:1rem!important}.mb-xl-4{margin-bottom:1.5rem!important}.mb-xl-5{margin-bottom:3rem!important}.mb-xl-auto{margin-bottom:auto!important}.ms-xl-0{margin-left:0!important}.ms-xl-1{margin-left:.25rem!important}.ms-xl-2{margin-left:.5rem!important}.ms-xl-3{margin-left:1rem!important}.ms-xl-4{margin-left:1.5rem!important}.ms-xl-5{margin-left:3rem!important}.ms-xl-auto{margin-left:auto!important}.p-xl-0{padding:0!important}.p-xl-1{padding:.25rem!important}.p-xl-2{padding:.5rem!important}.p-xl-3{padding:1rem!important}.p-xl-4{padding:1.5rem!important}.p-xl-5{padding:3rem!important}.px-xl-0{padding-right:0!important;padding-left:0!important}.px-xl-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-xl-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-xl-3{padding-right:1rem!important;padding-left:1rem!important}.px-xl-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-xl-5{padding-right:3rem!important;padding-left:3rem!important}.py-xl-0{padding-top:0!important;padding-bottom:0!important}.py-xl-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-xl-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-xl-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-xl-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-xl-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-xl-0{padding-top:0!important}.pt-xl-1{padding-top:.25rem!important}.pt-xl-2{padding-top:.5rem!important}.pt-xl-3{padding-top:1rem!important}.pt-xl-4{padding-top:1.5rem!important}.pt-xl-5{padding-top:3rem!important}.pe-xl-0{padding-right:0!important}.pe-xl-1{padding-right:.25rem!important}.pe-xl-2{padding-right:.5rem!important}.pe-xl-3{padding-right:1rem!important}.pe-xl-4{padding-right:1.5rem!important}.pe-xl-5{padding-right:3rem!important}.pb-xl-0{padding-bottom:0!important}.pb-xl-1{padding-bottom:.25rem!important}.pb-xl-2{padding-bottom:.5rem!important}.pb-xl-3{padding-bottom:1rem!important}.pb-xl-4{padding-bottom:1.5rem!important}.pb-xl-5{padding-bottom:3rem!important}.ps-xl-0{padding-left:0!important}.ps-xl-1{padding-left:.25rem!important}.ps-xl-2{padding-left:.5rem!important}.ps-xl-3{padding-left:1rem!important}.ps-xl-4{padding-left:1.5rem!important}.ps-xl-5{padding-left:3rem!important}.gap-xl-0{gap:0!important}.gap-xl-1{gap:.25rem!important}.gap-xl-2{gap:.5rem!important}.gap-xl-3{gap:1rem!important}.gap-xl-4{gap:1.5rem!important}.gap-xl-5{gap:3rem!important}.row-gap-xl-0{row-gap:0!important}.row-gap-xl-1{row-gap:.25rem!important}.row-gap-xl-2{row-gap:.5rem!important}.row-gap-xl-3{row-gap:1rem!important}.row-gap-xl-4{row-gap:1.5rem!important}.row-gap-xl-5{row-gap:3rem!important}.column-gap-xl-0{column-gap:0!important}.column-gap-xl-1{column-gap:.25rem!important}.column-gap-xl-2{column-gap:.5rem!important}.column-gap-xl-3{column-gap:1rem!important}.column-gap-xl-4{column-gap:1.5rem!important}.column-gap-xl-5{column-gap:3rem!important}.text-xl-start{text-align:left!important}.text-xl-end{text-align:right!important}.text-xl-center{text-align:center!important}}@media (min-width: 1400px){.float-xxl-start{float:left!important}.float-xxl-end{float:right!important}.float-xxl-none{float:none!important}.object-fit-xxl-contain{object-fit:contain!important}.object-fit-xxl-cover{object-fit:cover!important}.object-fit-xxl-fill{object-fit:fill!important}.object-fit-xxl-scale{object-fit:scale-down!important}.object-fit-xxl-none{object-fit:none!important}.d-xxl-inline{display:inline!important}.d-xxl-inline-block{display:inline-block!important}.d-xxl-block{display:block!important}.d-xxl-grid{display:grid!important}.d-xxl-inline-grid{display:inline-grid!important}.d-xxl-table{display:table!important}.d-xxl-table-row{display:table-row!important}.d-xxl-table-cell{display:table-cell!important}.d-xxl-flex{display:flex!important}.d-xxl-inline-flex{display:inline-flex!important}.d-xxl-none{display:none!important}.flex-xxl-fill{flex:1 1 auto!important}.flex-xxl-row{flex-direction:row!important}.flex-xxl-column{flex-direction:column!important}.flex-xxl-row-reverse{flex-direction:row-reverse!important}.flex-xxl-column-reverse{flex-direction:column-reverse!important}.flex-xxl-grow-0{flex-grow:0!important}.flex-xxl-grow-1{flex-grow:1!important}.flex-xxl-shrink-0{flex-shrink:0!important}.flex-xxl-shrink-1{flex-shrink:1!important}.flex-xxl-wrap{flex-wrap:wrap!important}.flex-xxl-nowrap{flex-wrap:nowrap!important}.flex-xxl-wrap-reverse{flex-wrap:wrap-reverse!important}.justify-content-xxl-start{justify-content:flex-start!important}.justify-content-xxl-end{justify-content:flex-end!important}.justify-content-xxl-center{justify-content:center!important}.justify-content-xxl-between{justify-content:space-between!important}.justify-content-xxl-around{justify-content:space-around!important}.justify-content-xxl-evenly{justify-content:space-evenly!important}.align-items-xxl-start{align-items:flex-start!important}.align-items-xxl-end{align-items:flex-end!important}.align-items-xxl-center{align-items:center!important}.align-items-xxl-baseline{align-items:baseline!important}.align-items-xxl-stretch{align-items:stretch!important}.align-content-xxl-start{align-content:flex-start!important}.align-content-xxl-end{align-content:flex-end!important}.align-content-xxl-center{align-content:center!important}.align-content-xxl-between{align-content:space-between!important}.align-content-xxl-around{align-content:space-around!important}.align-content-xxl-stretch{align-content:stretch!important}.align-self-xxl-auto{align-self:auto!important}.align-self-xxl-start{align-self:flex-start!important}.align-self-xxl-end{align-self:flex-end!important}.align-self-xxl-center{align-self:center!important}.align-self-xxl-baseline{align-self:baseline!important}.align-self-xxl-stretch{align-self:stretch!important}.order-xxl-first{order:-1!important}.order-xxl-0{order:0!important}.order-xxl-1{order:1!important}.order-xxl-2{order:2!important}.order-xxl-3{order:3!important}.order-xxl-4{order:4!important}.order-xxl-5{order:5!important}.order-xxl-last{order:6!important}.m-xxl-0{margin:0!important}.m-xxl-1{margin:.25rem!important}.m-xxl-2{margin:.5rem!important}.m-xxl-3{margin:1rem!important}.m-xxl-4{margin:1.5rem!important}.m-xxl-5{margin:3rem!important}.m-xxl-auto{margin:auto!important}.mx-xxl-0{margin-right:0!important;margin-left:0!important}.mx-xxl-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-xxl-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-xxl-3{margin-right:1rem!important;margin-left:1rem!important}.mx-xxl-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-xxl-5{margin-right:3rem!important;margin-left:3rem!important}.mx-xxl-auto{margin-right:auto!important;margin-left:auto!important}.my-xxl-0{margin-top:0!important;margin-bottom:0!important}.my-xxl-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-xxl-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-xxl-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-xxl-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-xxl-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-xxl-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-xxl-0{margin-top:0!important}.mt-xxl-1{margin-top:.25rem!important}.mt-xxl-2{margin-top:.5rem!important}.mt-xxl-3{margin-top:1rem!important}.mt-xxl-4{margin-top:1.5rem!important}.mt-xxl-5{margin-top:3rem!important}.mt-xxl-auto{margin-top:auto!important}.me-xxl-0{margin-right:0!important}.me-xxl-1{margin-right:.25rem!important}.me-xxl-2{margin-right:.5rem!important}.me-xxl-3{margin-right:1rem!important}.me-xxl-4{margin-right:1.5rem!important}.me-xxl-5{margin-right:3rem!important}.me-xxl-auto{margin-right:auto!important}.mb-xxl-0{margin-bottom:0!important}.mb-xxl-1{margin-bottom:.25rem!important}.mb-xxl-2{margin-bottom:.5rem!important}.mb-xxl-3{margin-bottom:1rem!important}.mb-xxl-4{margin-bottom:1.5rem!important}.mb-xxl-5{margin-bottom:3rem!important}.mb-xxl-auto{margin-bottom:auto!important}.ms-xxl-0{margin-left:0!important}.ms-xxl-1{margin-left:.25rem!important}.ms-xxl-2{margin-left:.5rem!important}.ms-xxl-3{margin-left:1rem!important}.ms-xxl-4{margin-left:1.5rem!important}.ms-xxl-5{margin-left:3rem!important}.ms-xxl-auto{margin-left:auto!important}.p-xxl-0{padding:0!important}.p-xxl-1{padding:.25rem!important}.p-xxl-2{padding:.5rem!important}.p-xxl-3{padding:1rem!important}.p-xxl-4{padding:1.5rem!important}.p-xxl-5{padding:3rem!important}.px-xxl-0{padding-right:0!important;padding-left:0!important}.px-xxl-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-xxl-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-xxl-3{padding-right:1rem!important;padding-left:1rem!important}.px-xxl-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-xxl-5{padding-right:3rem!important;padding-left:3rem!important}.py-xxl-0{padding-top:0!important;padding-bottom:0!important}.py-xxl-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-xxl-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-xxl-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-xxl-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-xxl-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-xxl-0{padding-top:0!important}.pt-xxl-1{padding-top:.25rem!important}.pt-xxl-2{padding-top:.5rem!important}.pt-xxl-3{padding-top:1rem!important}.pt-xxl-4{padding-top:1.5rem!important}.pt-xxl-5{padding-top:3rem!important}.pe-xxl-0{padding-right:0!important}.pe-xxl-1{padding-right:.25rem!important}.pe-xxl-2{padding-right:.5rem!important}.pe-xxl-3{padding-right:1rem!important}.pe-xxl-4{padding-right:1.5rem!important}.pe-xxl-5{padding-right:3rem!important}.pb-xxl-0{padding-bottom:0!important}.pb-xxl-1{padding-bottom:.25rem!important}.pb-xxl-2{padding-bottom:.5rem!important}.pb-xxl-3{padding-bottom:1rem!important}.pb-xxl-4{padding-bottom:1.5rem!important}.pb-xxl-5{padding-bottom:3rem!important}.ps-xxl-0{padding-left:0!important}.ps-xxl-1{padding-left:.25rem!important}.ps-xxl-2{padding-left:.5rem!important}.ps-xxl-3{padding-left:1rem!important}.ps-xxl-4{padding-left:1.5rem!important}.ps-xxl-5{padding-left:3rem!important}.gap-xxl-0{gap:0!important}.gap-xxl-1{gap:.25rem!important}.gap-xxl-2{gap:.5rem!important}.gap-xxl-3{gap:1rem!important}.gap-xxl-4{gap:1.5rem!important}.gap-xxl-5{gap:3rem!important}.row-gap-xxl-0{row-gap:0!important}.row-gap-xxl-1{row-gap:.25rem!important}.row-gap-xxl-2{row-gap:.5rem!important}.row-gap-xxl-3{row-gap:1rem!important}.row-gap-xxl-4{row-gap:1.5rem!important}.row-gap-xxl-5{row-gap:3rem!important}.column-gap-xxl-0{column-gap:0!important}.column-gap-xxl-1{column-gap:.25rem!important}.column-gap-xxl-2{column-gap:.5rem!important}.column-gap-xxl-3{column-gap:1rem!important}.column-gap-xxl-4{column-gap:1.5rem!important}.column-gap-xxl-5{column-gap:3rem!important}.text-xxl-start{text-align:left!important}.text-xxl-end{text-align:right!important}.text-xxl-center{text-align:center!important}}@media (min-width: 1200px){.fs-1{font-size:2.5rem!important}.fs-2{font-size:2rem!important}.fs-3{font-size:1.75rem!important}.fs-4{font-size:1.5rem!important}}@media print{.d-print-inline{display:inline!important}.d-print-inline-block{display:inline-block!important}.d-print-block{display:block!important}.d-print-grid{display:grid!important}.d-print-inline-grid{display:inline-grid!important}.d-print-table{display:table!important}.d-print-table-row{display:table-row!important}.d-print-table-cell{display:table-cell!important}.d-print-flex{display:flex!important}.d-print-inline-flex{display:inline-flex!important}.d-print-none{display:none!important}} + diff --git a/web/templates/style.css b/web/static/style.css similarity index 100% rename from web/templates/style.css rename to web/static/style.css diff --git a/web/templates/base.html b/web/templates/base.html deleted file mode 100644 index 3f1258c..0000000 --- a/web/templates/base.html +++ /dev/null @@ -1,59 +0,0 @@ - - - - - - {{.Title}} - NZBreX - - - {{block "extra_head" .}}{{end}} - - - - - - -
- {{block "content" .}}{{end}} -
- - -
- - -
- NZBreX - NZB Refresh X | Usenet Article Restoration Tool -
- - - {{block "extra_scripts" .}}{{end}} - - diff --git a/web/templates/index.html b/web/templates/index.html index b192617..b132811 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -4,7 +4,7 @@ NZBreX - NZB Refresh X - + @@ -32,7 +32,7 @@

NZBreX

- + + +