diff --git a/AutoRegistrator/AutoRegistrator.cs b/AutoRegistrator/AutoRegistrator.cs index 90244a5c2..80667954b 100644 --- a/AutoRegistrator/AutoRegistrator.cs +++ b/AutoRegistrator/AutoRegistrator.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Data.Entity; using System.Diagnostics; using System.IO; using System.Linq; @@ -153,31 +154,66 @@ private void OnRapidChanged() private void SynchronizeMapsFromSpringFiles() { - if (GlobalConst.Mode == ModeType.Live) - { - var db = new ZkDataContext(); + if (GlobalConst.Mode != ModeType.Live) return; - var processedFiles = db.ResourceContentFiles.Where(x => x.Resource.TypeID == ResourceType.Map).Select(x => x.FileName.ToLower()).Distinct().ToLookup(x => x); - var triedFiles = db.SpringFilesUnitsyncAttempts.Select(x => x.FileName.ToLower()).Distinct().ToLookup(x => x); + HashSet processedFiles, triedFiles; + using (var db = new ZkDataContext()) + { + processedFiles = new HashSet( + db.ResourceContentFiles.AsNoTracking() + .Where(x => x.Resource.TypeID == ResourceType.Map) + .Select(x => x.FileName.ToLower())); + triedFiles = new HashSet( + db.SpringFilesUnitsyncAttempts.AsNoTracking().Select(x => x.FileName.ToLower())); + } - var webSyncer = new WebFolderSyncer(); + var webSyncer = new WebFolderSyncer(); + var autoregMaps = Path.Combine(Paths.WritableDirectory, "maps"); + var contentMaps = Path.Combine(sitePath, "content", "maps"); + if (!Directory.Exists(contentMaps)) Directory.CreateDirectory(contentMaps); - foreach (var file in webSyncer.GetFileList()) + foreach (var file in webSyncer.GetFileList()) + { + var fileLc = file.ToLower(); + var alreadyRegistered = processedFiles.Contains(fileLc); + var alreadyAttempted = triedFiles.Contains(fileLc); + var contentFile = Path.Combine(contentMaps, file); + var hasLocal = File.Exists(contentFile); + + // skip cases: + // - registered AND local copy present → nothing to do + // - previously attempted but never made it into DB → known failure, don't retry + if (alreadyRegistered && hasLocal) continue; + if (alreadyAttempted && !alreadyRegistered) continue; + + if (!webSyncer.DownloadFile(autoregMaps, file)) continue; + var srcFile = Path.Combine(autoregMaps, file); + + var registered = alreadyRegistered; + if (!alreadyRegistered) { - if (processedFiles.Contains(file.ToLower()) || triedFiles.Contains(file.ToLower())) continue; - - webSyncer.DownloadFile(Path.Combine(Paths.WritableDirectory,"maps"), file); - - UnitSyncer.Scan(); + var results = UnitSyncer.Scan(); + var ourResult = results?.FirstOrDefault(r => string.Equals(r.ResourceInfo?.ArchiveName, file, StringComparison.OrdinalIgnoreCase)); + registered = ourResult != null && ourResult.Status != UnitSyncer.ResourceFileStatus.RegistrationError; - db.SpringFilesUnitsyncAttempts.Add(new SpringFilesUnitsyncAttempt() { FileName = file }); - db.SaveChanges(); + using (var db = new ZkDataContext()) + { + db.SpringFilesUnitsyncAttempts.Add(new SpringFilesUnitsyncAttempt() { FileName = file }); + db.SaveChanges(); + } + triedFiles.Add(fileLc); + } - try + if (registered && !hasLocal) + { + try { File.Copy(srcFile, contentFile); } + catch (Exception ex) { - File.Delete(Path.Combine(Paths.WritableDirectory, "maps", file)); - } catch (Exception ex) { } + Trace.TraceWarning("Copy {0} to content/maps failed: {1}", file, ex.Message); + } } + + try { File.Delete(srcFile); } catch { } } } diff --git a/AutoRegistrator/WebFolderSync.cs b/AutoRegistrator/WebFolderSync.cs index c11199c54..de7ee3dfd 100644 --- a/AutoRegistrator/WebFolderSync.cs +++ b/AutoRegistrator/WebFolderSync.cs @@ -1,69 +1,62 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Linq; using System.Net; -using System.Text.RegularExpressions; +using Newtonsoft.Json; +using ZkData; namespace ZeroKWeb { public class WebFolderSyncer { - private string urlSource; - - public WebFolderSyncer(string urlSource = "http://api.springfiles.com/files/maps/") - { - this.urlSource = urlSource; - } - public List GetFileList() { + var ret = new List(); using (var wc = new WebClient()) { - var str = wc.DownloadString(urlSource); - return (from Match m in Regex.Matches(str, "([^<]+)") - where m.Success && m.Groups[1].Value == m.Groups[2].Value - select m.Groups[1].Value).ToList(); + var url = $"{GlobalConst.SpringfilesBaseUrl}json.php?category=map&springname=*&logical=or&nosensitive=on&limit=1000000&offset=0"; + var json = wc.DownloadString(url); + var entries = JsonConvert.DeserializeObject>(json); + if (entries == null) return ret; + foreach (var e in entries) + { + if (e?.path == "maps" && !string.IsNullOrEmpty(e.filename) && e.mirrors != null && e.mirrors.Length > 0) + ret.Add(e.filename); + } } + return ret; } - public void DownloadFile(string targetFolder, string file) + public bool DownloadFile(string targetFolder, string file) { if (!Directory.Exists(targetFolder)) Directory.CreateDirectory(targetFolder); var targetFile = Path.Combine(targetFolder, file); - if (!File.Exists(targetFile)) + if (File.Exists(targetFile)) return true; + + var tempFile = Path.GetTempFileName(); + using (var wc = new WebClient()) { - using (var wc = new WebClient()) + try { - var tempFile = Path.GetTempFileName(); - try - { - wc.DownloadFile(urlSource + file, tempFile); - } - catch (Exception ex) - { - Trace.TraceWarning("Download of file {0} failed: {1}", file, ex.Message); - File.Delete(tempFile); - } - try - { - File.Move(tempFile, targetFile); - } - catch { } + wc.DownloadFile($"{GlobalConst.SpringfilesBaseUrl}files/maps/{file}", tempFile); + File.Move(tempFile, targetFile); + return true; + } + catch (Exception ex) + { + Trace.TraceWarning("Springfiles download of {0} failed: {1}", file, ex.Message); + try { File.Delete(tempFile); } catch { } + return false; } } } - - public void SynchronizeFolders(string targetFolder) + private class SpringfilesEntry { - if (!Directory.Exists(targetFolder)) Directory.CreateDirectory(targetFolder); - foreach (var file in GetFileList()) - { - DownloadFile(targetFolder, file); - } + public string filename { get; set; } + public string path { get; set; } + public string[] mirrors { get; set; } } - } } diff --git a/Shared/PlasmaShared/GlobalConst.cs b/Shared/PlasmaShared/GlobalConst.cs index 3b49c896b..497d4a0a5 100644 --- a/Shared/PlasmaShared/GlobalConst.cs +++ b/Shared/PlasmaShared/GlobalConst.cs @@ -79,8 +79,6 @@ static void SetMode(ModeType newMode) break; } - if (IsLongAfterSteam) DefaultDownloadMirrors = new[] { BaseSiteUrl +"/content/%t/%f" }; - ResourceBaseUrl = string.Format("{0}/Resources", BaseSiteUrl); BaseImageUrl = string.Format("{0}/img/", BaseSiteUrl); SelfUpdaterBaseUrl = string.Format("{0}/lobby", BaseSiteUrl); @@ -98,6 +96,8 @@ static void SetMode(ModeType newMode) public static string BaseImageUrl; public static string BaseSiteUrl; + public const string SpringfilesBaseUrl = "https://springfiles.springrts.com/"; + public static string DefaultZkTag => Mode == ModeType.Live ? "zk:stable" : "zk:test"; public static string DefaultChobbyTag => Mode == ModeType.Live ? "zkmenu:stable" : "zkmenu:test"; @@ -235,7 +235,6 @@ static void SetMode(ModeType newMode) public static string ResourceBaseUrl; public static string SelfUpdaterBaseUrl; - public static string[] DefaultDownloadMirrors = {}; public static string LobbyServerHost; public static int LobbyServerPort; public static bool LobbyServerUpdateSpectatorsInstantly = false; diff --git a/Zero-K.info/AppCode/ResourceLinkProvider.cs b/Zero-K.info/AppCode/ResourceLinkProvider.cs index dafd09c3b..ed88e3882 100644 --- a/Zero-K.info/AppCode/ResourceLinkProvider.cs +++ b/Zero-K.info/AppCode/ResourceLinkProvider.cs @@ -1,209 +1,175 @@ -using System; -using System.Collections.Generic; -using System.Configuration; -using System.Diagnostics; -using System.Linq; -using System.Net; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using ZkData; - -namespace ZeroKWeb -{ - public static class ResourceLinkProvider - { - const double CheckPeriodForMissingLinks = 1; // check invalid every minute - const double CheckPeriodForValidLinks = 60 * 12; // check links every 12 hours - static readonly Dictionary Requests = new Dictionary(); - - public static string[] Mirrors = GlobalConst.DefaultDownloadMirrors; - - static ResourceLinkProvider() - { - } - - public static bool GetLinksAndTorrent(string internalName, - out List links, - out byte[] torrent, - out List dependencies, - out ResourceType resourceType, - out string torrentFileName) - { - var db = new ZkDataContext(); - - var resource = db.Resources.SingleOrDefault(x => x.InternalName == internalName); - if (resource == null) - { - torrent = null; - links = null; - dependencies = null; - resourceType = ResourceType.Map; - torrentFileName = null; - return false; - } - - dependencies = resource.ResourceDependencies.Select(x => x.NeedsInternalName).ToList(); - resourceType = resource.TypeID; - - var bestOld = resource.ResourceContentFiles.FirstOrDefault(x => x.LinkCount == resource.ResourceContentFiles.Max(y => y.LinkCount)); - if (bestOld != null && bestOld.LinkCount > 0 && (resource.MissionID != null || - (resource.LastLinkCheck != null && DateTime.UtcNow.Subtract(resource.LastLinkCheck.Value).TotalHours < 2))) - { - // use cached values for missions or resources checked less than 1 day ago - links = PlasmaServer.GetLinkArray(bestOld); - torrent = PlasmaServer.GetTorrentData(bestOld); - torrentFileName = PlasmaServer.GetTorrentFileName(bestOld); - if (links.Count > 0) db.Database.ExecuteSqlCommand("UPDATE Resources SET DownloadCount = DownloadCount+1 WHERE ResourceID={0}", resource.ResourceID); - else db.Database.ExecuteSqlCommand("UPDATE Resources SET NoLinkDownloadCount = NoLinkDownloadCount+1 WHERE ResourceID={0}", resource.ResourceID); - - return true; - } - - RequestData data; - var isNew = false; - lock (Requests) - { - if (!Requests.TryGetValue(resource.ResourceID, out data)) - { - data = new RequestData(resource.ResourceID); - isNew = true; - Requests.Add(resource.ResourceID, data); - } - } - - if (!isNew) - { - // request is ongoing, wait for completion - data.WaitHandle.WaitOne(); - torrentFileName = PlasmaServer.GetTorrentFileName(data.ContentFile); - links = PlasmaServer.GetLinkArray(data.ContentFile); - torrent = PlasmaServer.GetTorrentData(data.ContentFile); - if (links.Count > 0) db.Database.ExecuteSqlCommand("UPDATE Resources SET DownloadCount = DownloadCount+1 WHERE ResourceID={0}", resource.ResourceID); - else db.Database.ExecuteSqlCommand("UPDATE Resources SET NoLinkDownloadCount = NoLinkDownloadCount+1 WHERE ResourceID={0}", resource.ResourceID); - return true; - } - else - { - // new request - actually perform it - try - { - var toCheck = from x in resource.ResourceContentFiles - group x by new { x.FileName, x.Length } - into g - where !g.Key.FileName.EndsWith(".sdp") - select g.First(); - - Task.WaitAll(toCheck.Select(x => Task.Factory.StartNew(() => UpdateLinks(x))).ToArray()); - - db.SaveChanges(); - - // find best content file - the one with most links - var best = resource.ResourceContentFiles.FirstOrDefault(x => x.LinkCount == resource.ResourceContentFiles.Max(y => y.LinkCount)); - - if (best != null) data.ContentFile = best; - else data.ContentFile = resource.ResourceContentFiles.First(); // all content files sux, reurn any - - links = PlasmaServer.GetLinkArray(data.ContentFile); - torrent = PlasmaServer.GetTorrentData(data.ContentFile); - torrentFileName = PlasmaServer.GetTorrentFileName(data.ContentFile); - if (links.Count > 0) resource.DownloadCount++; - else resource.NoLinkDownloadCount++; - db.SaveChanges(); - return true; - } - finally - { - lock (Requests) Requests.Remove(data.ResourceID); - data.WaitHandle.Set(); // notify other waiting Requests that its done - } - } - } - - public static void ValidateLink(string link, int length, List valids) - { - string realLink; - if (GetLinkLength(link, out realLink) != length) lock (valids) valids.Remove(link); // invalid length, remove - else if (link != realLink) - { - lock (valids) // redirect, update url - { - valids.Remove(link); - valids.Add(realLink); - } - } - } - - static long GetLinkLength(string url, out string redirectUrl) - { - redirectUrl = url; - try - { - var wr = (HttpWebRequest)WebRequest.Create(url); - wr.Timeout = 2000; - wr.Method = "HEAD"; - var res = wr.GetResponse(); - redirectUrl = res.ResponseUri.ToString(); - var cl = res.ContentLength; - wr.Abort(); - return cl; - } - catch - { - return 0; - } - } - - public static void UpdateLinks(ResourceContentFile content) - { - var valids = new List(); - if (content.LinkCount > 0 || content.Links != null) valids = new List(content.Links.Split('\n')); // get previous links - - if (content.FileName.EndsWith(".sdp")) return; - - if (!Debugger.IsAttached) - { - // should we use cached entries or run full check? - if (content.Resource.LastLinkCheck != null) - { - if (content.LinkCount > 0 && - DateTime.UtcNow.Subtract(content.Resource.LastLinkCheck.Value).TotalMinutes < CheckPeriodForValidLinks) return; - if (content.LinkCount == 0 && - DateTime.UtcNow.Subtract(content.Resource.LastLinkCheck.Value).TotalMinutes < CheckPeriodForMissingLinks) return; - } - } - - // combine with hardcoded mirrors - foreach (var url in Mirrors) - { - var replaced = url.Replace("%t", content.Resource.TypeID == ResourceType.Map ? "maps" : "games").Replace("%f", content.FileName); - if (!valids.Contains(replaced)) valids.Add(replaced); - } - - // check validity of all links at once - - Task.WaitAll(new List(valids).Select(link => Task.Factory.StartNew(() => ValidateLink(link, content.Length, valids))).ToArray()); - - valids = valids.Distinct().ToList(); - - lock (content) - { - content.LinkCount = valids.Count; - content.Resource.LastLinkCheck = DateTime.UtcNow; - content.Links = string.Join("\n", valids.ToArray()); - } - } - - class RequestData - { - public ResourceContentFile ContentFile; - public readonly int ResourceID; - public readonly EventWaitHandle WaitHandle = new EventWaitHandle(false, EventResetMode.ManualReset); - - public RequestData(int resourceID) - { - ResourceID = resourceID; - } - } - } -} \ No newline at end of file +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net; +using System.Threading; +using ZkData; + +namespace ZeroKWeb +{ + public static class ResourceLinkProvider + { + const double SpringfilesRecheckHours = 24; + + // dedup concurrent springfiles probes per Md5 — Lazy with ExecutionAndPublication + // ensures the factory (the actual probe) runs exactly once; concurrent callers block on .Value + static readonly ConcurrentDictionary> InflightProbes + = new ConcurrentDictionary>(); + + public static bool GetLinksAndTorrent(string internalName, + out List links, + out byte[] torrent, + out List dependencies, + out ResourceType resourceType, + out string torrentFileName) + { + var db = new ZkDataContext(); + var resource = db.Resources.SingleOrDefault(x => x.InternalName == internalName); + if (resource == null) + { + torrent = null; links = null; dependencies = null; + resourceType = ResourceType.Map; torrentFileName = null; + return false; + } + + dependencies = resource.ResourceDependencies.Select(x => x.NeedsInternalName).ToList(); + resourceType = resource.TypeID; + + var content = resource.ResourceContentFiles.Where(x => !x.FileName.EndsWith(".sdp")) + .OrderByDescending(x => x.LinkCount).FirstOrDefault() + ?? resource.ResourceContentFiles.FirstOrDefault(); + + if (content == null) + { + links = new List(); + torrent = null; torrentFileName = null; + return true; + } + + torrentFileName = PlasmaServer.GetTorrentFileName(content); + torrent = PlasmaServer.GetTorrentData(content); + links = BuildLinks(content); + + if (links.Count > 0) + db.Database.ExecuteSqlCommand("UPDATE Resources SET DownloadCount = DownloadCount+1 WHERE ResourceID={0}", resource.ResourceID); + else + db.Database.ExecuteSqlCommand("UPDATE Resources SET NoLinkDownloadCount = NoLinkDownloadCount+1 WHERE ResourceID={0}", resource.ResourceID); + return true; + } + + public static List BuildLinks(ResourceContentFile content) + { + // missions: URLs are baked in by MissionUpdater + if (content.Resource.MissionID != null) + return string.IsNullOrEmpty(content.Links) + ? new List() + : content.Links.Split('\n').Where(s => s.Length > 0).ToList(); + + if (content.FileName.EndsWith(".sdp")) return new List(); + + var subfolder = content.Resource.TypeID == ResourceType.Map ? "maps" : "games"; + var localUrl = $"{GlobalConst.BaseSiteUrl}/content/{subfolder}/{content.FileName}"; + var springfilesUrl = $"{GlobalConst.SpringfilesBaseUrl}files/{subfolder}/{content.FileName}"; + + var links = new List(); + if (File.Exists(Global.MapPath($"~/content/{subfolder}/{content.FileName}"))) links.Add(localUrl); + if (IsSpringfilesAvailable(content, springfilesUrl)) links.Add(springfilesUrl); + return links; + } + + // 24 h cache + dedup: at most one HEAD per Md5 in flight regardless of caller count. + // LinkCount in DB encodes the cached probe result (1 = springfiles had it, 0 = it didn't). + static bool IsSpringfilesAvailable(ResourceContentFile content, string url) + { + if (content.Resource.LastLinkCheck.HasValue + && DateTime.UtcNow.Subtract(content.Resource.LastLinkCheck.Value).TotalHours < SpringfilesRecheckHours) + return content.LinkCount > 0; + + var md5 = content.Md5; + return InflightProbes.GetOrAdd(md5, key => new Lazy( + () => RunProbe(key), + LazyThreadSafetyMode.ExecutionAndPublication)).Value; + } + + // The cache re-check inside the using block covers the gap between Lazy completion + // and InflightProbes eviction — a caller arriving in that window creates a new Lazy + // but the new probe sees the just-persisted LastLinkCheck and skips the HEAD. + static bool RunProbe(string md5) + { + try + { + using (var db = new ZkDataContext()) + { + var cf = db.ResourceContentFiles.SingleOrDefault(x => x.Md5 == md5); + if (cf == null) return false; + if (cf.Resource.LastLinkCheck.HasValue + && DateTime.UtcNow.Subtract(cf.Resource.LastLinkCheck.Value).TotalHours < SpringfilesRecheckHours) + return cf.LinkCount > 0; + + var ok = ProbeAndAssign(cf); + db.SaveChanges(); + return ok; + } + } + catch (Exception ex) + { + Trace.TraceWarning("ResourceLinkProvider: probe failed for {0}: {1}", md5, ex.Message); + return false; + } + finally + { + Lazy _; + InflightProbes.TryRemove(md5, out _); + } + } + + // called once at registration time from PlasmaServer.RegisterResource; + // caller's db.SaveChanges() persists. Runs synchronously so newly-registered + // maps appear immediately on Detail pages without waiting for a first request. + internal static void UpdateLinks(ResourceContentFile content) + { + if (content.FileName.EndsWith(".sdp") || content.Resource.MissionID != null) return; + ProbeAndAssign(content); + } + + // HEAD-probes springfiles, then writes Links/LinkCount/LastLinkCheck on the entity. + // Caller is responsible for SaveChanges (or accepting unsaved entity state). + // Returns true iff springfiles has the file at the expected length. + static bool ProbeAndAssign(ResourceContentFile cf) + { + var subfolder = cf.Resource.TypeID == ResourceType.Map ? "maps" : "games"; + var localUrl = $"{GlobalConst.BaseSiteUrl}/content/{subfolder}/{cf.FileName}"; + var springfilesUrl = $"{GlobalConst.SpringfilesBaseUrl}files/{subfolder}/{cf.FileName}"; + + var ok = HeadCheck(springfilesUrl, cf.Length); + + var links = new List(); + if (File.Exists(Global.MapPath($"~/content/{subfolder}/{cf.FileName}"))) links.Add(localUrl); + if (ok) links.Add(springfilesUrl); + + cf.Links = string.Join("\n", links); + cf.LinkCount = ok ? 1 : 0; + cf.Resource.LastLinkCheck = DateTime.UtcNow; + return ok; + } + + static bool HeadCheck(string url, int length) + { + try + { + var wr = (HttpWebRequest)WebRequest.Create(url); + wr.Timeout = 5000; + wr.Method = "HEAD"; + using (var res = wr.GetResponse()) return res.ContentLength == length; + } + catch (Exception ex) + { + Trace.TraceWarning("Springfiles HEAD failed for {0}: {1}", url, ex.Message); + return false; + } + } + } +} diff --git a/Zero-K.info/Controllers/MapsController.cs b/Zero-K.info/Controllers/MapsController.cs index 7f0873993..76b400f57 100644 --- a/Zero-K.info/Controllers/MapsController.cs +++ b/Zero-K.info/Controllers/MapsController.cs @@ -452,12 +452,18 @@ public ActionResult UploadResource(HttpPostedFileBase file, bool specialMap) using (var db = new ZkDataContext()) { var resource = db.Resources.FirstOrDefault(x => x.InternalName == res.ResourceInfo.Name); - var contentFile = resource.ResourceContentFiles.FirstOrDefault(x => x.FileName == file.FileName); - contentFile.Links = $"{GlobalConst.BaseSiteUrl}/content/{subfolder}/{file.FileName}"; - contentFile.LinkCount = 1; + // case-insensitive FileName match: DB row may have different casing than the + // freshly-uploaded file (springfiles vs original-upload casing) — see issue #3052 + var contentFile = resource?.ResourceContentFiles.FirstOrDefault( + x => string.Equals(x.FileName, file.FileName, StringComparison.OrdinalIgnoreCase)); + if (contentFile != null) + { + contentFile.Links = $"{GlobalConst.BaseSiteUrl}/content/{subfolder}/{file.FileName}"; + contentFile.LinkCount = 1; + } // tag as special if required - if (res.Status == UnitSyncer.ResourceFileStatus.Registered && specialMap) + if (resource != null && res.Status == UnitSyncer.ResourceFileStatus.Registered && specialMap) { resource.MapIsSpecial = true; }