diff --git a/UT4MasterServer.Models/DTO/Response/LogReportCreatedResponse.cs b/UT4MasterServer.Models/DTO/Response/LogReportCreatedResponse.cs new file mode 100644 index 00000000..69ac112c --- /dev/null +++ b/UT4MasterServer.Models/DTO/Response/LogReportCreatedResponse.cs @@ -0,0 +1,14 @@ +using System.Text.Json.Serialization; + +namespace UT4MasterServer.Models.DTO.Responses; + +public sealed class LogReportCreatedResponse +{ + [JsonPropertyName("reportId")] + public string ReportID { get; set; } + + public LogReportCreatedResponse(string reportID) + { + ReportID = reportID; + } +} diff --git a/UT4MasterServer.Models/DTO/Response/LogReportResponse.cs b/UT4MasterServer.Models/DTO/Response/LogReportResponse.cs new file mode 100644 index 00000000..a898fcb1 --- /dev/null +++ b/UT4MasterServer.Models/DTO/Response/LogReportResponse.cs @@ -0,0 +1,35 @@ +using System.Text.Json.Serialization; +using UT4MasterServer.Models.Database; + +namespace UT4MasterServer.Models.DTO.Responses; + +public sealed class LogReportResponse +{ + [JsonPropertyName("reportId")] + public string ReportID { get; set; } + + [JsonPropertyName("createdAt")] + public DateTime CreatedAt { get; set; } + + [JsonPropertyName("player")] + public string? Player { get; set; } + + [JsonPropertyName("note")] + public string? Note { get; set; } + + [JsonPropertyName("length")] + public long Length { get; set; } + + [JsonPropertyName("contentType")] + public string ContentType { get; set; } + + public LogReportResponse(LogReport report) + { + ReportID = report.ReportID; + CreatedAt = report.CreatedAt; + Player = report.Player; + Note = report.Note; + Length = report.Length; + ContentType = report.ContentType; + } +} diff --git a/UT4MasterServer.Models/Database/LogReport.cs b/UT4MasterServer.Models/Database/LogReport.cs new file mode 100644 index 00000000..61c647c9 --- /dev/null +++ b/UT4MasterServer.Models/Database/LogReport.cs @@ -0,0 +1,46 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace UT4MasterServer.Models.Database; + +/// +/// Metadata about a single uploaded bug-report log file. +/// The log file bytes themselves are stored in a GridFS bucket and referenced by . +/// +public sealed class LogReport +{ + [BsonId, BsonIgnoreIfDefault, BsonElement("_id"), BsonRepresentation(BsonType.ObjectId)] + public string ID { get; set; } = default!; + + /// + /// Short, human-friendly identifier of this report (8 uppercase Crockford base32 characters). + /// + public string ReportID { get; set; } = string.Empty; + + /// + /// ID of the GridFS file containing the uploaded log bytes. + /// + public ObjectId FileID { get; set; } + + [BsonIgnoreIfNull] + public string? Player { get; set; } + + [BsonIgnoreIfNull] + public string? Note { get; set; } + + /// + /// SHA-256 hash (lowercase hex) of the uploader's IP address. The raw IP is never stored. + /// + public string ClientIPHash { get; set; } = string.Empty; + + public string UserAgent { get; set; } = string.Empty; + + public DateTime CreatedAt { get; set; } = DateTime.UtcNow; + + /// + /// Size of the uploaded log file in bytes. + /// + public long Length { get; set; } + + public string ContentType { get; set; } = string.Empty; +} diff --git a/UT4MasterServer.Models/Settings/LogReportSettings.cs b/UT4MasterServer.Models/Settings/LogReportSettings.cs new file mode 100644 index 00000000..feb6bc84 --- /dev/null +++ b/UT4MasterServer.Models/Settings/LogReportSettings.cs @@ -0,0 +1,24 @@ +namespace UT4MasterServer.Models.Settings; + +public sealed class LogReportSettings +{ + /// + /// Name of the GridFS bucket in which uploaded log files are stored. + /// + public string BucketName { get; set; } = "bugreports"; + + /// + /// Maximum allowed size of a single uploaded log file in bytes. + /// + public int MaxFileSizeBytes { get; set; } = 25 * 1024 * 1024; + + /// + /// Maximum number of uploads a single client IP may perform within . + /// + public int RateLimitMaxUploads { get; set; } = 10; + + /// + /// Size of the sliding rate-limit window in minutes. + /// + public int RateLimitWindowMinutes { get; set; } = 60; +} diff --git a/UT4MasterServer.Services/Hosted/ApplicationStartupService.cs b/UT4MasterServer.Services/Hosted/ApplicationStartupService.cs index b9f021dc..243524fa 100644 --- a/UT4MasterServer.Services/Hosted/ApplicationStartupService.cs +++ b/UT4MasterServer.Services/Hosted/ApplicationStartupService.cs @@ -14,13 +14,16 @@ public sealed class ApplicationStartupService : IHostedService private readonly CloudStorageService cloudStorageService; private readonly ClientService clientService; private readonly RatingsService ratingsService; + private readonly LogReportService logReportService; public ApplicationStartupService( ILogger logger, ILogger statsLogger, IOptions settings, ILogger cloudStorageLogger, - ILogger ratingsLogger) + ILogger ratingsLogger, + ILogger logReportLogger, + IOptions logReportSettings) { this.logger = logger; var db = new DatabaseContext(settings); @@ -29,6 +32,7 @@ public ApplicationStartupService( cloudStorageService = new CloudStorageService(db, cloudStorageLogger); clientService = new ClientService(db); ratingsService = new RatingsService(ratingsLogger, db); + logReportService = new LogReportService(logReportLogger, db, logReportSettings); } public async Task StartAsync(CancellationToken cancellationToken) @@ -37,6 +41,7 @@ public async Task StartAsync(CancellationToken cancellationToken) await accountService.CreateIndexesAsync(); await statisticsService.CreateIndexesAsync(); await ratingsService.CreateIndexesAsync(); + await logReportService.CreateIndexesAsync(); logger.LogInformation("Initializing MongoDB CloudStorage."); await cloudStorageService.EnsureSystemFilesExistAsync(); diff --git a/UT4MasterServer.Services/Scoped/LogReportService.cs b/UT4MasterServer.Services/Scoped/LogReportService.cs new file mode 100644 index 00000000..79348b72 --- /dev/null +++ b/UT4MasterServer.Services/Scoped/LogReportService.cs @@ -0,0 +1,153 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; +using MongoDB.Driver.GridFS; +using System.Security.Cryptography; +using UT4MasterServer.Models.Database; +using UT4MasterServer.Models.DTO.Responses; +using UT4MasterServer.Models.Settings; + +namespace UT4MasterServer.Services.Scoped; + +/// +/// Stores and retrieves bug-report log files. Log bytes are kept in a GridFS +/// bucket while a small metadata document per report is kept in the +/// "logreports" collection for cheap listing. +/// +public sealed class LogReportService +{ + /// + /// Crockford base32 alphabet (no I, L, O, U) used for human-friendly report IDs. + /// + private const string ReportIDAlphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"; + private const int ReportIDLength = 8; + private const int MaxReportIDGenerationAttempts = 10; + + private readonly ILogger logger; + private readonly IMongoCollection logReportCollection; + private readonly GridFSBucket bucket; + + public LogReportService( + ILogger logger, + DatabaseContext dbContext, + IOptions settings) + { + this.logger = logger; + logReportCollection = dbContext.Database.GetCollection("logreports"); + bucket = new GridFSBucket(dbContext.Database, new GridFSBucketOptions() + { + BucketName = settings.Value.BucketName + }); + } + + public async Task CreateIndexesAsync() + { + var indexes = new[] + { + new CreateIndexModel( + Builders.IndexKeys.Ascending(x => x.ReportID), + new CreateIndexOptions() { Unique = true }), + new CreateIndexModel( + Builders.IndexKeys.Descending(x => x.CreatedAt)) + }; + await logReportCollection.Indexes.CreateManyAsync(indexes); + } + + public async Task CreateReportAsync( + Stream logStream, long length, string? player, string? note, + string clientIPHash, string userAgent, string contentType) + { + var reportID = await GenerateUniqueReportIDAsync(); + + ObjectId fileID = await bucket.UploadFromStreamAsync(reportID, logStream, new GridFSUploadOptions() + { + Metadata = new BsonDocument() + { + { "ReportID", reportID }, + { "ContentType", contentType } + } + }); + + var report = new LogReport() + { + ReportID = reportID, + FileID = fileID, + Player = player, + Note = note, + ClientIPHash = clientIPHash, + UserAgent = userAgent, + CreatedAt = DateTime.UtcNow, + Length = length, + ContentType = contentType + }; + + await logReportCollection.InsertOneAsync(report); + + logger.LogInformation("Stored bug report {ReportID} ({Length} bytes).", reportID, length); + + return report; + } + + public async Task> ListReportsAsync(int skip, int limit) + { + FilterDefinition? filter = Builders.Filter.Empty; + + var count = await logReportCollection.CountDocumentsAsync(filter); + List? data = await logReportCollection + .Find(filter) + .SortByDescending(x => x.CreatedAt) + .Skip(skip) + .Limit(limit) + .ToListAsync(); + + return new PagedResponse() + { + Count = count, + Data = data + }; + } + + public async Task GetReportAsync(string reportID) + { + IAsyncCursor? cursor = await logReportCollection.FindAsync( + Builders.Filter.Eq(x => x.ReportID, reportID)); + return await cursor.SingleOrDefaultAsync(); + } + + public async Task OpenLogStreamAsync(LogReport report) + { + return await bucket.OpenDownloadStreamAsync(report.FileID); + } + + private async Task GenerateUniqueReportIDAsync() + { + for (var attempt = 0; attempt < MaxReportIDGenerationAttempts; attempt++) + { + var reportID = GenerateReportID(); + + // the unique index on ReportID is the final guarantee, this check + // just avoids an insert failure in the common case + var existing = await logReportCollection.CountDocumentsAsync( + Builders.Filter.Eq(x => x.ReportID, reportID)); + if (existing == 0) + { + return reportID; + } + } + + throw new InvalidOperationException("Failed to generate a unique report ID"); + } + + private static string GenerateReportID() + { + // 256 % 32 == 0, so mapping bytes onto the alphabet introduces no modulo bias + var bytes = RandomNumberGenerator.GetBytes(ReportIDLength); + var chars = new char[ReportIDLength]; + for (var i = 0; i < ReportIDLength; i++) + { + chars[i] = ReportIDAlphabet[bytes[i] % ReportIDAlphabet.Length]; + } + return new string(chars); + } +} diff --git a/UT4MasterServer.Services/Singleton/LogReportRateLimitService.cs b/UT4MasterServer.Services/Singleton/LogReportRateLimitService.cs new file mode 100644 index 00000000..8db9d6ae --- /dev/null +++ b/UT4MasterServer.Services/Singleton/LogReportRateLimitService.cs @@ -0,0 +1,60 @@ +namespace UT4MasterServer.Services.Singleton; + +/// +/// Simple in-memory sliding-window rate limiter for anonymous log uploads. +/// Chosen over an external store on purpose: if master dies, the counters +/// disappear, which is acceptable for abuse protection of a low-volume +/// endpoint (same reasoning as keeping codes in memory). +/// +public sealed class LogReportRateLimitService +{ + private const int CleanupThreshold = 1024; + + private readonly Dictionary> uploadsPerClient = new(); + + /// + /// Records an upload attempt for and returns + /// whether the attempt is allowed to proceed. + /// + public bool TryRecordUpload(string clientKey, int maxUploads, TimeSpan window) + { + var now = DateTime.UtcNow; + + lock (uploadsPerClient) // Make sure counters are thread-safe + { + if (uploadsPerClient.Count >= CleanupThreshold) + { + RemoveStaleClients(now, window); + } + + if (!uploadsPerClient.TryGetValue(clientKey, out List? timestamps)) + { + timestamps = new List(); + uploadsPerClient.Add(clientKey, timestamps); + } + + timestamps.RemoveAll(x => now - x > window); + + if (timestamps.Count >= maxUploads) + { + return false; + } + + timestamps.Add(now); + return true; + } + } + + private void RemoveStaleClients(DateTime now, TimeSpan window) + { + var staleKeys = uploadsPerClient + .Where(x => !x.Value.Any(t => now - t <= window)) + .Select(x => x.Key) + .ToList(); + + foreach (var key in staleKeys) + { + uploadsPerClient.Remove(key); + } + } +} diff --git a/UT4MasterServer.Services/UT4MasterServer.Services.csproj b/UT4MasterServer.Services/UT4MasterServer.Services.csproj index 33e9edf4..54f1c872 100644 --- a/UT4MasterServer.Services/UT4MasterServer.Services.csproj +++ b/UT4MasterServer.Services/UT4MasterServer.Services.csproj @@ -9,6 +9,7 @@ + diff --git a/UT4MasterServer/Controllers/UT/LogsController.cs b/UT4MasterServer/Controllers/UT/LogsController.cs new file mode 100644 index 00000000..ac56eafc --- /dev/null +++ b/UT4MasterServer/Controllers/UT/LogsController.cs @@ -0,0 +1,211 @@ +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Options; +using System.Net; +using System.Security.Cryptography; +using System.Text; +using UT4MasterServer.Authentication; +using UT4MasterServer.Common.Enums; +using UT4MasterServer.Common.Helpers; +using UT4MasterServer.Models.Database; +using UT4MasterServer.Models.DTO.Responses; +using UT4MasterServer.Models.Settings; +using UT4MasterServer.Services.Scoped; +using UT4MasterServer.Services.Singleton; + +namespace UT4MasterServer.Controllers.UT; + +/// +/// Bug-report / log-file ingest endpoint. +/// Uploading is anonymous (rate-limited), retrieval requires admin privileges. +/// +[ApiController] +[Route("ut/api/logs")] +[AuthorizeBearer] +[Produces("application/json")] +public sealed class LogsController : JsonAPIController +{ + /// + /// Hard cap on the whole multipart request body. Slightly above + /// to leave room for multipart overhead. + /// + private const long MaxRequestSizeBytes = 26 * 1024 * 1024; + + private const int MaxPlayerLength = 64; + private const int MaxNoteLength = 2048; + private const int MaxUserAgentLength = 256; + + private static readonly string[] knownContentTypes = new string[] + { + "text/plain", + "application/gzip", + "application/x-gzip" + }; + + private readonly LogReportService logReportService; + private readonly LogReportRateLimitService rateLimitService; + private readonly AccountService accountService; + private readonly IOptions applicationSettings; + private readonly IOptions logReportSettings; + + public LogsController( + ILogger logger, + LogReportService logReportService, + LogReportRateLimitService rateLimitService, + AccountService accountService, + IOptions applicationSettings, + IOptions logReportSettings) : base(logger) + { + this.logReportService = logReportService; + this.rateLimitService = rateLimitService; + this.accountService = accountService; + this.applicationSettings = applicationSettings; + this.logReportSettings = logReportSettings; + } + + [AllowAnonymous] + [HttpPost] + [RequestSizeLimit(MaxRequestSizeBytes)] + public async Task UploadLog() + { + LogReportSettings settings = logReportSettings.Value; + + // validate content length before touching the body + if (Request.ContentLength == null || Request.ContentLength <= 0) + { + return BadRequest("Missing request body"); + } + if (Request.ContentLength > MaxRequestSizeBytes) + { + return StatusCode(StatusCodes.Status413PayloadTooLarge, "Upload is too large"); + } + + IPAddress? ip = GetClientIP(applicationSettings); + if (ip == null) + { + return BadRequest("Could not determine client address"); + } + + var ipHash = HashIP(ip); + var window = TimeSpan.FromMinutes(settings.RateLimitWindowMinutes); + if (!rateLimitService.TryRecordUpload(ipHash, settings.RateLimitMaxUploads, window)) + { + logger.LogWarning("Rate-limited bug report upload from client {ClientIPHash}.", ipHash); + return StatusCode(StatusCodes.Status429TooManyRequests, "Too many uploads, try again later"); + } + + IFormCollection? formCollection = await Request.ReadFormAsync(); + IFormFile? file = formCollection.Files.GetFile("log"); + if (file == null) + { + return BadRequest("Missing 'log' file"); + } + if (file.Length <= 0) + { + return BadRequest("Cannot upload empty file"); + } + if (file.Length > settings.MaxFileSizeBytes) + { + return StatusCode(StatusCodes.Status413PayloadTooLarge, "Log file is too large"); + } + + var player = Truncate(formCollection["player"].ToString(), MaxPlayerLength); + var note = Truncate(formCollection["note"].ToString(), MaxNoteLength); + var userAgent = Truncate(Request.Headers.UserAgent.ToString(), MaxUserAgentLength) ?? string.Empty; + var contentType = knownContentTypes.Contains(file.ContentType) ? file.ContentType : "application/octet-stream"; + + LogReport report; + using (Stream? stream = file.OpenReadStream()) + { + report = await logReportService.CreateReportAsync(stream, file.Length, player, note, ipHash, userAgent, contentType); + } + + // intentionally return only the report id, never any stored content + return Ok(new LogReportCreatedResponse(report.ReportID)); + } + + [HttpGet] + public async Task ListReports(int skip = 0, int limit = 50) + { + await VerifyAccessAsync(AccountFlags.ACL_Maintenance); + + if (skip < 0) skip = 0; + if (limit < 1 || limit > 100) limit = 100; + + PagedResponse? reports = await logReportService.ListReportsAsync(skip, limit); + + return Ok(new PagedResponse() + { + Count = reports.Count, + Data = reports.Data.Select(x => new LogReportResponse(x)).ToList() + }); + } + + [HttpGet("{reportId}"), Produces("application/octet-stream")] + public async Task GetReport(string reportId) + { + await VerifyAccessAsync(AccountFlags.ACL_Maintenance); + + reportId = reportId.Trim().ToUpperInvariant(); + if (!IsValidReportID(reportId)) + { + return BadRequest("Invalid report id"); + } + + LogReport? report = await logReportService.GetReportAsync(reportId); + if (report == null) + { + return NotFound(new ErrorResponse() { ErrorMessage = "Report not found" }); + } + + Stream stream = await logReportService.OpenLogStreamAsync(report); + + var extension = report.ContentType is "application/gzip" or "application/x-gzip" ? ".log.gz" : ".log"; + return File(stream, report.ContentType, report.ReportID + extension); + } + + private async Task<(Session Session, Account Account)> VerifyAccessAsync(params AccountFlags[] aclAny) + { + if (User.Identity is not EpicUserIdentity user) + { + throw new UnauthorizedAccessException("User not logged in"); + } + + Account? account = await accountService.GetAccountAsync(user.Session.AccountID); + if (account == null) + { + throw new UnauthorizedAccessException("User not found"); + } + + AccountFlags combinedAcl = aclAny.Aggregate((x, y) => x | y) | AccountFlags.Admin; + + if (!account.Flags.HasFlagAny(combinedAcl)) + { + throw new UnauthorizedAccessException("User has insufficient privileges"); + } + + return (user.Session, account); + } + + private static bool IsValidReportID(string reportID) + { + return reportID.Length == 8 && reportID.All(x => x is (>= '0' and <= '9') or (>= 'A' and <= 'Z')); + } + + private static string HashIP(IPAddress ip) + { + var hashedBytes = SHA256.HashData(Encoding.UTF8.GetBytes(ip.ToString())); + return Convert.ToHexString(hashedBytes).ToLower(); + } + + private static string? Truncate(string value, int maxLength) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + value = value.Trim(); + return value.Length <= maxLength ? value : value[..maxLength]; + } +} diff --git a/UT4MasterServer/Program.cs b/UT4MasterServer/Program.cs index 4f8b9e69..3a8e7ba3 100644 --- a/UT4MasterServer/Program.cs +++ b/UT4MasterServer/Program.cs @@ -64,7 +64,8 @@ public static void Main(string[] args) builder.Services .Configure(builder.Configuration.GetSection("ApplicationSettings")) .Configure(builder.Configuration.GetSection("StatisticsSettings")) - .Configure(builder.Configuration.GetSection("ReCaptchaSettings")); + .Configure(builder.Configuration.GetSection("ReCaptchaSettings")) + .Configure(builder.Configuration.GetSection("LogReportSettings")); builder.Services.Configure(x => { @@ -115,13 +116,15 @@ public static void Main(string[] args) .AddScoped() .AddScoped() .AddScoped() - .AddScoped(); + .AddScoped() + .AddScoped(); // services whose instance is created once and are persistent builder.Services .AddSingleton() .AddSingleton() - .AddSingleton(); + .AddSingleton() + .AddSingleton(); // hosted services builder.Services