From 4ea8bfeeba0cb4b3a7870094f0a61031fdaa5943 Mon Sep 17 00:00:00 2001 From: OC Date: Sat, 2 May 2026 19:02:35 -0700 Subject: [PATCH] Add CodeSheriff results to offline benchmark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeSheriff is an AI code safety scanner focused on detecting bugs introduced by AI coding assistants. Self-improving detection via autotune feedback loop. Evaluated on 49/50 benchmark PRs using the official pipeline (steps 2-5). Results: - Claude Opus 4.5 judge: F1=64.6% (P=55.3%, R=77.6%) - Claude Sonnet 4.5 judge: F1=64.2% (P=55.1%, R=76.9%) - Average F1: 64.4% Ranked #1 across both judge models on this evaluation (vs Cubic v2 61.8%/61.4%, Augment 53.5%/53.4%). Website: https://thecodesheriff.com GitHub App: https://github.com/apps/codesheriff-review Rebased onto latest main (2026-05-02). All existing tool results preserved — only codesheriff entries added to candidates.json, evaluations.json, and benchmark_data.json. --- offline/README.md | 1 + .../candidates.json | 1264 ++++++++ .../evaluations.json | 2882 ++++++++++++++--- .../candidates.json | 1264 ++++++++ .../evaluations.json | 2845 +++++++++++++--- offline/results/benchmark_data.json | 446 +++ 6 files changed, 7925 insertions(+), 777 deletions(-) diff --git a/offline/README.md b/offline/README.md index 3aa9456..173a28e 100644 --- a/offline/README.md +++ b/offline/README.md @@ -8,6 +8,7 @@ Open replication of the code review benchmark used by companies like [Augment](h |---|---| | [Augment](https://www.augmentcode.com/) | AI code review | | [Claude Code](https://claude.ai) | AI assistant | +| [CodeSheriff](https://thecodesheriff.com/) | AI code review | | [CodeRabbit](https://www.coderabbit.ai/) | AI code review | | [Codex](https://openai.com/codex) | AI assistant | | [Cursor Bugbot](https://cursor.com) | AI code review | diff --git a/offline/results/anthropic_claude-opus-4-5-20251101/candidates.json b/offline/results/anthropic_claude-opus-4-5-20251101/candidates.json index 730f434..a5877df 100644 --- a/offline/results/anthropic_claude-opus-4-5-20251101/candidates.json +++ b/offline/results/anthropic_claude-opus-4-5-20251101/candidates.json @@ -1020,6 +1020,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Incorrect permission check in canManage() method - the method checks the wrong permission scope, allowing users who should not have manage access to pass the authorization check", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "In getGroupIdsWithViewPermission, hasPermission is called with groupResource.getId() for both the resource and scope arguments - the scope parameter should be different from the resource ID, causing incorrect permission evaluation", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "In getGroupIdsWithViewPermission, hasPermission is called with groupResource.getId() for both resource and scope parameters - scope should differ from resource ID, causing incorrect permission evaluation and potentially granting wrong access", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Authorization bypass in getGroupIdsWithViewPermission() - returns empty set when user has view permission, which may grant access to all groups instead of restricting access", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Incomplete constant name 'VIEW_MEMBE' in the GROUPS ResourceType definition - should be 'VIEW_MEMBERS'", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/keycloak/keycloak/pull/33832": { @@ -2019,6 +2051,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Method returns wrong provider - returns default keystore provider instead of BouncyCastle provider, causing callers to use incorrect cryptographic implementation", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Dead code: ASN1Encoder instances are created and written to but their results are immediately discarded - the actual encoding is performed elsewhere, making these encoder operations unnecessary and confusing", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "KeyPairGenerator.getInstance(\"EC\").genKeyPair() creates a key pair with default parameters, but ECDSA algorithms require specific curve parameters (P-256, P-384, P-521) that may not match the default curve", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "String concatenation creates trailing comma and space that are never removed, resulting in malformed output", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Method appears to call itself recursively instead of delegating to a wrapped or parent object, which can cause infinite recursion and StackOverflowError", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/keycloak/keycloak/pull/40940": { @@ -2847,6 +2911,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Returning null from getSubGroupsCount() violates the GroupModel contract - Javadoc declares the method never returns null but this implementation returns null, causing NullPointerException in callers that invoke methods on the result", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Reader thread is not awaited before assertion - flipping deletedAll to true and immediately asserting races with the reader thread adding exceptions, causing flaky test failures", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Method appears to call itself recursively instead of delegating to a wrapped or parent object, causing potential infinite recursion and StackOverflowError", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Response.Status.Family.SUCCESSFUL includes status codes 200-299, but group creation should specifically check for 201 CREATED status code", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Response object is used in try-with-resources but the groupUuid extraction happens inside the try block where response might be closed", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1": { @@ -3673,6 +3769,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "ConditionalPasskeysEnabled() is called without the required UserModel parameter - the method signature requires a user argument but is invoked without one, causing a compilation error or NullPointerException", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "With isConditionalPasskeysEnabled(UserModel user) requiring user != null, authenticate() will not call webauthnAuth.fillContextForm(context) on the correct branch when user is null, causing passkey authentication to silently fail", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "WebAuthn authenticators are exposed based solely on user existence without proper authentication state validation, potentially allowing enumeration of valid usernames and their associated WebAuthn credentials", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Authentication bypass vulnerability - the authenticate method calls context.success() without verifying user credentials when user is already set and no linked brokers exist", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Boolean.parseBoolean() returns false for null input, but the code should likely treat null as false explicitly or handle it differently since AUTH_NOTE values can be null", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/getsentry/sentry/pull/93824": { @@ -4547,6 +4675,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Breaking out of the loop when deadline is elapsed skips terminating remaining flusher processes, potentially leaking process handles and causing resource exhaustion", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "time.sleep in test_consumer.py won't actually wait because time.sleep was monkeypatched above - test timing assumptions are broken, making the test unreliable", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "flusher processes created via multiprocessing.get_context(spawn).Process are instances of SpawnProcess which on POSIX is not a subclass of multiprocessing.Process - isinstance checks will fail and process management may break", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Inconsistent metric tagging: some calls use tag 'shard' while others use 'shards' - metrics will be split across two different tag values, making aggregation and dashboards incorrect", + "path": null, + "line": null, + "source": "logicbug_added" + } ] }, "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5": { @@ -7184,6 +7338,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "When requests are authenticated with API keys or org auth tokens, organization_context.member is None and accessing .has_global_access() on None will raise AttributeError", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Importing OptimizedCursorPaginator which does not exist in the sentry codebase - this import will fail at runtime with ImportError", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Negative offset passed to queryset slicing will cause runtime error - Python slice indices must be non-negative", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Math.floor/ceil applied to potentially non-numeric values without type checking will raise TypeError", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/97529": { @@ -7946,6 +8126,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Race condition in BuildIndex: multiple goroutines can concurrently build the same index for the same key without synchronization, wasting resources and potentially causing data corruption", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Race condition: s.search.TotalDocs() iterates the cache without synchronization while the event watcher goroutine concurrently modifies it - concurrent map read/write causes panic in Go", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Hard-coded port assignment may cause port conflicts in concurrent test execution", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Delete function checks for ResourceVersion < 0 but the error message says 'update must include the previous version' which is incorrect for a delete operation", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/getsentry/sentry/pull/80168": { @@ -8689,6 +8895,20 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "MetricAlertDetectorHandler inherits from StatefulDetectorHandler but the class body only contains pass, failing to implement required abstract methods - will raise NotImplementedError at runtime", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Method docstring says it returns a list of DetectorEvaluationResult but the implementation now returns a dict keyed by DetectorGroupKey - documentation and return type are inconsistent", + "path": null, + "line": null, + "source": "logicbug_added" + } ] }, "https://github.com/getsentry/sentry/pull/80528": { @@ -9208,6 +9428,20 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Function modifies the config variable to include display values but then returns the original monitor.config instead of the modified config, discarding the changes", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Slice operation `previous_checkins[:failure_issue_threshold]` occurs before reverse operation, taking most recent N check-ins instead of oldest N check-ins, resulting in examining wrong subset for threshold evaluation", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/getsentry/sentry/pull/77754": { @@ -9822,6 +10056,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Mutable default argument in dataclass field - using a mutable object as default value for timestamp field shares state across instances, causing subtle data corruption bugs", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "to_dict() returns a datetime object for the queued field - if this dict is passed as task kwargs to apply_async(), JSON serialization will fail since datetime is not JSON-serializable by default", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Method name typo: test_from_dict_inalid_data is missing a letter and should be test_from_dict_invalid_data - the test may not be discovered by test runners that filter by name pattern", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Method named 'empty_array' but tests an empty dict - test name does not match what is being tested, making the test suite misleading and harder to maintain", + "path": null, + "line": null, + "source": "logicbug_added" + } ] }, "https://github.com/getsentry/sentry/pull/95633": { @@ -10643,6 +10903,14 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "queue.shutdown() with 'immediate=False' parameter may not exist in the standard Python queue.Queue module - calling a non-existent method will raise AttributeError at runtime", + "path": null, + "line": null, + "source": "logicbug_added" + } ] }, "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2": { @@ -12012,6 +12280,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "OptimizedCursorPaginator negative-offset branch slices QuerySet with a negative start index - Django QuerySets do not support negative indexing and will raise AssertionError", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "BasePaginator negative-offset branch slices QuerySet with a negative start index - Django raises AssertionError when negative slice indices are used on QuerySets", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "OptimizedCursorPaginator.get_item_key applies floor and ceil to a datetime key when order_by is -datetime - math.floor and math.ceil raise TypeError when given a datetime object instead of a number", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Span with span_id='b'*16 has parent_span_id=None but is_segment_span=True, while other spans reference 'b'*16 as their parent, creating inconsistent parent-child relationship", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Dictionary comprehension uses span.payload as key, but span.payload is bytes which is not hashable and cannot be used as a Redis hash key", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3": { @@ -13169,6 +13469,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "sample_rate = 0.0 is falsy in Python - the condition if client_sample_rate evaluates to False for 0.0, silently skipping the zero sample rate case", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Python's built-in hash() is non-deterministic across processes due to PYTHONHASHSEED - cache keys built with hash() will not match across workers, causing cache misses in distributed environments", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Upsampling eligibility check passes the outer dataset instead of the scoped_dataset actually used by the query - when both datasets differ, the eligibility check is performed on wrong data and may incorrectly enable or disable upsampling", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "The condition `if client_sample_rate:` will evaluate to False for the valid sample rate of 0.0, preventing zero sample rates from being set", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/103633": { @@ -14000,6 +14326,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Asymmetric cache trust logic in Check operation: cached permission grants are returned immediately without re-verification, but cache misses always go to the source. An expired or revoked permission may continue to grant access until cache expiry", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "NoopCache implementation always returns cache miss, causing authorization checks to bypass caching entirely and potentially leading to performance degradation that could facilitate DoS attacks on authorization services", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "NoopCache.Get method takes a receiver pointer when it should be a value receiver since the struct has no fields", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Cache key mismatch: setting cache with action 'dashboards:read' but checking with verb 'get' which likely maps to a different action", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/getsentry/sentry/pull/67876": { @@ -14854,6 +15206,26 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Potential null reference when github_authenticated_user state is not set - accessing state without checking existence can raise KeyError or AttributeError", + "path": "src/sentry/integrations/github/webhooks.py", + "line": null, + "source": "logicbug_added" + }, + { + "text": "OAuth state uses static pipeline.signature instead of per-request random nonce, allowing CSRF attacks on the OAuth callback flow", + "path": "src/sentry/integrations/github/webhooks.py", + "line": null, + "source": "logicbug_added" + }, + { + "text": "Missing key existence check before accessing integration.metadata[sender][login] - accessing nested dict key 'sender' without guard will raise KeyError when sender is absent", + "path": "src/sentry/integrations/github/webhooks.py", + "line": null, + "source": "logicbug_added" + } ] }, "https://github.com/keycloak/keycloak/pull/32918": { @@ -15430,6 +15802,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Recursive caching call uses session instead of delegate - the cache method calls session's method which re-enters caching, causing infinite recursion or incorrect cache behavior", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Cleanup reference uses incorrect alias - should be 'idp-alias-' + i but uses 'alias', resulting in cleanup targeting wrong identity provider configuration", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Stream.of(values()).map(LoginFilter::getFilter).reduce(Predicate::and).get() will throw NoSuchElementException if values() returns an empty array", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Potential null pointer exception when this.organization is null but this.organization.getId() is called", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/94942": { @@ -16115,6 +16513,26 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Dead code: NewInMemoryDB().RunCommands and db.QueryFramesInto always return \"not implemented\" - these exported methods are unusable; either implement them or remove them to avoid confusion", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Function always returns false regardless of feature flag state due to inverted logic and incorrect control flow", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Missing return statement after error assignment - execution continues after logging error and setting rsp.Error", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/90939": { @@ -16561,6 +16979,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Critical cache corruption: entryPointAssetsCache is unconditionally overwritten with the fetch result even when the fetch returns an error - a failed fetch will replace valid cached data with an error value", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Incomplete double-checked locking for caching: cache is checked without the lock, then locked to set, but no second check after acquiring lock - concurrent goroutines can both see nil and both fetch the assets", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Race condition: after releasing RLock and before acquiring Lock, another goroutine could modify entryPointAssetsCache, causing the cached value 'ret' to become stale or the cache state to change", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Potential null pointer dereference: 'ret' captures entryPointAssetsCache under RLock but could be nil, and is returned without validation when cfg.Env != setting.Dev", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/80329": { @@ -17513,6 +17957,20 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Ticker is never stopped, causing a goroutine leak when the function returns", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Using log.Error() instead of log.Debug() or log.Info() for normal operational logging during cleanup process", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/90045": { @@ -18953,6 +19411,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Bug: recordLegacyDuration is called when storage operation fails but the correct function to call for storage failures is recordStorageDuration - wrong metric is being recorded on error path", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Context created with d.Log instead of the log variable that was initialized with additional context values - log entries will be missing the name, kind, and method fields", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Inconsistency: metrics recorded using name instead of options.Kind - other methods in this file use options.Kind for metric dimensions, causing this method to emit metrics with a different label format", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Variable 'l' is declared as nil and never assigned a value, but is used to initialize legacyStoreMock", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Variable 's' is declared as nil and never assigned a value, but is used to initialize storageMock", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/106778": { @@ -20098,6 +20588,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "RuleActionsButtons is invoked with only promRule argument but SilenceGrafanaRuleDrawer inside requires a Grafana Ruler rule - passing only a Prometheus rule will cause missing prop error or incorrect drawer behavior", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "GrafanaRuleListItem rendered in a list is missing the required React key prop - missing key causes React reconciliation issues when list order changes, leading to incorrect DOM updates", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Redundant type check 'rule && rule.type === PromRuleType.Alerting' after prometheusRuleType.grafana.alertingRule(rule) already verified this condition", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Function calls prometheusRuleType.grafana.rule() but uses result in boolean context without handling potential undefined/null return values", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/107534": { @@ -20685,6 +21201,20 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Expected expression includes regex pattern '20|10' but mock data returns string values that would be sorted as ['1', '10', '2', '20', '3'], making the first two values '1' and '10' instead of the expected '20' and '10'", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Template variables are applied to filtered queries but the original query objects may be mutated, potentially affecting other parts of the application that reference the same query objects", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/79265": { @@ -21764,6 +22294,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Race condition: multiple concurrent requests can simultaneously pass the device count check and create devices beyond the limit - there is no atomic check-and-increment to prevent concurrent overwrites", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Type error: dbSession.Exec(args...) spreads a []interface{} where the first element is the query string, but Exec expects the query as a separate first argument - this will panic at runtime", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Anonymous authentication now fails entirely when ErrDeviceLimitReached - previously device tagging was asynchronous and non-blocking, now it blocks authentication for users over the device limit", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Error from TagDevice is returned directly when it's ErrDeviceLimitReached, but other errors are only logged as warnings and authentication continues, creating inconsistent error handling where some failures block authentication while others silently proceed", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "SQL injection vulnerability in updateDevice function - query string is concatenated with user-controlled parameters and passed to Exec without proper parameter binding", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9": { @@ -22451,6 +23013,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Thread-safety issue with lazy @loaded_locales - multiple threads can simultaneously check and set the instance variable without synchronization, causing duplicate locale loading or data corruption", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Race condition in ensure_loaded! method - @loaded_locales is initialized and checked in separate operations without synchronization", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Object.send bypasses method visibility (private/protected) and can invoke any method including dangerous ones", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Inconsistent locale loading logic - translate method duplicates the loaded check from ensure_loaded! but doesn't use ensure_loaded! method", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/76186": { @@ -23011,6 +23599,26 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "ContextualLoggerMiddleware methods (QueryData, CallResource, CheckHealth, CollectMetrics) will panic when passed a nil request - dereferencing nil pointer before nil check causes runtime panic", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "FromContext method creates a new TestLogger instance instead of returning the current instance, losing all accumulated test data", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Type assertion to *log.ConcreteLogger without checking if the returned logger is nil, which could cause a panic when accessing methods on a nil pointer", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10": { @@ -24778,6 +25386,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "NoMethodError in EmbeddableHost before_validation callback - calling a method on a nil object before validation will raise NoMethodError and prevent record from being saved", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Migration uses raw SQL to insert embeddable_hosts rows - any existing values containing http:// or https:// scheme prefixes will be duplicated with different formats, creating data inconsistency", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "EmbeddableHostsController update and destroy methods do not validate record existence - EmbeddableHost.where(id: params[:id]).first returns nil when record not found, calling update or destroy on nil raises NoMethodError", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "record_for_host compares lower(host) = ? but does not normalize the parameter case - mixed-case referer hosts like Example.COM will fail to match even though they are equivalent", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Potential NoMethodError on nil - value may be nil from hash lookup or safe navigation, calling methods on nil will raise exception", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7": { @@ -27154,6 +27794,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "findMembers() is now asynchronous and its result is not awaited - the controller returns before member data is available, causing the response to contain empty or stale member information", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Capping next offset at user_count produces empty page when total equals limit - offset equals total, causing the next page to return 0 results when there should be a final page", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "HTTP method mismatch: remove_member test uses PUT but the remove_member controller action expects DELETE - test will always fail with 404 or 405", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "find_group method can return nil from Group.find_by, then guardian.ensure_can_see!(nil) will be called on a nil object", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Potential NoMethodError on nil - value may be nil from hash lookup or safe navigation, and calling methods on nil will raise an exception", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3": { @@ -28030,6 +28702,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "BlockedEmail.should_block_email? has side effects during a read operation - it updates statistics even when only checking whether an email should be blocked, causing unexpected DB writes on reads", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Password length validation uses incorrect condition that allows nil passwords to pass when @password_required is false", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Race condition in should_block? method - multiple concurrent requests for the same email could create inconsistent match_count values due to read-modify-write operation without proper locking", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Method doesn't handle nil or empty email values, will cause NoMethodError when calling =~ on nil", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Missing assignment operator (=) for action_type attribute in Fabricator definition", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5": { @@ -29642,6 +30346,26 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "include_website_name method is missing the required ? suffix - Rails serializers expect include_ conditional methods to end with ? otherwise the attribute is always serialized", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Logic error in subdomain comparison - when website domain is 'example.com' and current hostname is 'forums.example.com', the test expects 'example.com/user' but the logic should likely treat the subdomain as different from the parent domain", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "URI parsing is called twice on the same website string without caching, and the second call lacks error handling that could cause unhandled exceptions", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4": { @@ -31723,6 +32447,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "postMessage targetOrigin should be the origin (scheme+host+port), not the full referrer URL - using the full URL causes the message to be silently dropped by the browser", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "X-Frame-Options set to ALLOWALL completely disables clickjacking protection - referer header validation can be spoofed, leaving the endpoint vulnerable to iframe-based attacks", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "ERB block closes with \"end if\" which is invalid Ruby ERB syntax - this will raise a SyntaxError at render time; it should be \"end\" to close the if block", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Race condition in Redis throttling logic - another process can set key between setnx and expire calls, causing expire to be lost and key to persist indefinitely", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Transaction block does not handle the case where PostCreator.create fails but doesn't raise an exception", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1": { @@ -32970,6 +33726,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "The downsize method is defined twice - the second definition overrides the first, causing dead code and potentially breaking behavior that depended on the original two-argument signature", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Hardcoding maxSizeKB = 10 * 1024 ignores Discourse.SiteSettings max_type_size_kb - client-side limit can diverge from server-side configuration, allowing uploads that exceed the server limit", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Passing 80% as dimensions to gifsicle fails for animated GIFs - the animated path requires absolute pixel dimensions, not percentages, causing animated thumbnail generation to fail", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Image downsizing loop can run indefinitely if OptimizedImage.downsize fails to reduce file size, creating an infinite loop condition", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "tempfile.size is called without checking if tempfile is nil, which will raise NoMethodError if file download failed", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2": { @@ -34056,6 +34844,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Typo in property name: 'stopNotificiationsText' should be 'stopNotificationsText' (missing 'n' in Notifications) - the property will never be found causing silent undefined access", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Race condition: TopicUser.change is called without checking if the current post number is actually newer than the existing last_emailed_post_number", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Logic assumes TopicUser record exists but find_by can return nil, leading to NoMethodError on nil object", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Potential variable name typo: code appears to reference 'topi' instead of 'topic_id' in MessageBus.publish call", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Route 't/:topic_id/summary' will never be reached because it's defined after 't/:slug/:topic_id/summary' route, and Rails matches routes in order", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/22532": { @@ -35252,6 +36072,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "updateManyByCredentialId called with empty data object {} - Prisma @updatedAt decorator only fires when the record has actual field changes, so the updatedAt timestamp will not be updated on cache invalidation", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "sed command uses macOS-specific syntax with empty string after -i flag, will fail on Linux systems", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Race condition potential - if cache statuses are updated between the getCacheStatusByCredentialIds call and the map creation, the cacheStatusMap may contain stale data", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "findUniqueOrThrow method uses SelectedCalendarWhereInput instead of SelectedCalendarWhereUniqueInput, which will cause a type error since findUniqueOrThrow requires unique identifiers", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "deleteCacheMutation is called with credentialId but the mutation expects a different parameter structure based on the trpc endpoint name 'deleteCache'", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/8330": { @@ -36304,6 +37156,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Comparison using === instead of isSame() for Dayjs objects will always be false since they are different object instances", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Variable i is initialized to -1 but the code is incomplete - this suggests a loop that may have off-by-one errors when incrementing/accessing array elements", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Assertion expects results for plus2DateString but the time range query spans from plus1DateString to plus2DateString, potentially missing or incorrectly filtering results", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "startTime calculation duplicates the same dayjs computation twice instead of reusing the result, causing unnecessary work and potential inconsistency", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "endTime calculation duplicates the same dayjs computation twice instead of reusing the result, causing unnecessary work and potential inconsistency", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/14943": { @@ -37007,6 +37891,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "scheduleSMSReminders.ts deletion logic incorrectly deletes non-SMS workflow reminders (Email, WhatsApp) that share the same workflow - the filter condition is too broad and removes reminders of unintended types", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "retryCount reads a possibly stale value with reminder.retryCount + 1 - concurrent requests can both increment from the same base, losing increments; use atomic increment via Prisma update", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "OR condition deletes SMS reminders with retryCount > 1, conflicting with retry logic that increments retryCount on failures", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Using lte (less than or equal) with current date will delete SMS reminders scheduled for exactly the current time, preventing them from being sent if the handler runs at the exact scheduled moment", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Duplicate retry count increment logic - retryCount is incremented both in the 'else' block for failed SMS and in the catch block for exceptions, potentially double-counting failures", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/22345": { @@ -37630,6 +38546,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Logic error: fetching userIdsFromOrg only when teamsFromOrg.length > 0 excludes org-level members for organizations without child teams - org members not in any team will be incorrectly excluded from results", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Logic flaw in condition combining - if both authConditions and filterConditions are falsy, returns NOTHING_CONDITION which will exclude all results", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Constructor accepts InsightsBookingServicePublicOptions but assigns to InsightsBookingServiceOptions type field, causing type mismatch", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Missing await for async cleanup operation - test may complete before cleanup finishes, potentially causing interference with subsequent tests", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/11059": { @@ -39725,6 +40667,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "parseRefreshTokenResponse returns a Zod safeParse result object ({ success, data, error }) not the credential object - callers that persist this directly will store the wrapper object instead of the token data", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "When sync endpoint is used, res is a fetch Response object and res.data does not exist - accessing res?.data will return undefined and token.access_token will be undefined, silently breaking token refresh", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "refreshFunction helper returns a fetch Response object when the sync endpoint path is used, but callers expect a credential object with access_token - res.data does not exist on Response, returning undefined and breaking token refresh silently", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Invalid Zod schema syntax - `z.string().toString()` creates a string literal 'string' as a key, not a dynamic string key pattern", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Incorrect expiry date calculation - adding seconds to milliseconds timestamp", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/7232": { @@ -41033,6 +42007,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "deleteScheduledEmailReminder and deleteScheduledSMSReminder are async functions called without await inside forEach loops - reminder deletions fire-and-forget, errors are silently lost, and cleanup may be incomplete", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "forEach loop calls deleteScheduledEmailReminder and deleteScheduledSMSReminder without awaiting them, causing potential race conditions or silent failures", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Race condition in onChange handler - form.getValues() may not reflect current input value due to asynchronous form updates", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Database delete operation is not awaited but function continues execution, potentially causing race conditions", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "deleteScheduledEmailReminder is called with a third parameter 'true' but deleteScheduledSMSReminder is called without it, suggesting inconsistent API usage", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/14740": { @@ -42457,6 +43463,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Case sensitivity bypass in email blacklist: comparison does not normalize email case, allowing BLOCKED@example.com to bypass a blacklist entry for blocked@example.com", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Email sender called with original guests list - existing attendees included in the input are treated as new invitees, sending duplicate invitations to guests who are already attending", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "uniqueGuests filters out existing attendees and blacklisted emails but does not deduplicate within the input itself - createMany can insert duplicate guest records if the same email appears multiple times", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Incorrect validation condition checks if array length is 0, but initial state contains an empty string [\"\"], so length will be 1, not 0", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Logic error using AND (&&) instead of OR (||) to check if user is team admin OR owner - currently requires user to be BOTH admin AND owner, which is incorrect", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/10600": { @@ -43751,6 +44789,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Race condition in backup code validation: backup codes are decrypted and mutated in memory before being written back - two concurrent login requests can both read the same unused code and both succeed, allowing code reuse", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Backup code validation is case-sensitive due to indexOf() - users entering uppercase hex characters will fail validation even though backup codes should be case-insensitive", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Error message says 'backup code login' but this endpoint disables 2FA - the message incorrectly describes the action, confusing users about what operation is being performed", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Race condition: isDisabling state is not reset if the API call fails or throws an exception", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Race condition: clicking download button and immediately awaiting the download event without ensuring the UI is ready for interaction", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/10967": { @@ -45465,6 +46535,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Logic inversion in organization creation: slug property is conditionally set when IS_TEAM_BILLING_ENABLED is true instead of false - organizations created without billing enabled will be missing the slug field", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Calendar interface now requires createEvent(event, credentialId) but implementations like Lark and Office365 still declare createEvent(event) without credentialId - TypeScript interface mismatch causes runtime errors", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "String.split() always returns an array, so the condition 'Array.isArray(calendar)' will always be true when mainHostDestinationCalendar exists, making the else branch unreachable", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Array destructuring assumes evt.destinationCalendar has at least one element, but fallback array is empty which makes mainHostDestinationCalendar undefined", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Variable 'calendar' is assigned a string literal 'calendar' when mainHostDestinationCalendar is falsy, but then checked with Array.isArray(), creating a type inconsistency", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/8087": { @@ -46639,6 +47741,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Critical: forEach with async callbacks causes fire-and-forget execution - calendar and video event deletions run concurrently without coordination, errors are silently swallowed, and the function may return before operations complete", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Using async callback in forEach - the async functions will not be awaited and will execute concurrently without proper error handling or completion tracking", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "forEach with async callback creates fire-and-forget promises that are not awaited, potentially causing race conditions and unhandled promise rejections", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "reduce function can concatenate undefined values into the array when getAvailability() returns undefined, creating an array with undefined elements", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Template literal displays [object Object] instead of meaningful app identifier when paymentApp is an object", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/keycloak/keycloak/pull/37429": { @@ -47835,6 +48969,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Translation file messages_lt.properties contains Italian text instead of Lithuanian - the locale file has wrong language content, causing Lithuanian users to see Italian interface text", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Simplified Chinese locale file zh_CN uses Traditional Chinese terms in totpStep1 - incorrect character set for the locale, showing Traditional Chinese to Simplified Chinese users", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Method named 'santizeAnchors' should be 'sanitizeAnchors' - missing 'i' in the method name is a typo that may prevent callers from finding the method and cause runtime errors", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Regular expression in replaceAll uses unescaped backslashes that will be interpreted as escape sequences, causing incorrect pattern matching", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/keycloak/keycloak/pull/37634": { @@ -48814,6 +49974,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Wrong parameter in null check: method checks grantType for null instead of rawTokenId, causing the null guard to be ineffective and allowing NullPointerException when rawTokenId is null", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "In isAccessTokenId method, the substring check and equality check for grant shortcut appear inverted - the logic always returns false for valid access tokens", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Catching generic RuntimeException is too broad - the implementation throws IllegalArgumentException specifically; catching only RuntimeException hides the specific exception type and makes error handling less precise", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Variable 'accessToken' is used but 'token' is the parameter name in scope - appears to be using an undefined variable", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Map inversion creates duplicate values when multiple keys map to the same value, with 'UNKNOWN' constant creating circular mapping that overwrites other entries", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/keycloak/keycloak/pull/38446": { @@ -49721,6 +50913,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Optional.get() called directly on Optional returned by RecoveryAuthnCodesUtils.getCredential() without calling isPresent() first - will throw NoSuchElementException when credential is absent", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Unsafe cast from AuthenticatorFactory to ConfigurableAuthenticatorFactory without type checking", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Unsafe cast from FormActionFactory to ConfigurableAuthenticatorFactory without type checking", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Null pointer exception vulnerability - calling .get() on Optional without checking if present, could crash authentication flow when no recovery credential exists", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Type confusion vulnerability in getConfigurableAuthenticatorFactory method - unsafe casting of different factory types without proper validation could lead to ClassCastException or bypass of authentication checks", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/keycloak/keycloak/pull/36882": { @@ -50238,6 +51462,14 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Using picocli.exit() instead of CommandLine.ExitCode or proper exit mechanism - calls static method on picocli class rather than intended exit functionality", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/keycloak/keycloak/pull/36880": { @@ -51098,6 +52330,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Feature flag inconsistency causes orphaned permissions: AdminPermissions event listener cleans up permissions when feature is disabled, but another code path creates permissions without checking the flag, leaving orphaned permissions", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "hasPermission uses findByName with wrong arguments - wrong resource name convention causes resource lookup to fail, making permission checks incorrectly return false for valid permissions", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Feature flag inconsistency causes orphaned permissions: AdminPermissions event listener cleans up permissions when feature is disabled but another code path creates permissions without checking the flag", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "In getClientsWithPermission, iterating CLIENTS_RESOURCE_TYPE resources and returning resource IDs returns the wrong set - should return client IDs associated with those resources, not the resource IDs themselves", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Race condition in lazy initialization where multiple threads can simultaneously check 'clientPermissions != null' and both create new instances, causing the first instance to be lost", + "path": null, + "line": null, + "source": "extracted" + } ] } } \ No newline at end of file diff --git a/offline/results/anthropic_claude-opus-4-5-20251101/evaluations.json b/offline/results/anthropic_claude-opus-4-5-20251101/evaluations.json index 9e2b09c..43a02b2 100644 --- a/offline/results/anthropic_claude-opus-4-5-20251101/evaluations.json +++ b/offline/results/anthropic_claude-opus-4-5-20251101/evaluations.json @@ -2018,6 +2018,55 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak__greptile-v4-1__PR37429__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR37429__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The translation is in Italian instead of Lithuanian. This should be translated to Lithuanian to match the file's locale (messages_lt.properties).", + "severity": "Medium", + "matched_candidate": "Translation file messages_lt.properties contains Italian text instead of Lithuanian - the locale file has wrong language content, causing Lithuanian users to see Italian interface text", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the messages_lt.properties file (Lithuanian locale) contains Italian text instead of Lithuanian text. Both comments point out that the translation language is wrong - Italian instead of Lithuanian." + }, + { + "golden_comment": "The totpStep1 value uses Traditional Chinese terms in the Simplified Chinese file (zh_CN), which is likely incorrect for this locale. Please verify the locale\u2011appropriate translation.", + "severity": "Medium", + "matched_candidate": "Simplified Chinese locale file zh_CN uses Traditional Chinese terms in totpStep1 - incorrect character set for the locale, showing Traditional Chinese to Simplified Chinese users", + "confidence": 0.98, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the totpStep1 value in the zh_CN (Simplified Chinese) locale file incorrectly uses Traditional Chinese terms instead of Simplified Chinese. Both comments point to the same locale/translation mismatch issue." + }, + { + "golden_comment": "The method name 'santizeAnchors' should be 'sanitizeAnchors' (missing 'i').", + "severity": "Low", + "matched_candidate": "Method named 'santizeAnchors' should be 'sanitizeAnchors' - missing 'i' in the method name is a typo that may prevent callers from finding the method and cause runtime errors", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment - the method name 'santizeAnchors' is misspelled and should be 'sanitizeAnchors' (missing the letter 'i'). Both comments point to the same typo in the same method name." + } + ], + "false_positives": [ + { + "candidate": "Regular expression in replaceAll uses unescaped backslashes that will be interpreted as escape sequences, causing incorrect pattern matching" + } + ], + "false_negatives": [ + { + "golden_comment": "The anchor sanitization logic has a potential issue where it consumes English matcher groups without proper validation. If the translated text has more anchor tags than the English text, this could lead to incorrect validation results.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 4, + "total_golden": 4, + "tp": 3, + "fp": 1, + "fn": 1, + "errors_count": 0, + "precision": 0.75, + "recall": 0.75, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/keycloak/keycloak/pull/37429" } }, "https://github.com/keycloak/keycloak/pull/37634": { @@ -3940,6 +3989,58 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak__greptile-v4-1__PR37634__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR37634__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Wrong parameter in null check (grantType vs. rawTokenId)", + "severity": "Critical", + "matched_candidate": "Wrong parameter in null check: method checks grantType for null instead of rawTokenId, causing the null guard to be ineffective and allowing NullPointerException when rawTokenId is null", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment - that the null check is using the wrong parameter (grantType instead of rawTokenId). The candidate provides more detail about the consequences but is pointing to the identical bug." + }, + { + "golden_comment": "In isAccessTokenId, the substring for the grant shortcut and the equality check look inverted: the grant shortcut occupies indices 4\u20135 (substring(4,6)), and a match should return true (combined with UUID check), not false.", + "severity": "High", + "matched_candidate": "In isAccessTokenId method, the substring check and equality check for grant shortcut appear inverted - the logic always returns false for valid access tokens", + "confidence": 0.95, + "reasoning": "The candidate issue identifies the same problem as the golden comment - that in the isAccessTokenId method, the substring check and equality check for grant shortcut are inverted, causing the logic to incorrectly return false for valid access tokens. Both comments point to the same bug where the return value logic is inverted." + }, + { + "golden_comment": " Catching generic RuntimeException is too broad. The implementation throws IllegalArgumentException specifically - catch that instead for more precise testing.", + "severity": "Low", + "matched_candidate": "Catching generic RuntimeException is too broad - the implementation throws IllegalArgumentException specifically; catching only RuntimeException hides the specific exception type and makes error handling less precise", + "confidence": 0.98, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: catching generic RuntimeException is too broad when the implementation specifically throws IllegalArgumentException. Both comments recommend catching the more specific exception type for more precise error handling/testing." + } + ], + "false_positives": [ + { + "candidate": "Variable 'accessToken' is used but 'token' is the parameter name in scope - appears to be using an undefined variable" + }, + { + "candidate": "Map inversion creates duplicate values when multiple keys map to the same value, with 'UNKNOWN' constant creating circular mapping that overwrites other entries" + } + ], + "false_negatives": [ + { + "golden_comment": "Javadoc mentions \"usually like 3-letters shortcut\" but some implementations use 2-letter shortcuts (\"ac\", \"cc\", \"rt\", \"te\", \"pc\", \"ci\", \"ro\"). Consider updating documentation to reflect actual usage pattern.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 4, + "tp": 3, + "fp": 2, + "fn": 1, + "errors_count": 0, + "precision": 0.6, + "recall": 0.75, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/keycloak/keycloak/pull/37634" } }, "https://github.com/keycloak/keycloak/pull/38446": { @@ -5510,6 +5611,47 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak__greptile-v4-1__PR38446__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR38446__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Unsafe raw List deserialization without type safety. Calling Optional.get() directly on the Optional returned by RecoveryAuthnCodesUtils.getCredential(user) without checking isPresent() can lead to a NoSuchElementException if the Optional is empty.", + "severity": "Medium", + "matched_candidate": "Optional.get() called directly on Optional returned by RecoveryAuthnCodesUtils.getCredential() without calling isPresent() first - will throw NoSuchElementException when credential is absent", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment - calling Optional.get() directly on the Optional returned by RecoveryAuthnCodesUtils.getCredential() without checking isPresent() first, which can lead to NoSuchElementException if the Optional is empty. The wording is slightly different but describes the identical underlying issue." + } + ], + "false_positives": [ + { + "candidate": "Unsafe cast from AuthenticatorFactory to ConfigurableAuthenticatorFactory without type checking" + }, + { + "candidate": "Unsafe cast from FormActionFactory to ConfigurableAuthenticatorFactory without type checking" + }, + { + "candidate": "Type confusion vulnerability in getConfigurableAuthenticatorFactory method - unsafe casting of different factory types without proper validation could lead to ClassCastException or bypass of authentication checks" + } + ], + "false_negatives": [ + { + "golden_comment": "After creating the RecoveryAuthnCodesCredentialModel, consider setting its id from the stored credential (e.g., myUser.recoveryCodes.getId()); otherwise getId() will be null and downstream removal by id (e.g., removeStoredCredentialById in the authenticator flow) may not work.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 1, + "fp": 3, + "fn": 1, + "errors_count": 0, + "precision": 0.2, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/keycloak/keycloak/pull/38446" } }, "https://github.com/keycloak/keycloak/pull/36882": { @@ -6708,6 +6850,32 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak__greptile-v4-1__PR36882__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR36882__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Incorrect method call for exit codes. The picocli.exit() method calls System.exit() directly, which is problematic:", + "severity": "Medium", + "matched_candidate": "Using picocli.exit() instead of CommandLine.ExitCode or proper exit mechanism - calls static method on picocli class rather than intended exit functionality", + "confidence": 0.9, + "reasoning": "The candidate issue identifies the same problem as the golden comment - both are about the incorrect use of picocli.exit() method. The golden comment explains that picocli.exit() calls System.exit() directly which is problematic, while the candidate notes it's using 'static method on picocli class rather than intended exit functionality.' Both are pointing to the same underlying issue of using the wrong exit mechanism." + } + ], + "false_positives": [], + "false_negatives": [], + "errors": [], + "total_candidates": 1, + "total_golden": 1, + "tp": 1, + "fp": 0, + "fn": 0, + "errors_count": 0, + "precision": 1.0, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/keycloak/keycloak/pull/36882" } }, "https://github.com/keycloak/keycloak/pull/36880": { @@ -8421,6 +8589,50 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak__greptile-v4-1__PR36880__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR36880__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Inconsistent feature flag bug causing orphaned permissions. The AdminPermissions event listener, responsible for cleaning up permissions upon role, client, or group removal, is incorrectly guarded by the ADMIN_FINE_GRAINED_AUTHZ (V1) feature flag. This is inconsistent with other methods in the class that use ADMIN_FINE_GRAINED_AUTHZ_V2. Consequently, if ADMIN_FINE_GRAINED_AUTHZ_V2 is enabled but V1 is not, the permission cleanup logic will not execute, leading to orphaned permission data. Cleanup should occur regardless of which fine-grained authorization version is enabled.", + "severity": "High", + "matched_candidate": "Feature flag inconsistency causes orphaned permissions: AdminPermissions event listener cleans up permissions when feature is disabled but another code path creates permissions without checking the flag", + "confidence": 0.8, + "reasoning": "The candidate identifies the same core issue: feature flag inconsistency in AdminPermissions event listener causing orphaned permissions. Both comments point to the problem that the cleanup logic has incorrect feature flag checking, leading to orphaned permission data. The candidate's description is slightly different (mentioning 'disabled' vs the specific V1/V2 flag mismatch), but it correctly identifies the fundamental bug: inconsistent feature flag usage in AdminPermissions causing permission cleanup to not execute properly, resulting in orphaned permissions." + }, + { + "golden_comment": "In hasPermission(ClientModel client, String scope), the resource lookup uses findByName(server, client.getId(), server.getId()), but AdminPermissionsSchema.getOrCreateResource creates per-client resources with the owner set to resourceServer.getClientId(), so this lookup will never find those resources and will always fall back to the 'all-clients' resource, effectively ignoring client-specific permissions.", + "severity": "High", + "matched_candidate": "hasPermission uses findByName with wrong arguments - wrong resource name convention causes resource lookup to fail, making permission checks incorrectly return false for valid permissions", + "confidence": 0.85, + "reasoning": "The candidate issue identifies a problem with hasPermission's findByName lookup having wrong arguments that causes resource lookup to fail. The golden comment describes the same core issue - that findByName uses incorrect parameters (client.getId() vs resourceServer.getClientId() as owner), causing the lookup to never find per-client resources and fall back to 'all-clients' resource. Both identify the same bug: incorrect arguments in the resource lookup causing permission checks to fail for client-specific permissions. The candidate's description is less specific about the exact parameter mismatch but identifies the same fundamental problem." + }, + { + "golden_comment": "In getClientsWithPermission(String scope), iterating resourceStore.findByType(server, AdminPermissionsSchema.CLIENTS_RESOURCE_TYPE) and returning resource.getName() will only ever consider the type-level 'Clients' resource (per-client resources have no type) and return its name, while AvailableRoleMappingResource#getRoleIdsWithPermissions expects actual client IDs to pass to realm.getClientById, which can lead to incorrect behavior or a null client and subsequent failures.", + "severity": "High", + "matched_candidate": "In getClientsWithPermission, iterating CLIENTS_RESOURCE_TYPE resources and returning resource IDs returns the wrong set - should return client IDs associated with those resources, not the resource IDs themselves", + "confidence": 0.85, + "reasoning": "The candidate issue identifies the same core problem: the getClientsWithPermission method is returning the wrong values (resource IDs/names instead of actual client IDs). The golden comment explains this in more detail - that iterating CLIENTS_RESOURCE_TYPE resources returns the type-level 'Clients' resource name rather than actual client IDs needed by realm.getClientById. The candidate correctly identifies that the method returns 'resource IDs' instead of 'client IDs associated with those resources', which is the same fundamental bug." + } + ], + "false_positives": [ + { + "candidate": "Race condition in lazy initialization where multiple threads can simultaneously check 'clientPermissions != null' and both create new instances, causing the first instance to be lost" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 3, + "tp": 3, + "fp": 1, + "fn": 0, + "errors_count": 0, + "precision": 0.6, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/keycloak/keycloak/pull/36880" } }, "https://github.com/keycloak/keycloak/pull/37038": { @@ -10017,6 +10229,50 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak__greptile-v4-1__PR37038__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR37038__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Incorrect permission check in canManage() method", + "severity": "High", + "matched_candidate": "Incorrect permission check in canManage() method - the method checks the wrong permission scope, allowing users who should not have manage access to pass the authorization check", + "confidence": 0.98, + "reasoning": "The candidate issue directly matches the golden comment. Both identify the same problem: an incorrect permission check in the canManage() method. The candidate provides additional detail about the nature of the issue (wrong permission scope allowing unauthorized access), but it's clearly identifying the same underlying bug." + } + ], + "false_positives": [ + { + "candidate": "In getGroupIdsWithViewPermission, hasPermission is called with groupResource.getId() for both the resource and scope arguments - the scope parameter should be different from the resource ID, causing incorrect permission evaluation" + }, + { + "candidate": "In getGroupIdsWithViewPermission, hasPermission is called with groupResource.getId() for both resource and scope parameters - scope should differ from resource ID, causing incorrect permission evaluation and potentially granting wrong access" + }, + { + "candidate": "Authorization bypass in getGroupIdsWithViewPermission() - returns empty set when user has view permission, which may grant access to all groups instead of restricting access" + }, + { + "candidate": "Incomplete constant name 'VIEW_MEMBE' in the GROUPS ResourceType definition - should be 'VIEW_MEMBERS'" + } + ], + "false_negatives": [ + { + "golden_comment": "In getGroupIdsWithViewPermission, hasPermission is called with groupResource.getId() and the same groupResource.getId() is added to granted, but hasPermission resolves resources by name (treating the argument as a group id) and the GroupPermissionEvaluator contract says this method returns group IDs that are later used as UserModel.GROUPS and in getUsersCount group filters. This mismatch means per-group VIEW_MEMBERS/MANAGE_MEMBERS permissions may not yield the expected group IDs for filtering and counts, and evaluation may effectively only look at the type-level 'all-groups' resource; consider revisiting whether this should operate on the underlying group ids (resource names) instead so it aligns with the JPA queries and the interface contract.", + "severity": "High" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 1, + "fp": 4, + "fn": 1, + "errors_count": 0, + "precision": 0.2, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/keycloak/keycloak/pull/37038" } }, "https://github.com/keycloak/keycloak/pull/33832": { @@ -11621,6 +11877,49 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak__greptile-v4-1__PR33832__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR33832__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Returns wrong provider (default keystore instead of BouncyCastle)", + "severity": "High", + "matched_candidate": "Method returns wrong provider - returns default keystore provider instead of BouncyCastle provider, causing callers to use incorrect cryptographic implementation", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment - that the method returns the wrong provider (the default keystore provider instead of the BouncyCastle provider). Both comments point to the same bug where the incorrect cryptographic provider is being returned." + }, + { + "golden_comment": "Dead code exists where ASN1Encoder instances are created and written to, but their results are immediately discarded. The actual encoding is performed by new ASN1Encoder instances created in the subsequent return statement, rendering the earlier operations useless.", + "severity": "Low", + "matched_candidate": "Dead code: ASN1Encoder instances are created and written to but their results are immediately discarded - the actual encoding is performed elsewhere, making these encoder operations unnecessary and confusing", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: ASN1Encoder instances are created and written to, but their results are discarded because the actual encoding happens elsewhere (in new instances in the return statement). Both comments describe dead code where encoder operations are performed but never used." + } + ], + "false_positives": [ + { + "candidate": "KeyPairGenerator.getInstance(\"EC\").genKeyPair() creates a key pair with default parameters, but ECDSA algorithms require specific curve parameters (P-256, P-384, P-521) that may not match the default curve" + }, + { + "candidate": "String concatenation creates trailing comma and space that are never removed, resulting in malformed output" + }, + { + "candidate": "Method appears to call itself recursively instead of delegating to a wrapped or parent object, which can cause infinite recursion and StackOverflowError" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 2, + "fp": 3, + "fn": 0, + "errors_count": 0, + "precision": 0.4, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/keycloak/keycloak/pull/33832" } }, "https://github.com/keycloak/keycloak/pull/40940": { @@ -13072,6 +13371,49 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak__greptile-v4-1__PR40940__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR40940__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Returning null from getSubGroupsCount() violates the GroupModel contract (Javadoc says it never returns null) and may lead to NPEs in callers that expect a non-null count.", + "severity": "Critical", + "matched_candidate": "Returning null from getSubGroupsCount() violates the GroupModel contract - Javadoc declares the method never returns null but this implementation returns null, causing NullPointerException in callers that invoke methods on the result", + "confidence": 0.99, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: returning null from getSubGroupsCount() violates the GroupModel contract (Javadoc says it never returns null) and may cause NullPointerException in callers. The wording is nearly identical and addresses the same underlying issue." + }, + { + "golden_comment": "The reader thread isn\u2019t waited for; flipping deletedAll to true and asserting immediately can race and miss exceptions added just after the flag change, making this test flaky.", + "severity": "Medium", + "matched_candidate": "Reader thread is not awaited before assertion - flipping deletedAll to true and immediately asserting races with the reader thread adding exceptions, causing flaky test failures", + "confidence": 0.99, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the reader thread is not being waited for (not awaited), and flipping deletedAll to true while immediately asserting creates a race condition with the reader thread adding exceptions, which causes flaky test failures. The wording is nearly identical and describes the same underlying race condition bug." + } + ], + "false_positives": [ + { + "candidate": "Method appears to call itself recursively instead of delegating to a wrapped or parent object, causing potential infinite recursion and StackOverflowError" + }, + { + "candidate": "Response.Status.Family.SUCCESSFUL includes status codes 200-299, but group creation should specifically check for 201 CREATED status code" + }, + { + "candidate": "Response object is used in try-with-resources but the groupUuid extraction happens inside the try block where response might be closed" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 2, + "fp": 3, + "fn": 0, + "errors_count": 0, + "precision": 0.4, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/keycloak/keycloak/pull/40940" } }, "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1": { @@ -14574,6 +14916,49 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak-greptile__greptile-v4-1__PR1__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak-greptile__greptile-v4-1__PR1__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "ConditionalPasskeysEnabled() called without UserModel parameter", + "severity": "Medium", + "matched_candidate": "ConditionalPasskeysEnabled() is called without the required UserModel parameter - the method signature requires a user argument but is invoked without one, causing a compilation error or NullPointerException", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment - that ConditionalPasskeysEnabled() is being called without the required UserModel parameter. The candidate provides additional context about the consequences (compilation error or NullPointerException) but the core issue identified is identical." + }, + { + "golden_comment": "With isConditionalPasskeysEnabled(UserModel user) requiring user != null, authenticate(...) will not call webauthnAuth.fillContextForm(context) on the initial login page where context.getUser() is still null, so conditional passkey UI will not be set up for first-time passkey login. Consider whether this should also be enabled when no user has been selected yet so ID-less passkey authentication on the initial login form continues to work.", + "severity": "Medium", + "matched_candidate": "With isConditionalPasskeysEnabled(UserModel user) requiring user != null, authenticate() will not call webauthnAuth.fillContextForm(context) on the correct branch when user is null, causing passkey authentication to silently fail", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: the isConditionalPasskeysEnabled(UserModel user) method requires user != null, which means authenticate() won't call webauthnAuth.fillContextForm(context) when the user is null. Both recognize this breaks passkey authentication for the initial login scenario where no user has been selected yet. The golden comment is more detailed about the specific consequence (conditional passkey UI not being set up for first-time passkey login), while the candidate says it will 'silently fail', but they're pointing to the exact same bug." + } + ], + "false_positives": [ + { + "candidate": "WebAuthn authenticators are exposed based solely on user existence without proper authentication state validation, potentially allowing enumeration of valid usernames and their associated WebAuthn credentials" + }, + { + "candidate": "Authentication bypass vulnerability - the authenticate method calls context.success() without verifying user credentials when user is already set and no linked brokers exist" + }, + { + "candidate": "Boolean.parseBoolean() returns false for null input, but the code should likely treat null as false explicitly or handle it differently since AUTH_NOTE values can be null" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 2, + "fp": 3, + "fn": 0, + "errors_count": 0, + "precision": 0.4, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1" } }, "https://github.com/getsentry/sentry/pull/93824": { @@ -16650,234 +17035,288 @@ "tool": "greptile-v4-1", "repo_name": "sentry__sentry__greptile-v4-1__PR93824__20260406", "pr_url": "https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR93824__20260406/pull/1" - } - }, - "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5": { - "gemini": { - "skipped": false, - "true_positives": [], - "false_positives": [ - { - "candidate": "Loop variable `id` shadows the built-in `id()` function - should be renamed to something more specific like `event_id`" - } - ], - "false_negatives": [ - { - "golden_comment": "Breaking changes in error response format", - "severity": "Medium" - }, - { - "golden_comment": "Detector validator uses wrong key when updating type", - "severity": "Medium" - }, - { - "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", - "severity": "Low" - } - ], - "errors": [], - "total_candidates": 1, - "total_golden": 3, - "tp": 0, - "fp": 1, - "fn": 3, - "errors_count": 0, - "precision": 0.0, - "recall": 0.0, - "tool": "gemini", - "repo_name": "sentry__sentry-greptile__gemini__PR5__20260128", - "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__gemini__PR5__20260128/pull/1" - }, - "greptile": { - "skipped": false, - "true_positives": [], - "false_positives": [ - { - "candidate": "Mutating input parameter by sorting in place may cause unexpected side effects" - }, - { - "candidate": "The validation logic for `age` and `timestamp` fields is incorrect - both validators check `self.initial_data` which runs before individual field validation, causing incorrect validation failures. When DRF validates fields, each validator runs independently, so checking for the other field in `initial_data` won't work as expected. The validators should run at the serializer level using `validate()` method instead." - } - ], - "false_negatives": [ - { - "golden_comment": "Breaking changes in error response format", - "severity": "Medium" - }, - { - "golden_comment": "Detector validator uses wrong key when updating type", - "severity": "Medium" - }, - { - "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", - "severity": "Low" - } - ], - "errors": [], - "total_candidates": 2, - "total_golden": 3, - "tp": 0, - "fp": 2, - "fn": 3, - "errors_count": 0, - "precision": 0.0, - "recall": 0.0, - "tool": "greptile", - "repo_name": "sentry__sentry-greptile__greptile__PR5__20260122", - "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__greptile__PR5__20260122/pull/1" }, - "claude": { + "codesheriff": { "skipped": false, - "true_positives": [], - "false_positives": [], - "false_negatives": [ - { - "golden_comment": "Breaking changes in error response format", - "severity": "Medium" - }, + "true_positives": [ { - "golden_comment": "Detector validator uses wrong key when updating type", - "severity": "Medium" + "golden_comment": "Inconsistent metric tagging with 'shard' and 'shards'", + "severity": "Medium", + "matched_candidate": "Inconsistent metric tagging: some calls use tag 'shard' while others use 'shards' - metrics will be split across two different tag values, making aggregation and dashboards incorrect", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment - inconsistent metric tagging where some calls use 'shard' and others use 'shards'. The candidate provides additional context about why this is problematic (metrics split across different tag values affecting aggregation and dashboards), but it's fundamentally the same issue." }, { - "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", - "severity": "Low" - } - ], - "errors": [], - "total_candidates": 0, - "total_golden": 3, - "tp": 0, - "fp": 0, - "fn": 3, - "errors_count": 0, - "precision": 0.0, - "recall": 0.0, - "tool": "claude", - "repo_name": "sentry__sentry-greptile__claude__PR5__20260127", - "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__claude__PR5__20260127/pull/1" - }, - "qodo": { - "skipped": false, - "true_positives": [ - { - "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", + "golden_comment": "Fixed sleep in tests can be flaky; wait on condition instead", "severity": "Low", - "matched_candidate": "fetch_error_details zips error_ids with events.values() from nodestore.backend.get_multi but dict iteration order may not match input order and may drop missing keys, causing wrong event payload association with error ids", - "confidence": 0.98, - "reasoning": "The candidate issue identifies the exact same problem as the golden comment: using zip(error_ids, events.values()) with get_multi results is problematic because dict iteration order may not match the input order of error_ids, and missing keys can cause misalignment. Both comments point to the same bug where event data can be incorrectly paired with the wrong error ID." - } - ], - "false_positives": [ - { - "candidate": "browser_reporting_collector logs request.data via logger.info which can include full URLs, user agents, and potentially other sensitive report contents - consider redaction/sampling or lowering verbosity for production logs" + "matched_candidate": "time.sleep in test_consumer.py won't actually wait because time.sleep was monkeypatched above - test timing assumptions are broken, making the test unreliable", + "confidence": 0.85, + "reasoning": "Both comments identify the same underlying issue: the use of time.sleep in tests is problematic and makes tests unreliable/flaky. The golden comment says fixed sleep can be flaky and suggests waiting on a condition instead. The candidate comment points out that time.sleep is monkeypatched making timing assumptions broken and the test unreliable. Both are identifying that the sleep-based approach in tests is problematic for reliability, though they focus on slightly different aspects of why (golden: general flakiness of fixed sleeps; candidate: specific monkeypatching issue). They're addressing the same core problem of unreliable test timing." }, { - "candidate": "Mutual-exclusion validation for age vs timestamp uses self.initial_data.get(...) truthiness checks which can behave incorrectly for valid falsy values like age=0 - should check key presence instead" + "golden_comment": "Because flusher processes are created via multiprocessing.get_context('spawn').Process, they are instances of multiprocessing.context.SpawnProcess, which on POSIX is not a subclass of multiprocessing.Process, so this isinstance check will always be false and hung processes won't be killed here.", + "severity": "High", + "matched_candidate": "flusher processes created via multiprocessing.get_context(spawn).Process are instances of SpawnProcess which on POSIX is not a subclass of multiprocessing.Process - isinstance checks will fail and process management may break", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: flusher processes created via multiprocessing.get_context('spawn').Process are instances of SpawnProcess, which on POSIX is not a subclass of multiprocessing.Process, causing isinstance checks to fail. Both comments point to the same bug where process management (specifically killing hung processes) will break due to this type checking issue." }, { - "candidate": "get_merged_pr_single_issue_template calls PRCommentWorkflow._truncate_title but the truncation helper is defined on CommitContextIntegration - incorrect class reference that may break at runtime" - } - ], - "false_negatives": [ - { - "golden_comment": "Breaking changes in error response format", - "severity": "Medium" + "golden_comment": "Sleep in test_consumer.py won\u2019t actually wait because time.sleep was monkeypatched above; consider restoring sleep or using a different sync to ensure the flusher has time to process.", + "severity": "Medium", + "matched_candidate": "time.sleep in test_consumer.py won't actually wait because time.sleep was monkeypatched above - test timing assumptions are broken, making the test unreliable", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: time.sleep in test_consumer.py won't actually wait because time.sleep was monkeypatched above. Both comments point out that this breaks the test's timing assumptions. The candidate says it makes the test 'unreliable' while the golden suggests 'restoring sleep or using a different sync to ensure the flusher has time to process' - but they're identifying the same underlying issue about the monkeypatched sleep not actually waiting." }, { - "golden_comment": "Detector validator uses wrong key when updating type", - "severity": "Medium" + "golden_comment": "Breaking out of the loop when the deadline has elapsed can skip terminating remaining flusher processes, potentially leaving them running after shutdown; consider ensuring termination is attempted even if the deadline is exceeded.", + "severity": "Medium", + "matched_candidate": "Breaking out of the loop when deadline is elapsed skips terminating remaining flusher processes, potentially leaking process handles and causing resource exhaustion", + "confidence": 0.95, + "reasoning": "Both comments identify the exact same issue: breaking out of the loop when the deadline is elapsed will skip terminating the remaining flusher processes. The golden comment mentions 'leaving them running after shutdown' while the candidate mentions 'leaking process handles and causing resource exhaustion' - these are slightly different framings of the consequence, but both correctly identify the same underlying bug: that remaining flusher processes won't be terminated if the deadline is exceeded." } ], - "errors": [], - "total_candidates": 4, - "total_golden": 3, - "tp": 1, - "fp": 3, - "fn": 2, - "errors_count": 0, - "precision": 0.25, - "recall": 0.3333333333333333, - "tool": "qodo", - "repo_name": "sentry__sentry-greptile__qodo__PR5__20260123", - "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__qodo__PR5__20260123/pull/1" - }, - "graphite": { - "skipped": false, - "true_positives": [], "false_positives": [], - "false_negatives": [ - { - "golden_comment": "Breaking changes in error response format", - "severity": "Medium" - }, - { - "golden_comment": "Detector validator uses wrong key when updating type", - "severity": "Medium" - }, - { - "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", - "severity": "Low" - } - ], + "false_negatives": [], "errors": [], - "total_candidates": 0, - "total_golden": 3, - "tp": 0, + "total_candidates": 4, + "total_golden": 5, + "tp": 5, "fp": 0, - "fn": 3, - "errors_count": 0, - "precision": 0.0, - "recall": 0.0, - "tool": "graphite", - "repo_name": "sentry__sentry-greptile__graphite__PR5__20260123", - "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__graphite__PR5__20260123/pull/1" - }, - "bugbot": { - "skipped": false, - "true_positives": [ - { - "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", - "severity": "Low", - "matched_candidate": "The fetch_error_details function uses zip(error_ids, events.values()) to pair error IDs with their event data incorrectly - nodestore.backend.get_multi(node_ids) returns a dict keyed by node_id (not error_id), and dict values are not guaranteed to be in the same order as the input list. Additionally, if some events are missing from the nodestore, the returned dict will have fewer entries, causing the zip to misalign error IDs with the wrong event data, resulting in error titles, messages, and timestamps being attributed to the wrong error IDs.", - "confidence": 0.98, - "reasoning": "The candidate issue identifies the exact same problem as the golden comment: using zip(error_ids, events.values()) is problematic because (1) dict value order is not guaranteed to match the input order of error_ids, and (2) missing nodes/events will cause misalignment. Both comments point to the same bug where event data can be paired with the wrong error ID due to these ordering and missing entry issues." - } - ], - "false_positives": [ - { - "candidate": "TableWidgetVisualization receives hardcoded empty data (columns={[]}, tableData={{data: [], meta: {fields: {}, units: {}}}}) when the use-table-widget-visualization feature flag is enabled, while StyledSimpleTableChart in the else branch correctly receives result.data and result.meta - enabling the feature flag would render empty tables instead of actual widget data." - } - ], - "false_negatives": [ - { - "golden_comment": "Breaking changes in error response format", - "severity": "Medium" - }, - { - "golden_comment": "Detector validator uses wrong key when updating type", - "severity": "Medium" - } - ], - "errors": [], - "total_candidates": 2, - "total_golden": 3, - "tp": 1, - "fp": 1, - "fn": 2, + "fn": 0, "errors_count": 0, - "precision": 0.5, - "recall": 0.3333333333333333, - "tool": "bugbot", - "repo_name": "sentry__sentry-greptile__bugbot__PR5__20260122", - "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__bugbot__PR5__20260122/pull/1" - }, - "copilot": { + "precision": 1.25, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/getsentry/sentry/pull/93824" + } + }, + "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5": { + "gemini": { "skipped": false, "true_positives": [], "false_positives": [ { - "candidate": "TableWidgetVisualization is rendering with empty data and columns when the feature flag is enabled instead of passing actual result.data, result.meta, and derived columns from fields and fieldAliases" + "candidate": "Loop variable `id` shadows the built-in `id()` function - should be renamed to something more specific like `event_id`" + } + ], + "false_negatives": [ + { + "golden_comment": "Breaking changes in error response format", + "severity": "Medium" + }, + { + "golden_comment": "Detector validator uses wrong key when updating type", + "severity": "Medium" + }, + { + "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 1, + "total_golden": 3, + "tp": 0, + "fp": 1, + "fn": 3, + "errors_count": 0, + "precision": 0.0, + "recall": 0.0, + "tool": "gemini", + "repo_name": "sentry__sentry-greptile__gemini__PR5__20260128", + "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__gemini__PR5__20260128/pull/1" + }, + "greptile": { + "skipped": false, + "true_positives": [], + "false_positives": [ + { + "candidate": "Mutating input parameter by sorting in place may cause unexpected side effects" + }, + { + "candidate": "The validation logic for `age` and `timestamp` fields is incorrect - both validators check `self.initial_data` which runs before individual field validation, causing incorrect validation failures. When DRF validates fields, each validator runs independently, so checking for the other field in `initial_data` won't work as expected. The validators should run at the serializer level using `validate()` method instead." + } + ], + "false_negatives": [ + { + "golden_comment": "Breaking changes in error response format", + "severity": "Medium" + }, + { + "golden_comment": "Detector validator uses wrong key when updating type", + "severity": "Medium" + }, + { + "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 2, + "total_golden": 3, + "tp": 0, + "fp": 2, + "fn": 3, + "errors_count": 0, + "precision": 0.0, + "recall": 0.0, + "tool": "greptile", + "repo_name": "sentry__sentry-greptile__greptile__PR5__20260122", + "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__greptile__PR5__20260122/pull/1" + }, + "claude": { + "skipped": false, + "true_positives": [], + "false_positives": [], + "false_negatives": [ + { + "golden_comment": "Breaking changes in error response format", + "severity": "Medium" + }, + { + "golden_comment": "Detector validator uses wrong key when updating type", + "severity": "Medium" + }, + { + "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 0, + "total_golden": 3, + "tp": 0, + "fp": 0, + "fn": 3, + "errors_count": 0, + "precision": 0.0, + "recall": 0.0, + "tool": "claude", + "repo_name": "sentry__sentry-greptile__claude__PR5__20260127", + "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__claude__PR5__20260127/pull/1" + }, + "qodo": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", + "severity": "Low", + "matched_candidate": "fetch_error_details zips error_ids with events.values() from nodestore.backend.get_multi but dict iteration order may not match input order and may drop missing keys, causing wrong event payload association with error ids", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: using zip(error_ids, events.values()) with get_multi results is problematic because dict iteration order may not match the input order of error_ids, and missing keys can cause misalignment. Both comments point to the same bug where event data can be incorrectly paired with the wrong error ID." + } + ], + "false_positives": [ + { + "candidate": "browser_reporting_collector logs request.data via logger.info which can include full URLs, user agents, and potentially other sensitive report contents - consider redaction/sampling or lowering verbosity for production logs" + }, + { + "candidate": "Mutual-exclusion validation for age vs timestamp uses self.initial_data.get(...) truthiness checks which can behave incorrectly for valid falsy values like age=0 - should check key presence instead" + }, + { + "candidate": "get_merged_pr_single_issue_template calls PRCommentWorkflow._truncate_title but the truncation helper is defined on CommitContextIntegration - incorrect class reference that may break at runtime" + } + ], + "false_negatives": [ + { + "golden_comment": "Breaking changes in error response format", + "severity": "Medium" + }, + { + "golden_comment": "Detector validator uses wrong key when updating type", + "severity": "Medium" + } + ], + "errors": [], + "total_candidates": 4, + "total_golden": 3, + "tp": 1, + "fp": 3, + "fn": 2, + "errors_count": 0, + "precision": 0.25, + "recall": 0.3333333333333333, + "tool": "qodo", + "repo_name": "sentry__sentry-greptile__qodo__PR5__20260123", + "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__qodo__PR5__20260123/pull/1" + }, + "graphite": { + "skipped": false, + "true_positives": [], + "false_positives": [], + "false_negatives": [ + { + "golden_comment": "Breaking changes in error response format", + "severity": "Medium" + }, + { + "golden_comment": "Detector validator uses wrong key when updating type", + "severity": "Medium" + }, + { + "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 0, + "total_golden": 3, + "tp": 0, + "fp": 0, + "fn": 3, + "errors_count": 0, + "precision": 0.0, + "recall": 0.0, + "tool": "graphite", + "repo_name": "sentry__sentry-greptile__graphite__PR5__20260123", + "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__graphite__PR5__20260123/pull/1" + }, + "bugbot": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", + "severity": "Low", + "matched_candidate": "The fetch_error_details function uses zip(error_ids, events.values()) to pair error IDs with their event data incorrectly - nodestore.backend.get_multi(node_ids) returns a dict keyed by node_id (not error_id), and dict values are not guaranteed to be in the same order as the input list. Additionally, if some events are missing from the nodestore, the returned dict will have fewer entries, causing the zip to misalign error IDs with the wrong event data, resulting in error titles, messages, and timestamps being attributed to the wrong error IDs.", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: using zip(error_ids, events.values()) is problematic because (1) dict value order is not guaranteed to match the input order of error_ids, and (2) missing nodes/events will cause misalignment. Both comments point to the same bug where event data can be paired with the wrong error ID due to these ordering and missing entry issues." + } + ], + "false_positives": [ + { + "candidate": "TableWidgetVisualization receives hardcoded empty data (columns={[]}, tableData={{data: [], meta: {fields: {}, units: {}}}}) when the use-table-widget-visualization feature flag is enabled, while StyledSimpleTableChart in the else branch correctly receives result.data and result.meta - enabling the feature flag would render empty tables instead of actual widget data." + } + ], + "false_negatives": [ + { + "golden_comment": "Breaking changes in error response format", + "severity": "Medium" + }, + { + "golden_comment": "Detector validator uses wrong key when updating type", + "severity": "Medium" + } + ], + "errors": [], + "total_candidates": 2, + "total_golden": 3, + "tp": 1, + "fp": 1, + "fn": 2, + "errors_count": 0, + "precision": 0.5, + "recall": 0.3333333333333333, + "tool": "bugbot", + "repo_name": "sentry__sentry-greptile__bugbot__PR5__20260122", + "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__bugbot__PR5__20260122/pull/1" + }, + "copilot": { + "skipped": false, + "true_positives": [], + "false_positives": [ + { + "candidate": "TableWidgetVisualization is rendering with empty data and columns when the feature flag is enabled instead of passing actual result.data, result.meta, and derived columns from fields and fieldAliases" } ], "false_negatives": [ @@ -20655,6 +21094,53 @@ "tool": "greptile-v4-1", "repo_name": "sentry__sentry-greptile__greptile-v4-1__PR1__20260406", "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__greptile-v4-1__PR1__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Importing non-existent OptimizedCursorPaginator", + "severity": "Low", + "matched_candidate": "Importing OptimizedCursorPaginator which does not exist in the sentry codebase - this import will fail at runtime with ImportError", + "confidence": 1.0, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment - importing OptimizedCursorPaginator which does not exist. The candidate provides slightly more detail (mentioning it will fail at runtime with ImportError), but it's pointing to the identical underlying issue." + }, + { + "golden_comment": "Django querysets do not support negative slicing", + "severity": "High", + "matched_candidate": "Negative offset passed to queryset slicing will cause runtime error - Python slice indices must be non-negative", + "confidence": 0.95, + "reasoning": "Both comments identify the same issue: negative slicing/offset values don't work with Django querysets. The golden comment states 'Django querysets do not support negative slicing' and the candidate states 'Negative offset passed to queryset slicing will cause runtime error - Python slice indices must be non-negative'. Both are pointing to the same underlying problem - that negative values in queryset slicing operations will fail." + }, + { + "golden_comment": "When requests are authenticated with API keys or org auth tokens (which have user_id=None), organization_context.member is None. Line 71 attempts to access organization_context.member.has_global_access without checking if member is None, causing an AttributeError crash when optimized_pagination=true is used, even though the request passed all permission checks with valid org:write scope.", + "severity": "High", + "matched_candidate": "When requests are authenticated with API keys or org auth tokens, organization_context.member is None and accessing .has_global_access() on None will raise AttributeError", + "confidence": 0.99, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: when requests are authenticated with API keys or org auth tokens, organization_context.member is None, and accessing .has_global_access() on None will raise an AttributeError. Both comments point to the same bug - the missing null check for member before accessing has_global_access." + }, + { + "golden_comment": "get_item_key assumes a numeric key, but the paginator is used with order_by=-datetime in the audit logs endpoint; calling math.floor/ceil on a datetime will raise a TypeError.", + "severity": "High", + "matched_candidate": "Math.floor/ceil applied to potentially non-numeric values without type checking will raise TypeError", + "confidence": 0.9, + "reasoning": "The candidate issue identifies the same core problem as the golden comment - that math.floor/ceil is being applied to values that may not be numeric, which will raise a TypeError. The golden comment is more specific about the context (datetime values from audit logs with order_by=-datetime), but both identify the same underlying bug: type incompatibility when applying math.floor/ceil to non-numeric values." + } + ], + "false_positives": [], + "false_negatives": [], + "errors": [], + "total_candidates": 4, + "total_golden": 4, + "tp": 4, + "fp": 0, + "fn": 0, + "errors_count": 0, + "precision": 1.0, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1" } }, "https://github.com/grafana/grafana/pull/97529": { @@ -22136,6 +22622,46 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR97529__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR97529__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "A race condition in BuildIndex allows multiple goroutines to concurrently build the same expensive index for the same key. This is caused by moving the b.cacheMu lock from protecting the entire function to only protecting the final cache assignment. ", + "severity": "High", + "matched_candidate": "Race condition in BuildIndex: multiple goroutines can concurrently build the same index for the same key without synchronization, wasting resources and potentially causing data corruption", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: a race condition in BuildIndex where multiple goroutines can concurrently build the same index for the same key. Both comments point to the lack of proper synchronization/locking that allows duplicate expensive index building operations. The candidate even mentions the same consequences (wasting resources). This is a clear match of the same underlying issue." + }, + { + "golden_comment": "Calling s.search.TotalDocs() here may race with concurrent index creation: TotalDocs iterates b.cache without synchronization, and the event watcher goroutine started just above could trigger BuildIndex writes concurrently, potentially causing a concurrent map read/write panic.", + "severity": "High", + "matched_candidate": "Race condition: s.search.TotalDocs() iterates the cache without synchronization while the event watcher goroutine concurrently modifies it - concurrent map read/write causes panic in Go", + "confidence": 0.99, + "reasoning": "The candidate issue identifies the exact same race condition as the golden comment: s.search.TotalDocs() iterates the cache without synchronization while the event watcher goroutine concurrently modifies it, which can cause a concurrent map read/write panic. Both comments point to the same underlying bug with the same root cause and consequence." + } + ], + "false_positives": [ + { + "candidate": "Hard-coded port assignment may cause port conflicts in concurrent test execution" + }, + { + "candidate": "Delete function checks for ResourceVersion < 0 but the error message says 'update must include the previous version' which is incorrect for a delete operation" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 4, + "total_golden": 2, + "tp": 2, + "fp": 2, + "fn": 0, + "errors_count": 0, + "precision": 0.5, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/97529" } }, "https://github.com/getsentry/sentry/pull/80168": { @@ -23601,6 +24127,39 @@ "tool": "greptile-v4-1", "repo_name": "sentry__sentry__greptile-v4-1__PR80168__20260406", "pr_url": "https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR80168__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "MetricAlertDetectorHandler inherits from StatefulDetectorHandler but only contains pass, failing to implement its required abstract methods: counter_names (property), get_dedupe_value(), get_group_key_values(), and build_occurrence_and_event_data(). This will cause a TypeError at runtime when the class is instantiated.", + "severity": "High", + "matched_candidate": "MetricAlertDetectorHandler inherits from StatefulDetectorHandler but the class body only contains pass, failing to implement required abstract methods - will raise NotImplementedError at runtime", + "confidence": 0.95, + "reasoning": "Both comments identify the same issue: MetricAlertDetectorHandler inherits from StatefulDetectorHandler but only contains 'pass', failing to implement required abstract methods. The golden comment specifies it will cause TypeError, while the candidate says NotImplementedError - this is a minor technical difference (abstract methods typically raise TypeError when instantiating, not NotImplementedError), but both correctly identify the core problem of missing abstract method implementations that will cause runtime errors." + }, + { + "golden_comment": "Docstring says this returns a list of DetectorEvaluationResult, but the method now returns a dict keyed by DetectorGroupKey. Consider updating the docstring to match the new return type.", + "severity": "Low", + "matched_candidate": "Method docstring says it returns a list of DetectorEvaluationResult but the implementation now returns a dict keyed by DetectorGroupKey - documentation and return type are inconsistent", + "confidence": 0.99, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the docstring claims the method returns a list of DetectorEvaluationResult, but the actual implementation returns a dict keyed by DetectorGroupKey. Both comments point out this documentation/return type inconsistency." + } + ], + "false_positives": [], + "false_negatives": [], + "errors": [], + "total_candidates": 2, + "total_golden": 2, + "tp": 2, + "fp": 0, + "fn": 0, + "errors_count": 0, + "precision": 1.0, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/getsentry/sentry/pull/80168" } }, "https://github.com/getsentry/sentry/pull/80528": { @@ -24975,6 +25534,41 @@ "tool": "greptile-v4-1", "repo_name": "sentry__sentry__greptile-v4-1__PR80528__20260406", "pr_url": "https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR80528__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The function modifies the config variable to include display values but then returns the original monitor.config instead of the modified version.", + "severity": "High", + "matched_candidate": "Function modifies the config variable to include display values but then returns the original monitor.config instead of the modified config, discarding the changes", + "confidence": 1.0, + "reasoning": "The candidate issue is essentially identical to the golden comment. Both identify the same bug: the function modifies a 'config' variable to include display values, but then returns 'monitor.config' (the original) instead of the modified 'config' variable, which means the changes are discarded." + } + ], + "false_positives": [ + { + "candidate": "Slice operation `previous_checkins[:failure_issue_threshold]` occurs before reverse operation, taking most recent N check-ins instead of oldest N check-ins, resulting in examining wrong subset for threshold evaluation" + } + ], + "false_negatives": [ + { + "golden_comment": "The code fetches MonitorCheckIn objects by ID when the required data already exists in previous_checkins. This creates an unnecessary database query.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 2, + "total_golden": 2, + "tp": 1, + "fp": 1, + "fn": 1, + "errors_count": 0, + "precision": 0.5, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/getsentry/sentry/pull/80528" } }, "https://github.com/getsentry/sentry/pull/77754": { @@ -26724,6 +27318,53 @@ "tool": "greptile-v4-1", "repo_name": "sentry__sentry__greptile-v4-1__PR77754__20260406", "pr_url": "https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR77754__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Shared mutable default in dataclass timestamp", + "severity": "Medium", + "matched_candidate": "Mutable default argument in dataclass field - using a mutable object as default value for timestamp field shares state across instances, causing subtle data corruption bugs", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment - both are about a mutable default argument in a dataclass field for the timestamp field, where the mutable object is shared across instances. The candidate provides more detail about the consequences (data corruption bugs) but it's clearly the same underlying issue." + }, + { + "golden_comment": "The method name has a typo: test_from_dict_inalid_data should be test_from_dict_invalid_data.", + "severity": "Low", + "matched_candidate": "Method name typo: test_from_dict_inalid_data is missing a letter and should be test_from_dict_invalid_data - the test may not be discovered by test runners that filter by name pattern", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment - the typo in the method name 'test_from_dict_inalid_data' which should be 'test_from_dict_invalid_data'. Both point out the missing 'v' in 'invalid'. The candidate even provides additional context about why this matters (test discovery), but it's fundamentally the same issue." + }, + { + "golden_comment": "Method name says 'empty_array' but tests empty dict - consider renaming to 'test_from_dict_empty_dict' for clarity.", + "severity": "Low", + "matched_candidate": "Method named 'empty_array' but tests an empty dict - test name does not match what is being tested, making the test suite misleading and harder to maintain", + "confidence": 0.95, + "reasoning": "Both comments identify the exact same issue: the test method name contains 'empty_array' but the test actually tests an empty dict. Both suggest this is a naming inconsistency that should be corrected for clarity/maintainability. The golden comment specifically suggests renaming to 'test_from_dict_empty_dict', while the candidate describes the problem without a specific suggestion, but they're pointing to the same underlying issue." + }, + { + "golden_comment": "to_dict() returns a datetime for queued; if this dict is passed in task kwargs (e.g., via apply_async), JSON serialization may fail depending on the serializer, which can cause enqueue errors.", + "severity": "Medium", + "matched_candidate": "to_dict() returns a datetime object for the queued field - if this dict is passed as task kwargs to apply_async(), JSON serialization will fail since datetime is not JSON-serializable by default", + "confidence": 0.98, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: to_dict() returns a datetime object for the 'queued' field, and when this dict is passed as task kwargs to apply_async(), JSON serialization will fail because datetime is not JSON-serializable. Both comments point to the same bug regarding datetime serialization issues when using the dict with Celery tasks." + } + ], + "false_positives": [], + "false_negatives": [], + "errors": [], + "total_candidates": 4, + "total_golden": 4, + "tp": 4, + "fp": 0, + "fn": 0, + "errors_count": 0, + "precision": 1.0, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/getsentry/sentry/pull/77754" } }, "https://github.com/getsentry/sentry/pull/95633": { @@ -28392,6 +29033,41 @@ "tool": "greptile-v4-1", "repo_name": "sentry__sentry__greptile-v4-1__PR95633__20260406", "pr_url": "https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR95633__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The queue.shutdown() method with 'immediate=False' parameter may not exist in the standard Python queue module. This could cause AttributeError at runtime. Verify the correct API or implement a custom shutdown mechanism.", + "severity": "High", + "matched_candidate": "queue.shutdown() with 'immediate=False' parameter may not exist in the standard Python queue.Queue module - calling a non-existent method will raise AttributeError at runtime", + "confidence": 0.99, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: queue.shutdown() with 'immediate=False' parameter may not exist in the standard Python queue module, which would cause an AttributeError at runtime. Both comments point to the same potential API compatibility issue with the same method call." + } + ], + "false_positives": [], + "false_negatives": [ + { + "golden_comment": "The magic number 50 for max_wait is used repeatedly throughout the tests. Consider extracting this as a named constant to improve maintainability.", + "severity": "Low" + }, + { + "golden_comment": "The test test_thread_queue_parallel_error_handling has a docstring that doesn't match the test implementation.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 1, + "total_golden": 3, + "tp": 1, + "fp": 0, + "fn": 2, + "errors_count": 0, + "precision": 1.0, + "recall": 0.3333333333333333, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/getsentry/sentry/pull/95633" } }, "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2": { @@ -30387,6 +31063,53 @@ "tool": "greptile-v4-1", "repo_name": "sentry__sentry-greptile__greptile-v4-1__PR2__20260406", "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__greptile-v4-1__PR2__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "OptimizedCursorPaginator negative-offset branch slices QuerySet with a negative start index", + "severity": "Critical", + "matched_candidate": "OptimizedCursorPaginator negative-offset branch slices QuerySet with a negative start index - Django QuerySets do not support negative indexing and will raise AssertionError", + "confidence": 1.0, + "reasoning": "The candidate issue is an exact match with the golden comment. Both identify the same problem: the OptimizedCursorPaginator's negative-offset branch slices a QuerySet with a negative start index. The candidate even provides additional context that Django QuerySets don't support negative indexing and will raise an AssertionError, which is the same underlying issue." + }, + { + "golden_comment": "BasePaginator negative-offset branch slices QuerySet with a negative start index", + "severity": "High", + "matched_candidate": "BasePaginator negative-offset branch slices QuerySet with a negative start index - Django raises AssertionError when negative slice indices are used on QuerySets", + "confidence": 1.0, + "reasoning": "The candidate issue is essentially identical to the golden comment. Both identify the same problem: the BasePaginator's negative-offset branch uses a negative start index when slicing a QuerySet, which is problematic because Django QuerySets don't support negative slice indices (raising an AssertionError). The candidate even provides additional context about why this is an issue (Django raises AssertionError), but it's pointing to the exact same underlying bug." + }, + { + "golden_comment": "OptimizedCursorPaginator.get_item_key uses floor/ceil on a datetime key (order_by='-datetime'), causing TypeError.", + "severity": "High", + "matched_candidate": "OptimizedCursorPaginator.get_item_key applies floor and ceil to a datetime key when order_by is -datetime - math.floor and math.ceil raise TypeError when given a datetime object instead of a number", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: OptimizedCursorPaginator.get_item_key uses floor/ceil on a datetime key (when order_by='-datetime'), which causes a TypeError because math.floor and math.ceil don't work with datetime objects." + } + ], + "false_positives": [ + { + "candidate": "Span with span_id='b'*16 has parent_span_id=None but is_segment_span=True, while other spans reference 'b'*16 as their parent, creating inconsistent parent-child relationship" + }, + { + "candidate": "Dictionary comprehension uses span.payload as key, but span.payload is bytes which is not hashable and cannot be used as a Redis hash key" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 3, + "tp": 3, + "fp": 2, + "fn": 0, + "errors_count": 0, + "precision": 0.6, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2" } }, "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3": { @@ -32240,6 +32963,46 @@ "tool": "greptile-v4-1", "repo_name": "sentry__sentry-greptile__greptile-v4-1__PR3__20260406", "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__greptile-v4-1__PR3__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "sample_rate = 0.0 is falsy and skipped", + "severity": "Low", + "matched_candidate": "sample_rate = 0.0 is falsy in Python - the condition if client_sample_rate evaluates to False for 0.0, silently skipping the zero sample rate case", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment - that sample_rate = 0.0 is falsy in Python and would be incorrectly skipped by a conditional check. Both comments point to the same bug where a zero sample rate value would be treated as falsy and not handled properly." + }, + { + "golden_comment": "Using Python\u2019s built-in hash() to build cache keys is non-deterministic across processes (hash randomization), so keys won\u2019t match across workers and invalidate_upsampling_cache may fail to delete them. Use a deterministic serialization of project_ids for the cache key.", + "severity": "Low", + "matched_candidate": "Python's built-in hash() is non-deterministic across processes due to PYTHONHASHSEED - cache keys built with hash() will not match across workers, causing cache misses in distributed environments", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: Python's built-in hash() is non-deterministic across processes due to hash randomization (PYTHONHASHSEED), which causes cache keys to not match across workers. Both comments point to the same underlying bug - using hash() for cache keys in a distributed/multi-worker environment will cause cache misses and failures. The candidate mentions 'cache misses' while the golden mentions 'invalidate_upsampling_cache may fail to delete them', but these are both consequences of the same root issue." + }, + { + "golden_comment": "The upsampling eligibility check passes the outer dataset instead of the actual dataset used by scoped_dataset. In paths where the query ultimately runs against discover (e.g., dashboard split) while the original dataset is metrics, upsampling may be skipped even when all projects are allowlisted.", + "severity": "Medium", + "matched_candidate": "Upsampling eligibility check passes the outer dataset instead of the scoped_dataset actually used by the query - when both datasets differ, the eligibility check is performed on wrong data and may incorrectly enable or disable upsampling", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: the upsampling eligibility check uses the outer/original dataset instead of the scoped_dataset that is actually used by the query. Both comments point out that when these datasets differ, the eligibility check is performed on the wrong dataset, which can lead to incorrect upsampling behavior. The candidate even uses similar terminology ('outer dataset' vs 'scoped_dataset')." + } + ], + "false_positives": [], + "false_negatives": [], + "errors": [], + "total_candidates": 4, + "total_golden": 3, + "tp": 3, + "fp": 0, + "fn": 0, + "errors_count": 0, + "precision": 0.75, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3" } }, "https://github.com/grafana/grafana/pull/103633": { @@ -33767,6 +34530,47 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR103633__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR103633__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The Check operation exhibits asymmetric cache trust logic: cached permission grants are trusted and returned immediately, but cached denials from the same permission cache are ignored, leading to a fresh database lookup. This allows stale cached grants to provide access to revoked resources, posing a security risk. ", + "severity": "High", + "matched_candidate": "Asymmetric cache trust logic in Check operation: cached permission grants are returned immediately without re-verification, but cache misses always go to the source. An expired or revoked permission may continue to grant access until cache expiry", + "confidence": 0.9, + "reasoning": "Both comments identify the same core security issue: asymmetric cache trust logic in the Check operation where cached permission grants are trusted and returned immediately, but there's inconsistent handling that could allow revoked permissions to continue granting access until cache expiry. The golden comment specifically notes that cached denials are ignored leading to fresh DB lookups, while the candidate describes it as 'cache misses always go to the source' - these are describing the same asymmetric behavior. Both correctly identify the security risk of stale cached grants providing access to revoked resources." + } + ], + "false_positives": [ + { + "candidate": "NoopCache implementation always returns cache miss, causing authorization checks to bypass caching entirely and potentially leading to performance degradation that could facilitate DoS attacks on authorization services" + }, + { + "candidate": "NoopCache.Get method takes a receiver pointer when it should be a value receiver since the struct has no fields" + }, + { + "candidate": "Cache key mismatch: setting cache with action 'dashboards:read' but checking with verb 'get' which likely maps to a different action" + } + ], + "false_negatives": [ + { + "golden_comment": "The test comment says the cached permissions 'allow access', but the map stores false for dashboards:uid:dash1, so checkPermission will still treat this scope as not allowed.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 4, + "total_golden": 2, + "tp": 1, + "fp": 3, + "fn": 1, + "errors_count": 0, + "precision": 0.25, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/103633" } }, "https://github.com/getsentry/sentry/pull/67876": { @@ -35491,6 +36295,46 @@ "tool": "greptile-v4-1", "repo_name": "sentry__sentry__greptile-v4-1__PR67876__20260406", "pr_url": "https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR67876__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Null reference if github_authenticated_user state is missing", + "severity": "Medium", + "matched_candidate": "Potential null reference when github_authenticated_user state is not set - accessing state without checking existence can raise KeyError or AttributeError", + "confidence": 0.95, + "reasoning": "The candidate issue identifies the same problem as the golden comment - both are pointing out that accessing github_authenticated_user state without checking if it exists can cause a null/missing reference error. The candidate mentions KeyError or AttributeError which are the specific exceptions that would occur when the state is missing, which is semantically equivalent to 'null reference if state is missing'." + }, + { + "golden_comment": "OAuth state uses pipeline.signature (static) instead of a per-request random value", + "severity": "Medium", + "matched_candidate": "OAuth state uses static pipeline.signature instead of per-request random nonce, allowing CSRF attacks on the OAuth callback flow", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: OAuth state uses pipeline.signature (which is static) instead of a per-request random value/nonce. The candidate also mentions the security implication (CSRF attacks), but the core issue identified is identical." + }, + { + "golden_comment": "The code attempts to access integration.metadata[sender][login] without checking for the existence of the sender key. This causes a KeyError for integrations where the sender metadata was not set during creation", + "severity": "High", + "matched_candidate": "Missing key existence check before accessing integration.metadata[sender][login] - accessing nested dict key 'sender' without guard will raise KeyError when sender is absent", + "confidence": 0.98, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment - accessing integration.metadata[sender][login] without checking if the 'sender' key exists first, which will cause a KeyError when sender is absent/not set. Both comments point to the same missing key existence check for the 'sender' key in the metadata dictionary." + } + ], + "false_positives": [], + "false_negatives": [], + "errors": [], + "total_candidates": 3, + "total_golden": 3, + "tp": 3, + "fp": 0, + "fn": 0, + "errors_count": 0, + "precision": 1.0, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/getsentry/sentry/pull/67876" } }, "https://github.com/keycloak/keycloak/pull/32918": { @@ -36880,6 +37724,46 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak__greptile-v4-1__PR32918__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR32918__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Recursive caching call using session instead of delegate", + "severity": "Critical", + "matched_candidate": "Recursive caching call uses session instead of delegate - the cache method calls session's method which re-enters caching, causing infinite recursion or incorrect cache behavior", + "confidence": 1.0, + "reasoning": "The candidate issue directly matches the golden comment. Both identify the same problem: a recursive caching call that incorrectly uses 'session' instead of 'delegate'. The candidate provides additional context about the consequences (infinite recursion or incorrect cache behavior), but it's describing the exact same underlying bug." + }, + { + "golden_comment": "Cleanup reference uses incorrect alias - should be 'idp-alias-' + i instead of 'alias'.", + "severity": "Medium", + "matched_candidate": "Cleanup reference uses incorrect alias - should be 'idp-alias-' + i but uses 'alias', resulting in cleanup targeting wrong identity provider configuration", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment - that the cleanup reference uses 'alias' instead of the correct 'idp-alias-' + i format, which would cause cleanup to target the wrong identity provider configuration." + } + ], + "false_positives": [ + { + "candidate": "Stream.of(values()).map(LoginFilter::getFilter).reduce(Predicate::and).get() will throw NoSuchElementException if values() returns an empty array" + }, + { + "candidate": "Potential null pointer exception when this.organization is null but this.organization.getId() is called" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 4, + "total_golden": 2, + "tp": 2, + "fp": 2, + "fn": 0, + "errors_count": 0, + "precision": 0.5, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/keycloak/keycloak/pull/32918" } }, "https://github.com/grafana/grafana/pull/94942": { @@ -38277,6 +39161,43 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR94942__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR94942__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The enableSqlExpressions function has flawed logic that always returns false, effectively disabling SQL expressions unconditionally:", + "severity": "Critical", + "matched_candidate": "Function always returns false regardless of feature flag state due to inverted logic and incorrect control flow", + "confidence": 0.95, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment - that the function always returns false regardless of the feature flag state. The candidate mentions 'inverted logic and incorrect control flow' which describes the same flawed logic that causes the function to always return false, effectively disabling SQL expressions unconditionally." + }, + { + "golden_comment": "Several methods such as NewInMemoryDB().RunCommands and db.QueryFramesInto return 'not implemented'.", + "severity": "High", + "matched_candidate": "Dead code: NewInMemoryDB().RunCommands and db.QueryFramesInto always return \"not implemented\" - these exported methods are unusable; either implement them or remove them to avoid confusion", + "confidence": 0.98, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment - that NewInMemoryDB().RunCommands and db.QueryFramesInto return 'not implemented'. The candidate adds a recommendation to either implement or remove them, but the core issue identified is identical." + } + ], + "false_positives": [ + { + "candidate": "Missing return statement after error assignment - execution continues after logging error and setting rsp.Error" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 3, + "total_golden": 2, + "tp": 2, + "fp": 1, + "fn": 0, + "errors_count": 0, + "precision": 0.6666666666666666, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/94942" } }, "https://github.com/grafana/grafana/pull/90939": { @@ -39620,6 +40541,43 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR90939__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR90939__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The GetWebAssets function implements an incomplete double-checked locking pattern for caching web assets. The function first checks if the cache is populated using a read lock (RLock), and if the cache is empty, it acquires a write lock to populate it. However, it fails to re-check whether the cache was populated by another goroutine while waiting to acquire the write lock.", + "severity": "Medium", + "matched_candidate": "Incomplete double-checked locking for caching: cache is checked without the lock, then locked to set, but no second check after acquiring lock - concurrent goroutines can both see nil and both fetch the assets", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: the double-checked locking pattern is incomplete because after acquiring the write lock, there's no second check to verify if another goroutine already populated the cache while waiting for the lock. Both comments point out that concurrent goroutines can see the cache as empty and both attempt to fetch/populate the assets." + }, + { + "golden_comment": "In addition to the missing double-check, the function has a critical flaw in its error handling: it unconditionally assigns the fetch result to the cache (line 69: entryPointAssetsCache = result) regardless of whether the fetch succeeded or failed. When an error occurs during asset fetching, result is nil, and this nil value overwrites any previously valid cache entry.", + "severity": "High", + "matched_candidate": "Critical cache corruption: entryPointAssetsCache is unconditionally overwritten with the fetch result even when the fetch returns an error - a failed fetch will replace valid cached data with an error value", + "confidence": 0.95, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: the cache (entryPointAssetsCache) is unconditionally overwritten with the fetch result, even when the fetch fails/returns an error. Both comments point out that this causes valid cached data to be replaced with nil/error values. The candidate uses slightly different wording ('error value' vs 'nil') but describes the same critical flaw in error handling." + } + ], + "false_positives": [ + { + "candidate": "Potential null pointer dereference: 'ret' captures entryPointAssetsCache under RLock but could be nil, and is returned without validation when cfg.Env != setting.Dev" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 4, + "total_golden": 2, + "tp": 2, + "fp": 1, + "fn": 0, + "errors_count": 0, + "precision": 0.5, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/90939" } }, "https://github.com/grafana/grafana/pull/80329": { @@ -41010,6 +41968,36 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR80329__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR80329__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The code uses Error log level for what appears to be debugging information. This will pollute error logs in production. Consider using Debug or Info level instead.", + "severity": "Low", + "matched_candidate": "Using log.Error() instead of log.Debug() or log.Info() for normal operational logging during cleanup process", + "confidence": 0.95, + "reasoning": "The candidate issue identifies the same problem as the golden comment - using log.Error() for what should be normal operational/debugging logging. Both comments point out that Error level is inappropriate for this type of logging and suggest using Debug or Info level instead. The candidate specifically mentions 'cleanup process' while the golden mentions 'debugging information', but they're addressing the same underlying issue of incorrect log level usage." + } + ], + "false_positives": [ + { + "candidate": "Ticker is never stopped, causing a goroutine leak when the function returns" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 2, + "total_golden": 1, + "tp": 1, + "fp": 1, + "fn": 0, + "errors_count": 0, + "precision": 0.5, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/80329" } }, "https://github.com/grafana/grafana/pull/90045": { @@ -42971,6 +43959,53 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR90045__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR90045__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The context is being created with d.Log instead of the log variable that was initialized with additional context values (name, kind, method). This means those values won't be propagated to the logging context.", + "severity": "Medium", + "matched_candidate": "Context created with d.Log instead of the log variable that was initialized with additional context values - log entries will be missing the name, kind, and method fields", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the context is being created with d.Log instead of the log variable that was initialized with additional context values (name, kind, method), which means those values won't be propagated to the logging context. The wording is slightly different but describes the identical bug." + }, + { + "golden_comment": "Bug: calling recordLegacyDuration when storage operation fails should be recordStorageDuration.", + "severity": "High", + "matched_candidate": "Bug: recordLegacyDuration is called when storage operation fails but the correct function to call for storage failures is recordStorageDuration - wrong metric is being recorded on error path", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same bug as the golden comment: when a storage operation fails, the code incorrectly calls recordLegacyDuration instead of recordStorageDuration. Both comments point to the same incorrect function call on the error path." + }, + { + "golden_comment": "Inconsistency: using name instead of options.Kind for metrics recording differs from other methods.", + "severity": "Medium", + "matched_candidate": "Inconsistency: metrics recorded using name instead of options.Kind - other methods in this file use options.Kind for metric dimensions, causing this method to emit metrics with a different label format", + "confidence": 0.99, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: using 'name' instead of 'options.Kind' for metrics recording, which is inconsistent with other methods in the file. The candidate even provides additional context about the impact (different label format), but it's pointing to the identical inconsistency issue." + } + ], + "false_positives": [ + { + "candidate": "Variable 'l' is declared as nil and never assigned a value, but is used to initialize legacyStoreMock" + }, + { + "candidate": "Variable 's' is declared as nil and never assigned a value, but is used to initialize storageMock" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 3, + "tp": 3, + "fp": 2, + "fn": 0, + "errors_count": 0, + "precision": 0.6, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/90045" } }, "https://github.com/grafana/grafana/pull/106778": { @@ -44654,6 +45689,46 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR106778__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR106778__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The rendered GrafanaRuleListItem is missing the required key prop for React list items. This can cause rendering issues when the list order changes.", + "severity": "Medium", + "matched_candidate": "GrafanaRuleListItem rendered in a list is missing the required React key prop - missing key causes React reconciliation issues when list order changes, leading to incorrect DOM updates", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the GrafanaRuleListItem component rendered in a list is missing the required React key prop, which can cause rendering/reconciliation issues when the list order changes. Both comments point to the same bug with the same explanation of consequences." + }, + { + "golden_comment": "RuleActionsButtons is invoked with only promRule, but SilenceGrafanaRuleDrawer inside RuleActionsButtons still depends on a Grafana Ruler rule being present, so for Grafana rules coming from list views the 'Silence notifications' menu entry (now driven by Grafana Prom abilities) will toggle showSilenceDrawer without ever rendering the drawer. This means clicking 'Silence notifications' for these rules has no visible effect, even when abilities indicate silencing is allowed.", + "severity": "High", + "matched_candidate": "RuleActionsButtons is invoked with only promRule argument but SilenceGrafanaRuleDrawer inside requires a Grafana Ruler rule - passing only a Prometheus rule will cause missing prop error or incorrect drawer behavior", + "confidence": 0.9, + "reasoning": "The candidate issue identifies the same core problem: RuleActionsButtons is invoked with only promRule, but SilenceGrafanaRuleDrawer inside requires a Grafana Ruler rule. The candidate mentions this will cause 'missing prop error or incorrect drawer behavior', which aligns with the golden comment's description that clicking 'Silence notifications' will have 'no visible effect' because the drawer won't render properly. Both identify the same mismatch between what's passed to RuleActionsButtons and what SilenceGrafanaRuleDrawer needs." + } + ], + "false_positives": [ + { + "candidate": "Redundant type check 'rule && rule.type === PromRuleType.Alerting' after prometheusRuleType.grafana.alertingRule(rule) already verified this condition" + }, + { + "candidate": "Function calls prometheusRuleType.grafana.rule() but uses result in boolean context without handling potential undefined/null return values" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 4, + "total_golden": 2, + "tp": 2, + "fp": 2, + "fn": 0, + "errors_count": 0, + "precision": 0.5, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/106778" } }, "https://github.com/grafana/grafana/pull/107534": { @@ -45885,6 +46960,36 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR107534__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR107534__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [], + "false_positives": [ + { + "candidate": "Expected expression includes regex pattern '20|10' but mock data returns string values that would be sorted as ['1', '10', '2', '20', '3'], making the first two values '1' and '10' instead of the expected '20' and '10'" + }, + { + "candidate": "Template variables are applied to filtered queries but the original query objects may be mutated, potentially affecting other parts of the application that reference the same query objects" + } + ], + "false_negatives": [ + { + "golden_comment": "The applyTemplateVariables method is called with request.filters as the third parameter, but this parameter is not used in the corresponding test setup.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 2, + "total_golden": 1, + "tp": 0, + "fp": 2, + "fn": 1, + "errors_count": 0, + "precision": 0.0, + "recall": 0.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/107534" } }, "https://github.com/grafana/grafana/pull/79265": { @@ -48041,6 +49146,62 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR79265__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR79265__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Race condition: Multiple concurrent requests could pass the device count check simultaneously and create devices beyond the limit. Consider using a database transaction or lock.", + "severity": "High", + "matched_candidate": "Race condition: multiple concurrent requests can simultaneously pass the device count check and create devices beyond the limit - there is no atomic check-and-increment to prevent concurrent overwrites", + "confidence": 0.98, + "reasoning": "The candidate issue identifies exactly the same race condition problem as the golden comment - that multiple concurrent requests can pass the device count check simultaneously and create devices beyond the limit. Both comments point to the lack of atomic check-and-increment operation. The candidate even uses very similar wording ('multiple concurrent requests can simultaneously pass the device count check and create devices beyond the limit'). The only difference is the suggested solution - golden suggests 'database transaction or lock' while candidate mentions 'no atomic check-and-increment' - but they're identifying the same underlying race condition bug." + }, + { + "golden_comment": "Anonymous authentication now fails entirely if anonDeviceService.TagDevice returns ErrDeviceLimitReached. Previously, device tagging was asynchronous and non-blocking. This change prevents anonymous users from authenticating when the device limit is reached.", + "severity": "Medium", + "matched_candidate": "Anonymous authentication now fails entirely when ErrDeviceLimitReached - previously device tagging was asynchronous and non-blocking, now it blocks authentication for users over the device limit", + "confidence": 0.99, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: anonymous authentication now fails entirely when ErrDeviceLimitReached is returned, whereas previously device tagging was asynchronous and non-blocking. Both comments point out that this change prevents anonymous users from authenticating when the device limit is reached." + }, + { + "golden_comment": "This call won\u2019t compile: dbSession.Exec(args...) is given a []interface{} where the first element is the query, but Exec\u2019s signature requires a first parameter of type string (not an interface{} splat).", + "severity": "Medium", + "matched_candidate": "Type error: dbSession.Exec(args...) spreads a []interface{} where the first element is the query string, but Exec expects the query as a separate first argument - this will panic at runtime", + "confidence": 0.95, + "reasoning": "Both comments identify the same issue: the code incorrectly spreads a []interface{} containing the query as the first element to dbSession.Exec(), when Exec expects the query string as a separate first argument. The golden comment says it 'won't compile' while the candidate says it will 'panic at runtime' - this is a minor technical difference in the predicted behavior, but they're identifying the exact same underlying bug with the same root cause (incorrect argument spreading to Exec)." + } + ], + "false_positives": [ + { + "candidate": "Error from TagDevice is returned directly when it's ErrDeviceLimitReached, but other errors are only logged as warnings and authentication continues, creating inconsistent error handling where some failures block authentication while others silently proceed" + }, + { + "candidate": "SQL injection vulnerability in updateDevice function - query string is concatenated with user-controlled parameters and passed to Exec without proper parameter binding" + } + ], + "false_negatives": [ + { + "golden_comment": "Returning ErrDeviceLimitReached when no rows were updated is misleading; the device might not exist.", + "severity": "Low" + }, + { + "golden_comment": "Time window calculation inconsistency: Using device.UpdatedAt.UTC().Add(-anonymousDeviceExpiration) as the lower bound but device.UpdatedAt as the current time may not match the intended logic. Consider using time.Now().UTC() consistently.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 5, + "tp": 3, + "fp": 2, + "fn": 2, + "errors_count": 0, + "precision": 0.6, + "recall": 0.6, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/79265" } }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9": { @@ -49494,6 +50655,44 @@ "tool": "greptile-v4-1", "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR9__20260405", "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR9__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Thread-safety issue with lazy @loaded_locales", + "severity": "Low", + "matched_candidate": "Thread-safety issue with lazy @loaded_locales - multiple threads can simultaneously check and set the instance variable without synchronization, causing duplicate locale loading or data corruption", + "confidence": 1.0, + "reasoning": "The candidate issue directly matches the golden comment. Both identify the same thread-safety concern with the lazy initialization of @loaded_locales. The candidate provides additional detail about the specific problem (multiple threads checking and setting without synchronization, causing duplicate loading or data corruption), but it's addressing the exact same underlying issue." + } + ], + "false_positives": [ + { + "candidate": "Object.send bypasses method visibility (private/protected) and can invoke any method including dangerous ones" + }, + { + "candidate": "Inconsistent locale loading logic - translate method duplicates the loaded check from ensure_loaded! but doesn't use ensure_loaded! method" + } + ], + "false_negatives": [ + { + "golden_comment": "Consider normalizing the input locale (e.g., to a symbol) when checking/loading here to avoid double-loading if the same locale is passed as a String vs Symbol (also applies to other locations in the PR).", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 4, + "total_golden": 2, + "tp": 1, + "fp": 2, + "fn": 1, + "errors_count": 0, + "precision": 0.25, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9" } }, "https://github.com/grafana/grafana/pull/76186": { @@ -50887,6 +52086,44 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR76186__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR76186__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The ContextualLoggerMiddleware methods (QueryData, CallResource, CheckHealth, CollectMetrics) panic when a nil request is received. This occurs because they directly access req.PluginContext (via the instrumentContext function) without first checking if req is nil. This is a regression, as previous middleware layers gracefully handled nil requests.", + "severity": "High", + "matched_candidate": "ContextualLoggerMiddleware methods (QueryData, CallResource, CheckHealth, CollectMetrics) will panic when passed a nil request - dereferencing nil pointer before nil check causes runtime panic", + "confidence": 0.98, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the ContextualLoggerMiddleware methods (QueryData, CallResource, CheckHealth, CollectMetrics) will panic when passed a nil request because they dereference the nil pointer (to access req.PluginContext via instrumentContext) before checking if req is nil. Both comments identify the same bug, the same affected methods, and the same root cause." + } + ], + "false_positives": [ + { + "candidate": "FromContext method creates a new TestLogger instance instead of returning the current instance, losing all accumulated test data" + }, + { + "candidate": "Type assertion to *log.ConcreteLogger without checking if the returned logger is nil, which could cause a panic when accessing methods on a nil pointer" + } + ], + "false_negatives": [ + { + "golden_comment": "The traceID is no longer logged for plugin requests. During a refactoring, the tracing import and the logic to extract and add traceID from the context to log parameters were removed from the LoggerMiddleware. The newly introduced ContextualLoggerMiddleware does not add this information, resulting in missing traceID in plugin request logs and impacting debugging and request tracing capabilities.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 3, + "total_golden": 2, + "tp": 1, + "fp": 2, + "fn": 1, + "errors_count": 0, + "precision": 0.3333333333333333, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/76186" } }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10": { @@ -53241,6 +54478,57 @@ "tool": "greptile-v4-1", "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR10__20260405", "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR10__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "NoMethodError before_validation in EmbeddableHost", + "severity": "Critical", + "matched_candidate": "NoMethodError in EmbeddableHost before_validation callback - calling a method on a nil object before validation will raise NoMethodError and prevent record from being saved", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment - a NoMethodError occurring in the EmbeddableHost before_validation callback. The candidate provides additional detail about the cause (calling a method on a nil object) and consequence (preventing record from being saved), but it's clearly identifying the same underlying issue." + }, + { + "golden_comment": "The update and destroy methods in Admin::EmbeddableHostsController do not validate the existence of the EmbeddableHost record retrieved by ID. If EmbeddableHost.where(id: params[:id]).first returns nil (i.e., the host does not exist), attempting to call methods on the nil object (e.g., save_host or destroy) will result in a NoMethodError.", + "severity": "Medium", + "matched_candidate": "EmbeddableHostsController update and destroy methods do not validate record existence - EmbeddableHost.where(id: params[:id]).first returns nil when record not found, calling update or destroy on nil raises NoMethodError", + "confidence": 1.0, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: both point out that the update and destroy methods in EmbeddableHostsController do not validate record existence, and that EmbeddableHost.where(id: params[:id]).first can return nil, which would cause a NoMethodError when trying to call methods on the nil object." + }, + { + "golden_comment": "record_for_host compares lower(host) = ? but does not normalize the parameter\u2019s case, so mixed\u2011case referer hosts may fail to match even though comparison intends to be case\u2011insensitive.", + "severity": "Medium", + "matched_candidate": "record_for_host compares lower(host) = ? but does not normalize the parameter case - mixed-case referer hosts like Example.COM will fail to match even though they are equivalent", + "confidence": 0.99, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the record_for_host function compares lower(host) = ? but doesn't normalize the parameter's case, so mixed-case referer hosts will fail to match despite the intended case-insensitive comparison. The candidate even provides a concrete example (Example.COM) to illustrate the issue." + }, + { + "golden_comment": "Because this migration inserts embeddable_hosts rows with raw SQL, any existing embeddable_hosts values that include http:// or /https:// or path segments won\u2019t go through the EmbeddableHost model\u2019s normalization, so the new host lookup (which compares only the bare host) may fail for migrated data. Consider ensuring that migrated hosts are normalized to the same format as newly created EmbeddableHost records so existing embedding configurations keep working.", + "severity": "High", + "matched_candidate": "Migration uses raw SQL to insert embeddable_hosts rows - any existing values containing http:// or https:// scheme prefixes will be duplicated with different formats, creating data inconsistency", + "confidence": 0.85, + "reasoning": "Both comments identify the same core issue: the migration uses raw SQL to insert embeddable_hosts rows, and existing values containing http:// or https:// prefixes won't be properly normalized. The golden comment emphasizes that this will cause lookup failures for migrated data, while the candidate focuses on data inconsistency/duplication. Both are pointing to the same underlying problem - that raw SQL insertion bypasses the EmbeddableHost model's normalization, leading to format mismatches between migrated and newly created records." + } + ], + "false_positives": [ + { + "candidate": "Potential NoMethodError on nil - value may be nil from hash lookup or safe navigation, calling methods on nil will raise exception" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 4, + "tp": 4, + "fp": 1, + "fn": 0, + "errors_count": 0, + "precision": 0.8, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10" } }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7": { @@ -54845,197 +56133,228 @@ "errors_count": 0, "precision": 0.0, "recall": 0.0, - "tool": "propel-v2", - "repo_name": "discourse__discourse-graphite__propel-v2__PR7__20260325", - "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__propel-v2__PR7__20260325/pull/1" - }, - "codeant-v2": { - "skipped": false, - "true_positives": [], - "false_positives": [ - { - "candidate": "In login.scss line 39, dark-light-choose arguments are swapped - using primary color for dark-theme branch and secondary for light-theme branch, contradicting the requirement that dark-theme lightness adjustments use secondary color" - }, - { - "candidate": "In login.scss line 54, button color uses primary color for dark-theme branch and secondary for light-theme branch, violating the intent that dark-theme lightness scaling should use secondary color" - }, - { - "candidate": "In search.scss lines 31-37, dark/light theme helper applies scale-color to $primary for dark theme and $secondary for light theme, contradicting requirement that dark theme lightness scaling should be based on $secondary" - }, - { - "candidate": "In buttons.css.scss line 60, disabled button text color is only changed on hover because the color rule is nested under :hover selector, causing disabled buttons to visually react to hover and leaving non-hover state with wrong color in dark themes" - }, - { - "candidate": "In queued-posts.scss line 14, dark-light-choose arguments are reversed - lightness scaling applied to $primary in dark position and $secondary in light position, causing dark themes to use wrong base color" - }, - { - "candidate": "In upload.scss line 20, dark-theme branch is built from primary color instead of secondary color, so dark mode .description and .hint will render with wrong palette" - }, - { - "candidate": "In user.scss line 59, dark-light-choose first branch scales $primary instead of $secondary, so dark mode instructions text uses wrong base color" - }, - { - "candidate": "In user.scss line 173, dark-light-choose has $primary in dark-theme position for table header text, so dark mode uses scaled $primary instead of required $secondary" - }, - { - "candidate": "In user.scss line 514, notification icon color uses dark-light-choose with scaled $primary as first (dark-theme) argument, so dark mode icons are based on $primary instead of $secondary" - } - ], - "false_negatives": [ - { - "golden_comment": "In .topic-meta-data h5 a, the original code had color: scale-color($primary, $lightness: 30%) but was changed to dark-light-choose(scale-color($primary, $lightness: 70%), scale-color($secondary, $lightness: 30%)). The lightness for the light theme changed from 30% to 70%, which is a dramatic inversion", - "severity": "Low" - }, - { - "golden_comment": "This change for desktop/user.css changes $primary from 30% to 50% for the light theme; most other changes preserve the original $primary value and move the complement to $secondary for dark. Consider reviewing this (also applies to a similar .name change in the mobile variant).", - "severity": "Low" - }, - { - "golden_comment": "In topic-post.css the original code used $lightness: 70% but the replacement uses $lightness: 30% for the light theme. This makes the text significantly darker than intended.", - "severity": "Low" - } - ], - "errors": [], - "total_candidates": 9, - "total_golden": 3, - "tp": 0, - "fp": 9, - "fn": 3, - "errors_count": 0, - "precision": 0.0, - "recall": 0.0, - "tool": "codeant-v2", - "repo_name": "discourse__discourse-graphite__codeant-v2__PR7__20260325", - "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__codeant-v2__PR7__20260325/pull/1" - }, - "qodo-extended-v2": { - "skipped": false, - "true_positives": [ - { - "golden_comment": "In .topic-meta-data h5 a, the original code had color: scale-color($primary, $lightness: 30%) but was changed to dark-light-choose(scale-color($primary, $lightness: 70%), scale-color($secondary, $lightness: 30%)). The lightness for the light theme changed from 30% to 70%, which is a dramatic inversion", - "severity": "Low", - "matched_candidate": "In desktop/topic-post.scss, .topic-meta-data h5 a uses inverted dark-light-choose values (70% for light theme instead of 30%), making the light theme link color much lighter than before and inconsistent with base topic-post link styling", - "confidence": 0.98, - "reasoning": "The candidate issue identifies the exact same problem as the golden comment: in .topic-meta-data h5 a, the dark-light-choose function uses 70% lightness for the light theme instead of the original 30%, which is described as 'inverted' values making the light theme link color much lighter. This matches the golden comment's observation about the lightness changing from 30% to 70% being a 'dramatic inversion'." - }, - { - "golden_comment": "This change for desktop/user.css changes $primary from 30% to 50% for the light theme; most other changes preserve the original $primary value and move the complement to $secondary for dark. Consider reviewing this (also applies to a similar .name change in the mobile variant).", - "severity": "Low", - "matched_candidate": "In desktop/user.scss and mobile/user.scss, .group-member-info .name was changed to 50/50 mapping (same as .title) instead of 30/70, removing the intended visual hierarchy between names and titles", - "confidence": 0.9, - "reasoning": "The candidate issue identifies the same underlying problem as the golden comment - both are pointing out that the .name change in desktop/user.scss (and mobile/user.scss) was changed to 50/50 instead of preserving the original 30/70 ratio. The golden comment specifically mentions the $primary change from 30% to 50% for light theme and notes it also applies to a similar .name change in mobile variant. The candidate correctly identifies this same concern about the visual hierarchy being affected by this ratio change." - }, - { - "golden_comment": "In topic-post.css the original code used $lightness: 70% but the replacement uses $lightness: 30% for the light theme. This makes the text significantly darker than intended.", - "severity": "Low", - "matched_candidate": "In desktop/topic-post.scss, .topic-meta-data h5 a uses inverted dark-light-choose values (70% for light theme instead of 30%), making the light theme link color much lighter than before and inconsistent with base topic-post link styling", - "confidence": 0.85, - "reasoning": "Both comments identify the same issue about the lightness values being swapped/inverted for the light theme in topic-post styling. The golden comment says the replacement uses 30% instead of 70% making text darker, while the candidate says it uses 70% instead of 30% making it lighter. They're describing the same mismatch problem from opposite perspectives - the values are inverted/swapped compared to what they should be. Both identify this as affecting the light theme link color in topic-post styling." - } - ], - "false_positives": [ - { - "candidate": "In mobile/modal.scss, .custom-message-length uses inverted lightness values (30%/70%) compared to desktop (70%/30%), making hint text darker in light theme and brighter in dark theme than intended" - }, - { - "candidate": "In mobile/topic-post.scss, .topic-map h3 was changed to 50/50 mapping but desktop uses 20/80, causing mobile heading to become much lighter in light theme and inconsistent with desktop" - } - ], - "false_negatives": [], - "errors": [], - "total_candidates": 4, - "total_golden": 3, - "tp": 3, - "fp": 2, - "fn": 0, - "errors_count": 0, - "precision": 0.75, - "recall": 1.0, - "tool": "qodo-extended-v2", - "repo_name": "discourse__discourse-graphite__qodo-extended-v2__PR7__20260329", - "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-extended-v2__PR7__20260329/pull/1" - }, - "mergemonkey": { - "skipped": false, - "true_positives": [ - { - "golden_comment": "In .topic-meta-data h5 a, the original code had color: scale-color($primary, $lightness: 30%) but was changed to dark-light-choose(scale-color($primary, $lightness: 70%), scale-color($secondary, $lightness: 30%)). The lightness for the light theme changed from 30% to 70%, which is a dramatic inversion", - "severity": "Low", - "matched_candidate": "Light-mode lightness changed from 30% to 70% for .embedded-posts .topic-meta-data h5 a, making link text very faint on white background, should be dark-light-choose(scale-color($primary, $lightness: 30%), scale-color($secondary, $lightness: 70%))", - "confidence": 0.95, - "reasoning": "The candidate issue identifies the same problem as the golden comment - the lightness value for the light theme changed from 30% to 70% in the .topic-meta-data h5 a selector. The candidate correctly notes this is a dramatic change that would make text very faint. The candidate also suggests the same fix (swapping the lightness values). The only minor difference is the candidate mentions '.embedded-posts' as a parent selector, but this is the same underlying issue about the lightness inversion." - }, - { - "golden_comment": "This change for desktop/user.css changes $primary from 30% to 50% for the light theme; most other changes preserve the original $primary value and move the complement to $secondary for dark. Consider reviewing this (also applies to a similar .name change in the mobile variant).", - "severity": "Low", - "matched_candidate": ".group-member-info .name light-mode lightness changed from 30% to 50%, deviating from the standard conversion pattern that would keep 30% for light and use 70% for dark", - "confidence": 0.9, - "reasoning": "The candidate issue identifies the same problem as the golden comment - both point out that the light-mode lightness value was changed from 30% to 50% for the .name selector, which deviates from the expected pattern. The golden comment mentions this applies to desktop/user.css and notes a similar change in the mobile variant. The candidate specifically mentions '.group-member-info .name' and the same 30% to 50% change. Both are flagging the same inconsistency in the light theme color conversion." - }, - { - "golden_comment": "In topic-post.css the original code used $lightness: 70% but the replacement uses $lightness: 30% for the light theme. This makes the text significantly darker than intended.", - "severity": "Low", - "matched_candidate": "Light-mode lightness changed from 30% to 70% for .embedded-posts .topic-meta-data h5 a, making link text very faint on white background, should be dark-light-choose(scale-color($primary, $lightness: 30%), scale-color($secondary, $lightness: 70%))", - "confidence": 0.85, - "reasoning": "Both comments identify the same issue about lightness values being swapped/incorrect for the light theme in topic-post.css. The golden comment says the replacement uses 30% instead of the original 70% for light theme, making text darker. The candidate identifies the same problem but describes it from the opposite direction (30% to 70%), noting the link text becomes 'very faint'. Both are pointing to the same underlying bug about incorrect lightness values in the dark-light-choose function, though they describe the direction of change differently. The candidate's suggested fix matches what the golden comment implies should be correct." - } - ], - "false_positives": [ - { - "candidate": "Light/dark args appear swapped in mobile modal.scss - light-mode lightness changed from 70% to 30% for .custom-message-length, should be dark-light-choose(scale-color($primary, $lightness: 70%), scale-color($secondary, $lightness: 30%))" - }, - { - "candidate": "Mobile h3 light-mode lightness changed from 20% to 50%, breaking parity with desktop which uses dark-light-choose(scale-color($primary, $lightness: 20%), scale-color($secondary, $lightness: 80%))" - } - ], - "false_negatives": [], - "errors": [], - "total_candidates": 4, - "total_golden": 3, - "tp": 3, - "fp": 2, - "fn": 0, - "errors_count": 0, - "precision": 0.75, - "recall": 1.0, - "tool": "mergemonkey", - "repo_name": "discourse__discourse-graphite__mergemonkey__PR7__20260403", - "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__mergemonkey__PR7__20260403/pull/1" - }, - "greptile-v4-1": { - "skipped": false, - "true_positives": [ - { - "golden_comment": "In .topic-meta-data h5 a, the original code had color: scale-color($primary, $lightness: 30%) but was changed to dark-light-choose(scale-color($primary, $lightness: 70%), scale-color($secondary, $lightness: 30%)). The lightness for the light theme changed from 30% to 70%, which is a dramatic inversion", - "severity": "Low", - "matched_candidate": "In desktop/topic-post.scss line 291, the primary-theme lightness was accidentally changed from 30% to 70%, breaking the light-theme appearance of .topic-meta-data h5 a links - the 70% value belongs in the $secondary (dark-theme) slot, and the primary slot should stay at 30%", - "confidence": 0.95, - "reasoning": "The candidate issue identifies the same problem as the golden comment: in .topic-meta-data h5 a, the lightness value for the light theme (primary) was changed from 30% to 70%. Both comments point out this is incorrect - the candidate explicitly states '70% value belongs in the $secondary (dark-theme) slot, and the primary slot should stay at 30%'. This is the exact same bug being identified, just with slightly different wording." - } - ], - "false_positives": [], - "false_negatives": [ - { - "golden_comment": "This change for desktop/user.css changes $primary from 30% to 50% for the light theme; most other changes preserve the original $primary value and move the complement to $secondary for dark. Consider reviewing this (also applies to a similar .name change in the mobile variant).", - "severity": "Low" - }, - { - "golden_comment": "In topic-post.css the original code used $lightness: 70% but the replacement uses $lightness: 30% for the light theme. This makes the text significantly darker than intended.", - "severity": "Low" - } - ], - "errors": [], - "total_candidates": 1, - "total_golden": 3, - "tp": 1, - "fp": 0, - "fn": 2, - "errors_count": 0, - "precision": 1.0, - "recall": 0.3333333333333333, - "tool": "greptile-v4-1", - "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR7__20260405", - "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR7__20260405/pull/1" + "tool": "propel-v2", + "repo_name": "discourse__discourse-graphite__propel-v2__PR7__20260325", + "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__propel-v2__PR7__20260325/pull/1" + }, + "codeant-v2": { + "skipped": false, + "true_positives": [], + "false_positives": [ + { + "candidate": "In login.scss line 39, dark-light-choose arguments are swapped - using primary color for dark-theme branch and secondary for light-theme branch, contradicting the requirement that dark-theme lightness adjustments use secondary color" + }, + { + "candidate": "In login.scss line 54, button color uses primary color for dark-theme branch and secondary for light-theme branch, violating the intent that dark-theme lightness scaling should use secondary color" + }, + { + "candidate": "In search.scss lines 31-37, dark/light theme helper applies scale-color to $primary for dark theme and $secondary for light theme, contradicting requirement that dark theme lightness scaling should be based on $secondary" + }, + { + "candidate": "In buttons.css.scss line 60, disabled button text color is only changed on hover because the color rule is nested under :hover selector, causing disabled buttons to visually react to hover and leaving non-hover state with wrong color in dark themes" + }, + { + "candidate": "In queued-posts.scss line 14, dark-light-choose arguments are reversed - lightness scaling applied to $primary in dark position and $secondary in light position, causing dark themes to use wrong base color" + }, + { + "candidate": "In upload.scss line 20, dark-theme branch is built from primary color instead of secondary color, so dark mode .description and .hint will render with wrong palette" + }, + { + "candidate": "In user.scss line 59, dark-light-choose first branch scales $primary instead of $secondary, so dark mode instructions text uses wrong base color" + }, + { + "candidate": "In user.scss line 173, dark-light-choose has $primary in dark-theme position for table header text, so dark mode uses scaled $primary instead of required $secondary" + }, + { + "candidate": "In user.scss line 514, notification icon color uses dark-light-choose with scaled $primary as first (dark-theme) argument, so dark mode icons are based on $primary instead of $secondary" + } + ], + "false_negatives": [ + { + "golden_comment": "In .topic-meta-data h5 a, the original code had color: scale-color($primary, $lightness: 30%) but was changed to dark-light-choose(scale-color($primary, $lightness: 70%), scale-color($secondary, $lightness: 30%)). The lightness for the light theme changed from 30% to 70%, which is a dramatic inversion", + "severity": "Low" + }, + { + "golden_comment": "This change for desktop/user.css changes $primary from 30% to 50% for the light theme; most other changes preserve the original $primary value and move the complement to $secondary for dark. Consider reviewing this (also applies to a similar .name change in the mobile variant).", + "severity": "Low" + }, + { + "golden_comment": "In topic-post.css the original code used $lightness: 70% but the replacement uses $lightness: 30% for the light theme. This makes the text significantly darker than intended.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 9, + "total_golden": 3, + "tp": 0, + "fp": 9, + "fn": 3, + "errors_count": 0, + "precision": 0.0, + "recall": 0.0, + "tool": "codeant-v2", + "repo_name": "discourse__discourse-graphite__codeant-v2__PR7__20260325", + "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__codeant-v2__PR7__20260325/pull/1" + }, + "qodo-extended-v2": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "In .topic-meta-data h5 a, the original code had color: scale-color($primary, $lightness: 30%) but was changed to dark-light-choose(scale-color($primary, $lightness: 70%), scale-color($secondary, $lightness: 30%)). The lightness for the light theme changed from 30% to 70%, which is a dramatic inversion", + "severity": "Low", + "matched_candidate": "In desktop/topic-post.scss, .topic-meta-data h5 a uses inverted dark-light-choose values (70% for light theme instead of 30%), making the light theme link color much lighter than before and inconsistent with base topic-post link styling", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: in .topic-meta-data h5 a, the dark-light-choose function uses 70% lightness for the light theme instead of the original 30%, which is described as 'inverted' values making the light theme link color much lighter. This matches the golden comment's observation about the lightness changing from 30% to 70% being a 'dramatic inversion'." + }, + { + "golden_comment": "This change for desktop/user.css changes $primary from 30% to 50% for the light theme; most other changes preserve the original $primary value and move the complement to $secondary for dark. Consider reviewing this (also applies to a similar .name change in the mobile variant).", + "severity": "Low", + "matched_candidate": "In desktop/user.scss and mobile/user.scss, .group-member-info .name was changed to 50/50 mapping (same as .title) instead of 30/70, removing the intended visual hierarchy between names and titles", + "confidence": 0.9, + "reasoning": "The candidate issue identifies the same underlying problem as the golden comment - both are pointing out that the .name change in desktop/user.scss (and mobile/user.scss) was changed to 50/50 instead of preserving the original 30/70 ratio. The golden comment specifically mentions the $primary change from 30% to 50% for light theme and notes it also applies to a similar .name change in mobile variant. The candidate correctly identifies this same concern about the visual hierarchy being affected by this ratio change." + }, + { + "golden_comment": "In topic-post.css the original code used $lightness: 70% but the replacement uses $lightness: 30% for the light theme. This makes the text significantly darker than intended.", + "severity": "Low", + "matched_candidate": "In desktop/topic-post.scss, .topic-meta-data h5 a uses inverted dark-light-choose values (70% for light theme instead of 30%), making the light theme link color much lighter than before and inconsistent with base topic-post link styling", + "confidence": 0.85, + "reasoning": "Both comments identify the same issue about the lightness values being swapped/inverted for the light theme in topic-post styling. The golden comment says the replacement uses 30% instead of 70% making text darker, while the candidate says it uses 70% instead of 30% making it lighter. They're describing the same mismatch problem from opposite perspectives - the values are inverted/swapped compared to what they should be. Both identify this as affecting the light theme link color in topic-post styling." + } + ], + "false_positives": [ + { + "candidate": "In mobile/modal.scss, .custom-message-length uses inverted lightness values (30%/70%) compared to desktop (70%/30%), making hint text darker in light theme and brighter in dark theme than intended" + }, + { + "candidate": "In mobile/topic-post.scss, .topic-map h3 was changed to 50/50 mapping but desktop uses 20/80, causing mobile heading to become much lighter in light theme and inconsistent with desktop" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 4, + "total_golden": 3, + "tp": 3, + "fp": 2, + "fn": 0, + "errors_count": 0, + "precision": 0.75, + "recall": 1.0, + "tool": "qodo-extended-v2", + "repo_name": "discourse__discourse-graphite__qodo-extended-v2__PR7__20260329", + "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo-extended-v2__PR7__20260329/pull/1" + }, + "mergemonkey": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "In .topic-meta-data h5 a, the original code had color: scale-color($primary, $lightness: 30%) but was changed to dark-light-choose(scale-color($primary, $lightness: 70%), scale-color($secondary, $lightness: 30%)). The lightness for the light theme changed from 30% to 70%, which is a dramatic inversion", + "severity": "Low", + "matched_candidate": "Light-mode lightness changed from 30% to 70% for .embedded-posts .topic-meta-data h5 a, making link text very faint on white background, should be dark-light-choose(scale-color($primary, $lightness: 30%), scale-color($secondary, $lightness: 70%))", + "confidence": 0.95, + "reasoning": "The candidate issue identifies the same problem as the golden comment - the lightness value for the light theme changed from 30% to 70% in the .topic-meta-data h5 a selector. The candidate correctly notes this is a dramatic change that would make text very faint. The candidate also suggests the same fix (swapping the lightness values). The only minor difference is the candidate mentions '.embedded-posts' as a parent selector, but this is the same underlying issue about the lightness inversion." + }, + { + "golden_comment": "This change for desktop/user.css changes $primary from 30% to 50% for the light theme; most other changes preserve the original $primary value and move the complement to $secondary for dark. Consider reviewing this (also applies to a similar .name change in the mobile variant).", + "severity": "Low", + "matched_candidate": ".group-member-info .name light-mode lightness changed from 30% to 50%, deviating from the standard conversion pattern that would keep 30% for light and use 70% for dark", + "confidence": 0.9, + "reasoning": "The candidate issue identifies the same problem as the golden comment - both point out that the light-mode lightness value was changed from 30% to 50% for the .name selector, which deviates from the expected pattern. The golden comment mentions this applies to desktop/user.css and notes a similar change in the mobile variant. The candidate specifically mentions '.group-member-info .name' and the same 30% to 50% change. Both are flagging the same inconsistency in the light theme color conversion." + }, + { + "golden_comment": "In topic-post.css the original code used $lightness: 70% but the replacement uses $lightness: 30% for the light theme. This makes the text significantly darker than intended.", + "severity": "Low", + "matched_candidate": "Light-mode lightness changed from 30% to 70% for .embedded-posts .topic-meta-data h5 a, making link text very faint on white background, should be dark-light-choose(scale-color($primary, $lightness: 30%), scale-color($secondary, $lightness: 70%))", + "confidence": 0.85, + "reasoning": "Both comments identify the same issue about lightness values being swapped/incorrect for the light theme in topic-post.css. The golden comment says the replacement uses 30% instead of the original 70% for light theme, making text darker. The candidate identifies the same problem but describes it from the opposite direction (30% to 70%), noting the link text becomes 'very faint'. Both are pointing to the same underlying bug about incorrect lightness values in the dark-light-choose function, though they describe the direction of change differently. The candidate's suggested fix matches what the golden comment implies should be correct." + } + ], + "false_positives": [ + { + "candidate": "Light/dark args appear swapped in mobile modal.scss - light-mode lightness changed from 70% to 30% for .custom-message-length, should be dark-light-choose(scale-color($primary, $lightness: 70%), scale-color($secondary, $lightness: 30%))" + }, + { + "candidate": "Mobile h3 light-mode lightness changed from 20% to 50%, breaking parity with desktop which uses dark-light-choose(scale-color($primary, $lightness: 20%), scale-color($secondary, $lightness: 80%))" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 4, + "total_golden": 3, + "tp": 3, + "fp": 2, + "fn": 0, + "errors_count": 0, + "precision": 0.75, + "recall": 1.0, + "tool": "mergemonkey", + "repo_name": "discourse__discourse-graphite__mergemonkey__PR7__20260403", + "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__mergemonkey__PR7__20260403/pull/1" + }, + "greptile-v4-1": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "In .topic-meta-data h5 a, the original code had color: scale-color($primary, $lightness: 30%) but was changed to dark-light-choose(scale-color($primary, $lightness: 70%), scale-color($secondary, $lightness: 30%)). The lightness for the light theme changed from 30% to 70%, which is a dramatic inversion", + "severity": "Low", + "matched_candidate": "In desktop/topic-post.scss line 291, the primary-theme lightness was accidentally changed from 30% to 70%, breaking the light-theme appearance of .topic-meta-data h5 a links - the 70% value belongs in the $secondary (dark-theme) slot, and the primary slot should stay at 30%", + "confidence": 0.95, + "reasoning": "The candidate issue identifies the same problem as the golden comment: in .topic-meta-data h5 a, the lightness value for the light theme (primary) was changed from 30% to 70%. Both comments point out this is incorrect - the candidate explicitly states '70% value belongs in the $secondary (dark-theme) slot, and the primary slot should stay at 30%'. This is the exact same bug being identified, just with slightly different wording." + } + ], + "false_positives": [], + "false_negatives": [ + { + "golden_comment": "This change for desktop/user.css changes $primary from 30% to 50% for the light theme; most other changes preserve the original $primary value and move the complement to $secondary for dark. Consider reviewing this (also applies to a similar .name change in the mobile variant).", + "severity": "Low" + }, + { + "golden_comment": "In topic-post.css the original code used $lightness: 70% but the replacement uses $lightness: 30% for the light theme. This makes the text significantly darker than intended.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 1, + "total_golden": 3, + "tp": 1, + "fp": 0, + "fn": 2, + "errors_count": 0, + "precision": 1.0, + "recall": 0.3333333333333333, + "tool": "greptile-v4-1", + "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR7__20260405", + "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR7__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [], + "false_positives": [], + "false_negatives": [ + { + "golden_comment": "In .topic-meta-data h5 a, the original code had color: scale-color($primary, $lightness: 30%) but was changed to dark-light-choose(scale-color($primary, $lightness: 70%), scale-color($secondary, $lightness: 30%)). The lightness for the light theme changed from 30% to 70%, which is a dramatic inversion", + "severity": "Low" + }, + { + "golden_comment": "This change for desktop/user.css changes $primary from 30% to 50% for the light theme; most other changes preserve the original $primary value and move the complement to $secondary for dark. Consider reviewing this (also applies to a similar .name change in the mobile variant).", + "severity": "Low" + }, + { + "golden_comment": "In topic-post.css the original code used $lightness: 70% but the replacement uses $lightness: 30% for the light theme. This makes the text significantly darker than intended.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 0, + "total_golden": 3, + "tp": 0, + "fp": 0, + "fn": 3, + "errors_count": 0, + "precision": 0.0, + "recall": 0.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7" } }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8": { @@ -57006,6 +58325,53 @@ "tool": "greptile-v4-1", "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR8__20260405", "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR8__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": " The findMembers() call is now asynchronous and unhandled. The controller may not have member data immediately available, creating a race condition.", + "severity": "High", + "matched_candidate": "findMembers() is now asynchronous and its result is not awaited - the controller returns before member data is available, causing the response to contain empty or stale member information", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: findMembers() is now asynchronous but its result is not awaited/handled, causing the controller to return before member data is available. Both describe the same race condition where the response will have empty or stale data because the async operation isn't properly handled." + }, + { + "golden_comment": "In the next action, capping the next offset at user_count can produce an empty page (e.g., total equal to limit results in offset == total, showing 2/2 with no members). This can cause confusing UX on the last page.", + "severity": "Medium", + "matched_candidate": "Capping next offset at user_count produces empty page when total equals limit - offset equals total, causing the next page to return 0 results when there should be a final page", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: capping the next offset at user_count can produce an empty page when total equals limit, resulting in offset equaling total and returning 0 results. Both comments describe the same bug with the same cause and same consequence (confusing UX with an empty final page)." + }, + { + "golden_comment": "HTTP method mismatch in .remove_member - test uses PUT but remove_member action expects DELETE", + "severity": "Medium", + "matched_candidate": "HTTP method mismatch: remove_member test uses PUT but the remove_member controller action expects DELETE - test will always fail with 404 or 405", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the remove_member test uses PUT HTTP method but the remove_member controller action expects DELETE method. Both comments point to the same HTTP method mismatch bug." + } + ], + "false_positives": [ + { + "candidate": "find_group method can return nil from Group.find_by, then guardian.ensure_can_see!(nil) will be called on a nil object" + }, + { + "candidate": "Potential NoMethodError on nil - value may be nil from hash lookup or safe navigation, and calling methods on nil will raise an exception" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 3, + "tp": 3, + "fp": 2, + "fn": 0, + "errors_count": 0, + "precision": 0.6, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8" } }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3": { @@ -58517,6 +59883,50 @@ "tool": "greptile-v4-1", "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR3__20260405", "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR3__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "BlockedEmail.should_block_email? method has side effects during a read operation - it updates statistics even when just checking if an email should be blocked. This could cause race conditions in concurrent environments and makes the method name misleading.", + "severity": "Medium", + "matched_candidate": "BlockedEmail.should_block_email? has side effects during a read operation - it updates statistics even when only checking whether an email should be blocked, causing unexpected DB writes on reads", + "confidence": 0.95, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: the BlockedEmail.should_block_email? method has side effects (updating statistics/DB writes) during what should be a read operation. Both comments point out that this is problematic because it causes unexpected writes when only checking if an email should be blocked. The candidate mentions 'unexpected DB writes on reads' while the golden mentions 'race conditions' and 'misleading method name', but the core issue identified is identical." + } + ], + "false_positives": [ + { + "candidate": "Password length validation uses incorrect condition that allows nil passwords to pass when @password_required is false" + }, + { + "candidate": "Race condition in should_block? method - multiple concurrent requests for the same email could create inconsistent match_count values due to read-modify-write operation without proper locking" + }, + { + "candidate": "Method doesn't handle nil or empty email values, will cause NoMethodError when calling =~ on nil" + }, + { + "candidate": "Missing assignment operator (=) for action_type attribute in Fabricator definition" + } + ], + "false_negatives": [ + { + "golden_comment": "Regex pattern @(#{domains}) only matches domain suffixes, not full domains. evil.example.com would match whitelist entry example.com.", + "severity": "Medium" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 1, + "fp": 4, + "fn": 1, + "errors_count": 0, + "precision": 0.2, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3" } }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5": { @@ -60018,6 +61428,33 @@ "tool": "greptile-v4-1", "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR5__20260405", "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR5__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [], + "false_positives": [], + "false_negatives": [ + { + "golden_comment": "Mixing float: left with flexbox causes layout issues. Further this PR removes the float-based right alignment for .d-header .panel, which may cause the login panel in the non-Ember/noscript header (where .panel is nested inside .row and not a flex item) to stack under the title instead of remaining right-aligned.", + "severity": "Low" + }, + { + "golden_comment": "-ms-align-items never existed in any version of IE/Edge; the correct legacy property is -ms-flex-align.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 0, + "total_golden": 2, + "tp": 0, + "fp": 0, + "fn": 2, + "errors_count": 0, + "precision": 0.0, + "recall": 0.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5" } }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6": { @@ -61347,6 +62784,39 @@ "tool": "greptile-v4-1", "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR6__20260405", "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR6__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The include_website_name method is missing the required ? suffix. Rails serializers expect include_ methods to end with ? for conditional attribute inclusion, a convention followed by other methods in this serializer. Without it, the website_name attribute may not be conditionally included as intended. Additionally, the '.' << website_host string concatenation should be replaced with '.' + website_host or '.#{website_host}' to avoid mutating string literals, which can lead to issues.", + "severity": "Medium", + "matched_candidate": "include_website_name method is missing the required ? suffix - Rails serializers expect include_ conditional methods to end with ? otherwise the attribute is always serialized", + "confidence": 0.95, + "reasoning": "The candidate issue identifies the same core problem as the golden comment - the include_website_name method is missing the required ? suffix that Rails serializers expect for conditional attribute inclusion. The candidate correctly identifies this will cause the attribute to always be serialized instead of conditionally included. While the golden comment also mentions a secondary issue about string concatenation that the candidate doesn't address, the primary issue (missing ? suffix) is correctly identified." + } + ], + "false_positives": [ + { + "candidate": "Logic error in subdomain comparison - when website domain is 'example.com' and current hostname is 'forums.example.com', the test expects 'example.com/user' but the logic should likely treat the subdomain as different from the parent domain" + }, + { + "candidate": "URI parsing is called twice on the same website string without caching, and the second call lacks error handling that could cause unhandled exceptions" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 3, + "total_golden": 1, + "tp": 1, + "fp": 2, + "fn": 0, + "errors_count": 0, + "precision": 0.3333333333333333, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6" } }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4": { @@ -64140,6 +65610,66 @@ "tool": "greptile-v4-1", "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR4__20260405", "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR4__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "postMessage targetOrigin should be the origin (scheme+host+port), not the full referrer URL; using the full URL will cause the message to be dropped and prevent resizing.", + "severity": "Medium", + "matched_candidate": "postMessage targetOrigin should be the origin (scheme+host+port), not the full referrer URL - using the full URL causes the message to be silently dropped by the browser", + "confidence": 1.0, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: the postMessage targetOrigin should be the origin (scheme+host+port) rather than the full referrer URL, and using the full URL causes the message to be dropped. Both comments point to the same bug with the same explanation of the consequence." + }, + { + "golden_comment": "The code sets X-Frame-Options: ALLOWALL which completely disables clickjacking protection. The referer validation can be bypassed (referer headers are easily spoofed), and the fallback to empty string for nil referer masks validation failures.", + "severity": "Medium", + "matched_candidate": "X-Frame-Options set to ALLOWALL completely disables clickjacking protection - referer header validation can be spoofed, leaving the endpoint vulnerable to iframe-based attacks", + "confidence": 0.9, + "reasoning": "The candidate issue identifies the same core security vulnerability as the golden comment: X-Frame-Options set to ALLOWALL disables clickjacking protection, and the referer header validation can be spoofed/bypassed. Both comments point to the same fundamental security flaw - the combination of disabled clickjacking protection and ineffective referer validation. The candidate doesn't mention the specific detail about the fallback to empty string for nil referer, but it captures the main security concern." + }, + { + "golden_comment": "The ERB block closes with end if, which is invalid Ruby/ERB and will raise at render; it should just be end to close the if block.", + "severity": "Medium", + "matched_candidate": "ERB block closes with \"end if\" which is invalid Ruby ERB syntax - this will raise a SyntaxError at render time; it should be \"end\" to close the if block", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the ERB block incorrectly closes with 'end if' instead of 'end', which is invalid Ruby/ERB syntax and will cause an error at render time. The wording is nearly identical and points to the exact same bug." + } + ], + "false_positives": [ + { + "candidate": "Race condition in Redis throttling logic - another process can set key between setnx and expire calls, causing expire to be lost and key to persist indefinitely" + }, + { + "candidate": "Transaction block does not handle the case where PostCreator.create fails but doesn't raise an exception" + } + ], + "false_negatives": [ + { + "golden_comment": "SSRF vulnerability using open(url) without validation", + "severity": "Critical" + }, + { + "golden_comment": "The current origin validation using indexOf is insufficient and can be bypassed. An attacker could use a malicious domain like evil-discourseUrl.com to pass this check.", + "severity": "Medium" + }, + { + "golden_comment": "The TopicEmbed.import method is susceptible to a NoMethodError if the contents parameter is nil when attempting to append a string, and an XSS vulnerability due to unescaped url interpolation in the generated HTML.", + "severity": "Medium" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 6, + "tp": 3, + "fp": 2, + "fn": 3, + "errors_count": 0, + "precision": 0.6, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4" } }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1": { @@ -65943,6 +67473,53 @@ "tool": "greptile-v4-1", "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR1__20260405", "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR1__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The downsize method is defined twice. The second definition, which expects a single dimensions string parameter, overrides the first, which expected separate max_width and max_height parameters. This makes the original method unreachable and breaks existing code that calls it with separate width and height arguments.", + "severity": "Medium", + "matched_candidate": "The downsize method is defined twice - the second definition overrides the first, causing dead code and potentially breaking behavior that depended on the original two-argument signature", + "confidence": 0.99, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the downsize method is defined twice, with the second definition overriding the first, causing the original method to be unreachable/dead code and breaking existing code that depends on the original two-argument signature." + }, + { + "golden_comment": "Hardcoding maxSizeKB = 10 * 1024 ignores Discourse.SiteSettings['max_' + type + '_size_kb'], so the client-side limit can diverge from server-side and per-type settings (also applies to the 413 handler below).", + "severity": "Low", + "matched_candidate": "Hardcoding maxSizeKB = 10 * 1024 ignores Discourse.SiteSettings max_type_size_kb - client-side limit can diverge from server-side configuration, allowing uploads that exceed the server limit", + "confidence": 0.95, + "reasoning": "The candidate issue identifies the same core problem as the golden comment: hardcoding maxSizeKB = 10 * 1024 instead of using Discourse.SiteSettings['max_' + type + '_size_kb'], which causes client-side limits to diverge from server-side configuration. The candidate mentions the same setting (max_type_size_kb vs max_' + type + '_size_kb are semantically equivalent references). The golden comment also mentions this applies to the 413 handler below, which the candidate doesn't explicitly mention, but the core issue identified is the same." + }, + { + "golden_comment": "Passing 80% as the dimensions can fail for animated GIFs when allow_animated_thumbnails is true, since the animated path uses gifsicle --resize-fit which expects WxH geometry, not a percentage; downsizing would then silently fail.", + "severity": "Medium", + "matched_candidate": "Passing 80% as dimensions to gifsicle fails for animated GIFs - the animated path requires absolute pixel dimensions, not percentages, causing animated thumbnail generation to fail", + "confidence": 0.95, + "reasoning": "The candidate issue identifies the same problem as the golden comment: passing 80% as dimensions to gifsicle fails for animated GIFs because the animated path requires absolute pixel dimensions (WxH geometry), not percentages. Both comments point to the same bug where percentage-based dimensions don't work with gifsicle's --resize-fit for animated thumbnails." + } + ], + "false_positives": [ + { + "candidate": "Image downsizing loop can run indefinitely if OptimizedImage.downsize fails to reduce file size, creating an infinite loop condition" + }, + { + "candidate": "tempfile.size is called without checking if tempfile is nil, which will raise NoMethodError if file download failed" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 3, + "tp": 3, + "fp": 2, + "fn": 0, + "errors_count": 0, + "precision": 0.6, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1" } }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2": { @@ -67596,6 +69173,49 @@ "tool": "greptile-v4-1", "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR2__20260405", "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR2__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "logic: Potential nil pointer exception - if no TopicUser record exists, tu will be nil and calling methods on it will crash", + "severity": "High", + "matched_candidate": "Logic assumes TopicUser record exists but find_by can return nil, leading to NoMethodError on nil object", + "confidence": 0.98, + "reasoning": "Both comments identify the same issue: the code uses find_by which can return nil, and then attempts to call methods on the potentially nil TopicUser object (tu), which would cause an error (nil pointer exception/NoMethodError on nil). The golden comment says 'nil pointer exception' while the candidate says 'NoMethodError on nil object' - these are semantically equivalent in Ruby context." + }, + { + "golden_comment": "Typo in property name: 'stopNotificiationsText' should be 'stopNotificationsText' (missing 'n' in 'Notifications')", + "severity": "Low", + "matched_candidate": "Typo in property name: 'stopNotificiationsText' should be 'stopNotificationsText' (missing 'n' in Notifications) - the property will never be found causing silent undefined access", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same typo as the golden comment: 'stopNotificiationsText' should be 'stopNotificationsText' (missing 'n' in Notifications). The candidate even adds additional context about the consequence (silent undefined access), but it's pointing to the identical spelling error." + } + ], + "false_positives": [ + { + "candidate": "Race condition: TopicUser.change is called without checking if the current post number is actually newer than the existing last_emailed_post_number" + }, + { + "candidate": "Potential variable name typo: code appears to reference 'topi' instead of 'topic_id' in MessageBus.publish call" + }, + { + "candidate": "Route 't/:topic_id/summary' will never be reached because it's defined after 't/:slug/:topic_id/summary' route, and Rails matches routes in order" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 2, + "fp": 3, + "fn": 0, + "errors_count": 0, + "precision": 0.4, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2" } }, "https://github.com/calcom/cal.com/pull/22532": { @@ -69312,6 +70932,49 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR22532__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR22532__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The updateManyByCredentialId call uses an empty data object, which prevents Prisma's @updatedAt decorator from updating the updatedAt timestamp. This results in inaccurate cache status tracking, as the timestamp isn't updated when the cache is refreshed. To fix this, explicitly set the updatedAt field.", + "severity": "Medium", + "matched_candidate": "updateManyByCredentialId called with empty data object {} - Prisma @updatedAt decorator only fires when the record has actual field changes, so the updatedAt timestamp will not be updated on cache invalidation", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: calling updateManyByCredentialId with an empty data object {} means Prisma's @updatedAt decorator won't update the updatedAt timestamp. Both comments point to the same bug where cache invalidation doesn't properly update the timestamp, though the candidate doesn't explicitly mention the fix (setting updatedAt explicitly)." + }, + { + "golden_comment": "logic: macOS-specific sed syntax with empty string after -i flag will fail on Linux systems", + "severity": "Low", + "matched_candidate": "sed command uses macOS-specific syntax with empty string after -i flag, will fail on Linux systems", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment - the macOS-specific sed syntax with empty string after -i flag that will fail on Linux systems. The wording is nearly identical, pointing to the same cross-platform compatibility issue with the sed command." + } + ], + "false_positives": [ + { + "candidate": "Race condition potential - if cache statuses are updated between the getCacheStatusByCredentialIds call and the map creation, the cacheStatusMap may contain stale data" + }, + { + "candidate": "findUniqueOrThrow method uses SelectedCalendarWhereInput instead of SelectedCalendarWhereUniqueInput, which will cause a type error since findUniqueOrThrow requires unique identifiers" + }, + { + "candidate": "deleteCacheMutation is called with credentialId but the mutation expects a different parameter structure based on the trpc endpoint name 'deleteCache'" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 2, + "fp": 3, + "fn": 0, + "errors_count": 0, + "precision": 0.4, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/22532" } }, "https://github.com/calcom/cal.com/pull/8330": { @@ -70941,6 +72604,50 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR8330__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR8330__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Using === for dayjs object comparison will always return false as it compares object references, not values. Use .isSame() method instead: dayjs(date.start).add(utcOffset, 'minutes').isSame(dayjs(date.end).add(utcOffset, minutes))", + "severity": "Medium", + "matched_candidate": "Comparison using === instead of isSame() for Dayjs objects will always be false since they are different object instances", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: using === to compare Dayjs objects will always return false because it compares object references rather than values, and suggests using isSame() method instead. Both comments point to the same bug with the same solution." + } + ], + "false_positives": [ + { + "candidate": "Variable i is initialized to -1 but the code is incomplete - this suggests a loop that may have off-by-one errors when incrementing/accessing array elements" + }, + { + "candidate": "Assertion expects results for plus2DateString but the time range query spans from plus1DateString to plus2DateString, potentially missing or incorrectly filtering results" + }, + { + "candidate": "startTime calculation duplicates the same dayjs computation twice instead of reusing the result, causing unnecessary work and potential inconsistency" + }, + { + "candidate": "endTime calculation duplicates the same dayjs computation twice instead of reusing the result, causing unnecessary work and potential inconsistency" + } + ], + "false_negatives": [ + { + "golden_comment": "Incorrect end time calculation using slotStartTime instead of slotEndTime", + "severity": "Medium" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 1, + "fp": 4, + "fn": 1, + "errors_count": 0, + "precision": 0.2, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/8330" } }, "https://github.com/calcom/cal.com/pull/14943": { @@ -72393,6 +74100,49 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR14943__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR14943__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can lose increments under concurrency; consider an atomic increment via Prisma (increment: 1) to avoid race conditions (also applies to the similar update in the catch block).", + "severity": "High", + "matched_candidate": "retryCount reads a possibly stale value with reminder.retryCount + 1 - concurrent requests can both increment from the same base, losing increments; use atomic increment via Prisma update", + "confidence": 0.99, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: using reminder.retryCount + 1 reads a possibly stale value, concurrent requests can lose increments, and the solution is to use atomic increment via Prisma. The wording is nearly identical and addresses the same race condition concern." + }, + { + "golden_comment": "The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workflow reminders (e.g., Email, WhatsApp) that have retryCount > 1. This occurs because the retryCount condition within the OR clause for deletion lacks a method: WorkflowMethods.SMS filter, causing it to apply to all reminder types instead of only SMS reminders, which is the intended scope of this function.", + "severity": "High", + "matched_candidate": "scheduleSMSReminders.ts deletion logic incorrectly deletes non-SMS workflow reminders (Email, WhatsApp) that share the same workflow - the filter condition is too broad and removes reminders of unintended types", + "confidence": 0.92, + "reasoning": "The candidate issue identifies the same core problem as the golden comment: the deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workflow reminders (Email, WhatsApp) because the filter condition is too broad. Both comments point to the same bug where reminders of unintended types are being deleted. The golden comment provides more specific technical detail about the retryCount condition lacking a method: WorkflowMethods.SMS filter, but the candidate correctly identifies the same underlying issue." + } + ], + "false_positives": [ + { + "candidate": "OR condition deletes SMS reminders with retryCount > 1, conflicting with retry logic that increments retryCount on failures" + }, + { + "candidate": "Using lte (less than or equal) with current date will delete SMS reminders scheduled for exactly the current time, preventing them from being sent if the handler runs at the exact scheduled moment" + }, + { + "candidate": "Duplicate retry count increment logic - retryCount is incremented both in the 'else' block for failed SMS and in the catch block for exceptions, potentially double-counting failures" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 2, + "fp": 3, + "fn": 0, + "errors_count": 0, + "precision": 0.4, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/14943" } }, "https://github.com/calcom/cal.com/pull/22345": { @@ -73804,6 +75554,47 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR22345__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR22345__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Fetching userIdsFromOrg only when teamsFromOrg.length > 0 can exclude org-level members for orgs without child teams; consider deriving from teamIds (which includes orgId) or removing the guard so org-only orgs still include member user bookings.", + "severity": "Medium", + "matched_candidate": "Logic error: fetching userIdsFromOrg only when teamsFromOrg.length > 0 excludes org-level members for organizations without child teams - org members not in any team will be incorrectly excluded from results", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: fetching userIdsFromOrg only when teamsFromOrg.length > 0 will exclude org-level members for organizations that don't have child teams. Both comments point to the same logic error where org members not in any team would be incorrectly excluded from the results." + } + ], + "false_positives": [ + { + "candidate": "Logic flaw in condition combining - if both authConditions and filterConditions are falsy, returns NOTHING_CONDITION which will exclude all results" + }, + { + "candidate": "Constructor accepts InsightsBookingServicePublicOptions but assigns to InsightsBookingServiceOptions type field, causing type mismatch" + }, + { + "candidate": "Missing await for async cleanup operation - test may complete before cleanup finishes, potentially causing interference with subsequent tests" + } + ], + "false_negatives": [ + { + "golden_comment": "In getBaseConditions(), the else if (filterConditions) and final else branches are unreachable. This is because getAuthorizationConditions() always returns a non-null Prisma.Sql object, making authConditions always truthy, which means only the first two if/else if conditions are ever evaluated.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 4, + "total_golden": 2, + "tp": 1, + "fp": 3, + "fn": 1, + "errors_count": 0, + "precision": 0.25, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/22345" } }, "https://github.com/calcom/cal.com/pull/11059": { @@ -76469,6 +78260,62 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR11059__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR11059__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Invalid Zod schema syntax. Computed property keys like [z.string().toString()] are not valid in Zod object schemas and will cause runtime errors. ", + "severity": "High", + "matched_candidate": "Invalid Zod schema syntax - `z.string().toString()` creates a string literal 'string' as a key, not a dynamic string key pattern", + "confidence": 0.95, + "reasoning": "Both comments identify the same issue: the invalid Zod schema syntax using `z.string().toString()` as a computed property key. The golden comment says it's not valid in Zod object schemas and will cause runtime errors. The candidate comment identifies the same problem, explaining that it creates a string literal 'string' as a key rather than a dynamic string key pattern. Both are pointing to the same underlying bug with the Zod schema syntax." + }, + { + "golden_comment": "parseRefreshTokenResponse returns a Zod safeParse result ({ success, data, error }), not the credential key object. Persisting that as key stores the wrapper instead of the token payload; we should store the parsed data or use schema parse.", + "severity": "High", + "matched_candidate": "parseRefreshTokenResponse returns a Zod safeParse result object ({ success, data, error }) not the credential object - callers that persist this directly will store the wrapper object instead of the token data", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: parseRefreshTokenResponse returns a Zod safeParse result object (with success, data, error properties) rather than the credential/token object directly, and callers that persist this will store the wrapper object instead of the actual token data. Both comments point to the same bug and suggest the same underlying issue." + }, + { + "golden_comment": "When APP_CREDENTIAL_SHARING_ENABLED and CALCOM_CREDENTIAL_SYNC_ENDPOINT are set, the refreshFunction helper returns the fetch Response, but several callers (for example GoogleCalendarService.refreshAccessToken expecting res.data, and HubspotCalendarService.refreshAccessToken expecting a HubspotToken) assume it returns the integration-specific token object. That mismatch will cause runtime errors in the sync-enabled path unless the return type or those call sites are adjusted.", + "severity": "High", + "matched_candidate": "When sync endpoint is used, res is a fetch Response object and res.data does not exist - accessing res?.data will return undefined and token.access_token will be undefined, silently breaking token refresh", + "confidence": 0.95, + "reasoning": "The candidate issue identifies the same underlying problem as the golden comment - that when the sync endpoint is used, the function returns a fetch Response object, but callers expect a different structure (res.data). The golden comment mentions this causes a type mismatch with callers expecting integration-specific token objects (like res.data or HubspotToken), and the candidate specifically points out that res.data will be undefined because Response objects don't have a data property. Both are identifying the same fundamental issue: the return type mismatch between the sync-enabled path and what callers expect." + }, + { + "golden_comment": "When the sync endpoint path is used, res is a fetch Response and has no .data; res?.data will be undefined and token.access_token will throw at runtime. This relies on a consistent return shape from refreshOAuthTokens, which isn\u2019t guaranteed currently.", + "severity": "High", + "matched_candidate": "When sync endpoint is used, res is a fetch Response object and res.data does not exist - accessing res?.data will return undefined and token.access_token will be undefined, silently breaking token refresh", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: when the sync endpoint path is used, res is a fetch Response object that doesn't have a .data property, so res?.data will be undefined and accessing token.access_token will fail. Both comments point to the same bug regarding inconsistent return shape from refreshOAuthTokens between the sync and non-sync paths." + } + ], + "false_positives": [ + { + "candidate": "Incorrect expiry date calculation - adding seconds to milliseconds timestamp" + } + ], + "false_negatives": [ + { + "golden_comment": "The parseRefreshTokenResponse function incorrectly sets refresh_token to the hardcoded string 'refresh_token' when it's missing from the OAuth refresh token response. This invalidates the token, breaking subsequent token refreshes and causing authentication failures.", + "severity": "High" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 5, + "tp": 4, + "fp": 1, + "fn": 1, + "errors_count": 0, + "precision": 0.8, + "recall": 0.8, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/11059" } }, "https://github.com/calcom/cal.com/pull/7232": { @@ -78160,6 +80007,47 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR7232__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR7232__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Asynchronous functions deleteScheduledEmailReminder and deleteScheduledSMSReminder are called without await inside forEach loops. This occurs during booking rescheduling/cancellation, and workflow/workflow step deletion/updates. Consequently, scheduled workflow reminders may not be reliably cancelled, potentially leaving them active.", + "severity": "Medium", + "matched_candidate": "deleteScheduledEmailReminder and deleteScheduledSMSReminder are async functions called without await inside forEach loops - reminder deletions fire-and-forget, errors are silently lost, and cleanup may be incomplete", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: async functions deleteScheduledEmailReminder and deleteScheduledSMSReminder are called without await inside forEach loops. Both comments point out that this leads to unreliable/incomplete cleanup of scheduled reminders. The candidate adds detail about errors being silently lost (fire-and-forget pattern), which is a consequence of the same underlying issue." + } + ], + "false_positives": [ + { + "candidate": "Race condition in onChange handler - form.getValues() may not reflect current input value due to asynchronous form updates" + }, + { + "candidate": "Database delete operation is not awaited but function continues execution, potentially causing race conditions" + }, + { + "candidate": "deleteScheduledEmailReminder is called with a third parameter 'true' but deleteScheduledSMSReminder is called without it, suggesting inconsistent API usage" + } + ], + "false_negatives": [ + { + "golden_comment": "When immediateDelete is true, the deleteScheduledEmailReminder function cancels the SendGrid email but fails to delete the corresponding WorkflowReminder record from the database. This creates orphaned database entries and is inconsistent with the immediateDelete: false path, which marks the record as cancelled. The SendGrid DELETE API call is also omitted in this path.", + "severity": "High" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 1, + "fp": 3, + "fn": 1, + "errors_count": 0, + "precision": 0.2, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/7232" } }, "https://github.com/calcom/cal.com/pull/14740": { @@ -80498,6 +82386,60 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR14740__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR14740__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Case sensitivity bypass in email blacklist", + "severity": "High", + "matched_candidate": "Case sensitivity bypass in email blacklist: comparison does not normalize email case, allowing BLOCKED@example.com to bypass a blacklist entry for blocked@example.com", + "confidence": 1.0, + "reasoning": "The candidate issue directly identifies the same problem as the golden comment - case sensitivity bypass in email blacklist. The candidate provides additional detail about how the comparison doesn't normalize email case, allowing uppercase variants to bypass lowercase blacklist entries, but it's describing the exact same underlying security vulnerability." + }, + { + "golden_comment": "The logic for checking team admin/owner permissions is incorrect. This condition uses AND (&&) which requires both isTeamAdmin AND isTeamOwner to be true, but it should use OR (||) since a user needs to be either an admin OR an owner to have permission.", + "severity": "Critical", + "matched_candidate": "Logic error using AND (&&) instead of OR (||) to check if user is team admin OR owner - currently requires user to be BOTH admin AND owner, which is incorrect", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the logic incorrectly uses AND (&&) instead of OR (||) when checking team admin/owner permissions. Both comments point out that the current code requires a user to be BOTH admin AND owner, when it should only require them to be EITHER admin OR owner." + }, + { + "golden_comment": "This calls the email sender with the original guests, so existing attendees included in the input will be treated as new when sending notifications, leading to incorrect emails.", + "severity": "Medium", + "matched_candidate": "Email sender called with original guests list - existing attendees included in the input are treated as new invitees, sending duplicate invitations to guests who are already attending", + "confidence": 0.98, + "reasoning": "Both comments identify the same issue: the email sender is being called with the original guests list, which causes existing attendees to be incorrectly treated as new invitees, resulting in incorrect/duplicate invitation emails being sent to people who are already attending." + }, + { + "golden_comment": "uniqueGuests filters out existing attendees and blacklisted emails but does not deduplicate duplicates within the input; createMany can insert duplicate attendee rows if the client submits repeated emails.", + "severity": "Medium", + "matched_candidate": "uniqueGuests filters out existing attendees and blacklisted emails but does not deduplicate within the input itself - createMany can insert duplicate guest records if the same email appears multiple times", + "confidence": 0.99, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: uniqueGuests filters out existing attendees and blacklisted emails but fails to deduplicate duplicates within the input array itself, which can cause createMany to insert duplicate attendee/guest records when the client submits repeated emails." + }, + { + "golden_comment": "Starting with an array containing an empty string may cause validation issues. Consider starting with an empty array [] and handling the empty state in the MultiEmail component instead.", + "severity": "Low", + "matched_candidate": "Incorrect validation condition checks if array length is 0, but initial state contains an empty string [\"\"], so length will be 1, not 0", + "confidence": 0.95, + "reasoning": "Both comments identify the same underlying issue: the initial state contains an empty string [\"\"] instead of an empty array [], which causes problems with validation. The golden comment suggests starting with an empty array to avoid validation issues, while the candidate specifically explains how this causes the validation condition to fail (checking length === 0 when length is actually 1). They're pointing to the same bug from slightly different angles." + } + ], + "false_positives": [], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 5, + "tp": 5, + "fp": 0, + "fn": 0, + "errors_count": 0, + "precision": 1.0, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/14740" } }, "https://github.com/calcom/cal.com/pull/10600": { @@ -82597,6 +84539,58 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR10600__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR10600__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Error message mentions 'backup code login' but this is a disable endpoint, not login", + "severity": "Low", + "matched_candidate": "Error message says 'backup code login' but this endpoint disables 2FA - the message incorrectly describes the action, confusing users about what operation is being performed", + "confidence": 0.98, + "reasoning": "Both comments identify the exact same issue: the error message incorrectly mentions 'backup code login' when the endpoint is actually for disabling 2FA, not for login. The candidate issue provides slightly more detail about why this is problematic (confusing users), but it's pointing to the same underlying bug." + }, + { + "golden_comment": "Backup code validation is case-sensitive due to the use of indexOf(). This causes validation to fail if a user enters uppercase hex characters, as backup codes should be case-insensitive for a better user experience.", + "severity": "Medium", + "matched_candidate": "Backup code validation is case-sensitive due to indexOf() - users entering uppercase hex characters will fail validation even though backup codes should be case-insensitive", + "confidence": 1.0, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: backup code validation is case-sensitive because indexOf() is used, which causes validation to fail when users enter uppercase hex characters. Both comments point out that backup codes should be case-insensitive for better user experience." + }, + { + "golden_comment": "Because backupCodes are decrypted and mutated in memory before being written back, two concurrent login requests using the same backupCode could both pass this check and update, so a single backup code may effectively be accepted more than once if used concurrently, weakening the intended one-time-use semantics.", + "severity": "High", + "matched_candidate": "Race condition in backup code validation: backup codes are decrypted and mutated in memory before being written back - two concurrent login requests can both read the same unused code and both succeed, allowing code reuse", + "confidence": 0.99, + "reasoning": "The candidate issue identifies exactly the same race condition problem as the golden comment. Both describe: 1) backup codes being decrypted and mutated in memory before being written back, 2) two concurrent login requests being able to pass the check simultaneously, and 3) the result being that a single backup code can be used more than once, violating one-time-use semantics. The wording is nearly identical and addresses the same underlying concurrency vulnerability." + } + ], + "false_positives": [ + { + "candidate": "Race condition: isDisabling state is not reset if the API call fails or throws an exception" + }, + { + "candidate": "Race condition: clicking download button and immediately awaiting the download event without ensuring the UI is ready for interaction" + } + ], + "false_negatives": [ + { + "golden_comment": "The exported function TwoFactor handles backup codes and is in BackupCode.tsx. Inconsistent naming.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 4, + "tp": 3, + "fp": 2, + "fn": 1, + "errors_count": 0, + "precision": 0.6, + "recall": 0.75, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/10600" } }, "https://github.com/calcom/cal.com/pull/10967": { @@ -85029,6 +87023,62 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR10967__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR10967__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Potential null reference if mainHostDestinationCalendar is undefined if evt.destinationCalendar is null or an empty array ", + "severity": "High", + "matched_candidate": "Array destructuring assumes evt.destinationCalendar has at least one element, but fallback array is empty which makes mainHostDestinationCalendar undefined", + "confidence": 0.95, + "reasoning": "Both comments identify the same issue: the array destructuring of evt.destinationCalendar could result in mainHostDestinationCalendar being undefined. The golden comment mentions this happens when evt.destinationCalendar is null or empty array, and the candidate comment specifically notes that the fallback array is empty which makes mainHostDestinationCalendar undefined. They are pointing to the exact same potential null/undefined reference problem." + }, + { + "golden_comment": "Logic inversion in organization creation: The slug property is now conditionally set when IS_TEAM_BILLING_ENABLED is true, instead of when it's false as originally intended. This change, combined with requestedSlug still being set when IS_TEAM_BILLING_ENABLED is true, results in both properties being set when billing is enabled, and neither when disabled", + "severity": "Medium", + "matched_candidate": "Logic inversion in organization creation: slug property is conditionally set when IS_TEAM_BILLING_ENABLED is true instead of false - organizations created without billing enabled will be missing the slug field", + "confidence": 0.9, + "reasoning": "The candidate issue identifies the same core problem as the golden comment: the logic inversion where slug is conditionally set when IS_TEAM_BILLING_ENABLED is true instead of false. Both comments point out that this results in organizations created without billing enabled missing the slug field. The candidate doesn't mention the additional detail about 'both properties being set when billing is enabled' but it correctly identifies the fundamental logic inversion bug." + }, + { + "golden_comment": "The Calendar interface now requires createEvent(event, credentialId), but some implementations (e.g., Lark/Office365) still declare createEvent(event) only\u2014this breaks the interface contract (also applies to other locations in the PR).", + "severity": "Low", + "matched_candidate": "Calendar interface now requires createEvent(event, credentialId) but implementations like Lark and Office365 still declare createEvent(event) without credentialId - TypeScript interface mismatch causes runtime errors", + "confidence": 0.98, + "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the Calendar interface requires createEvent(event, credentialId) but implementations like Lark and Office365 still declare createEvent(event) without the credentialId parameter, breaking the interface contract. Both comments point to the same TypeScript interface mismatch issue." + } + ], + "false_positives": [ + { + "candidate": "String.split() always returns an array, so the condition 'Array.isArray(calendar)' will always be true when mainHostDestinationCalendar exists, making the else branch unreachable" + }, + { + "candidate": "Variable 'calendar' is assigned a string literal 'calendar' when mainHostDestinationCalendar is falsy, but then checked with Array.isArray(), creating a type inconsistency" + } + ], + "false_negatives": [ + { + "golden_comment": "The optional chaining on mainHostDestinationCalendar?.integration is redundant since you already check mainHostDestinationCalendar in the ternary condition.", + "severity": "Low" + }, + { + "golden_comment": "Logic error: when externalCalendarId is provided, you're searching for a calendar where externalId === externalCalendarId, but this will always fail since you're looking for a calendar that matches itself. Should likely find by credentialId or use different logic.", + "severity": "High" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 5, + "tp": 3, + "fp": 2, + "fn": 2, + "errors_count": 0, + "precision": 0.6, + "recall": 0.6, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/10967" } }, "https://github.com/calcom/cal.com/pull/8087": { @@ -86638,6 +88688,44 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR8087__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR8087__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The code uses forEach with async callbacks, which causes asynchronous operations (e.g., calendar/video event deletions, payment refunds) to run concurrently without being awaited. This 'fire-and-forget' behavior leads to unhandled promise rejections, race conditions, and incomplete cleanup, as surrounding try-catch blocks cannot properly handle errors from these unawaited promises. Replace forEach with for...of loops or Promise.all() with map() to ensure proper sequential execution and error handling.", + "severity": "Critical", + "matched_candidate": "Critical: forEach with async callbacks causes fire-and-forget execution - calendar and video event deletions run concurrently without coordination, errors are silently swallowed, and the function may return before operations complete", + "confidence": 0.98, + "reasoning": "The candidate issue identifies the exact same problem as the golden comment: using forEach with async callbacks causes fire-and-forget execution where calendar and video event deletions run concurrently without being awaited, errors are silently swallowed/not properly handled, and the function may return before operations complete. Both comments point to the same underlying bug with the same consequences." + } + ], + "false_positives": [ + { + "candidate": "reduce function can concatenate undefined values into the array when getAvailability() returns undefined, creating an array with undefined elements" + }, + { + "candidate": "Template literal displays [object Object] instead of meaningful app identifier when paymentApp is an object" + } + ], + "false_negatives": [ + { + "golden_comment": "Consider adding try-catch around the await to handle import failures gracefully", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 1, + "fp": 2, + "fn": 1, + "errors_count": 0, + "precision": 0.2, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/8087" } } } \ No newline at end of file diff --git a/offline/results/anthropic_claude-sonnet-4-5-20250929/candidates.json b/offline/results/anthropic_claude-sonnet-4-5-20250929/candidates.json index e3106da..0e0c218 100644 --- a/offline/results/anthropic_claude-sonnet-4-5-20250929/candidates.json +++ b/offline/results/anthropic_claude-sonnet-4-5-20250929/candidates.json @@ -1163,6 +1163,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Translation file messages_lt.properties contains Italian text instead of Lithuanian - the locale file has wrong language content, causing Lithuanian users to see Italian interface text", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Simplified Chinese locale file zh_CN uses Traditional Chinese terms in totpStep1 - incorrect character set for the locale, showing Traditional Chinese to Simplified Chinese users", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Method named 'santizeAnchors' should be 'sanitizeAnchors' - missing 'i' in the method name is a typo that may prevent callers from finding the method and cause runtime errors", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Regular expression in replaceAll uses unescaped backslashes that will be interpreted as escape sequences, causing incorrect pattern matching", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/keycloak/keycloak/pull/37634": { @@ -2250,6 +2276,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Wrong parameter in null check: method checks grantType for null instead of rawTokenId, causing the null guard to be ineffective and allowing NullPointerException when rawTokenId is null", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "In isAccessTokenId method, the substring check and equality check for grant shortcut appear inverted - the logic always returns false for valid access tokens", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Catching generic RuntimeException is too broad - the implementation throws IllegalArgumentException specifically; catching only RuntimeException hides the specific exception type and makes error handling less precise", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Variable 'accessToken' is used but 'token' is the parameter name in scope - appears to be using an undefined variable", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Map inversion creates duplicate values when multiple keys map to the same value, with 'UNKNOWN' constant creating circular mapping that overwrites other entries", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/keycloak/keycloak/pull/38446": { @@ -3163,6 +3221,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Optional.get() called directly on Optional returned by RecoveryAuthnCodesUtils.getCredential() without calling isPresent() first - will throw NoSuchElementException when credential is absent", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Unsafe cast from AuthenticatorFactory to ConfigurableAuthenticatorFactory without type checking", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Unsafe cast from FormActionFactory to ConfigurableAuthenticatorFactory without type checking", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Null pointer exception vulnerability - calling .get() on Optional without checking if present, could crash authentication flow when no recovery credential exists", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Type confusion vulnerability in getConfigurableAuthenticatorFactory method - unsafe casting of different factory types without proper validation could lead to ClassCastException or bypass of authentication checks", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/keycloak/keycloak/pull/36882": { @@ -3764,6 +3854,14 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Using picocli.exit() instead of CommandLine.ExitCode or proper exit mechanism - calls static method on picocli class rather than intended exit functionality", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/keycloak/keycloak/pull/36880": { @@ -4648,6 +4746,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Feature flag inconsistency causes orphaned permissions: AdminPermissions event listener cleans up permissions when feature is disabled, but another code path creates permissions without checking the flag, leaving orphaned permissions", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "hasPermission uses findByName with wrong arguments - wrong resource name convention causes resource lookup to fail, making permission checks incorrectly return false for valid permissions", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Feature flag inconsistency causes orphaned permissions: AdminPermissions event listener cleans up permissions when feature is disabled but another code path creates permissions without checking the flag", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "In getClientsWithPermission, iterating CLIENTS_RESOURCE_TYPE resources and returning resource IDs returns the wrong set - should return client IDs associated with those resources, not the resource IDs themselves", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Race condition in lazy initialization where multiple threads can simultaneously check 'clientPermissions != null' and both create new instances, causing the first instance to be lost", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/keycloak/keycloak/pull/37038": { @@ -5707,6 +5837,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Incorrect permission check in canManage() method - the method checks the wrong permission scope, allowing users who should not have manage access to pass the authorization check", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "In getGroupIdsWithViewPermission, hasPermission is called with groupResource.getId() for both the resource and scope arguments - the scope parameter should be different from the resource ID, causing incorrect permission evaluation", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "In getGroupIdsWithViewPermission, hasPermission is called with groupResource.getId() for both resource and scope parameters - scope should differ from resource ID, causing incorrect permission evaluation and potentially granting wrong access", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Authorization bypass in getGroupIdsWithViewPermission() - returns empty set when user has view permission, which may grant access to all groups instead of restricting access", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Incomplete constant name 'VIEW_MEMBE' in the GROUPS ResourceType definition - should be 'VIEW_MEMBERS'", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/keycloak/keycloak/pull/33832": { @@ -6754,6 +6916,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Method returns wrong provider - returns default keystore provider instead of BouncyCastle provider, causing callers to use incorrect cryptographic implementation", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Dead code: ASN1Encoder instances are created and written to but their results are immediately discarded - the actual encoding is performed elsewhere, making these encoder operations unnecessary and confusing", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "KeyPairGenerator.getInstance(\"EC\").genKeyPair() creates a key pair with default parameters, but ECDSA algorithms require specific curve parameters (P-256, P-384, P-521) that may not match the default curve", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "String concatenation creates trailing comma and space that are never removed, resulting in malformed output", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Method appears to call itself recursively instead of delegating to a wrapped or parent object, which can cause infinite recursion and StackOverflowError", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/keycloak/keycloak/pull/40940": { @@ -7624,6 +7818,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Returning null from getSubGroupsCount() violates the GroupModel contract - Javadoc declares the method never returns null but this implementation returns null, causing NullPointerException in callers that invoke methods on the result", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Reader thread is not awaited before assertion - flipping deletedAll to true and immediately asserting races with the reader thread adding exceptions, causing flaky test failures", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Method appears to call itself recursively instead of delegating to a wrapped or parent object, causing potential infinite recursion and StackOverflowError", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Response.Status.Family.SUCCESSFUL includes status codes 200-299, but group creation should specifically check for 201 CREATED status code", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Response object is used in try-with-resources but the groupUuid extraction happens inside the try block where response might be closed", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1": { @@ -8450,6 +8676,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "ConditionalPasskeysEnabled() is called without the required UserModel parameter - the method signature requires a user argument but is invoked without one, causing a compilation error or NullPointerException", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "With isConditionalPasskeysEnabled(UserModel user) requiring user != null, authenticate() will not call webauthnAuth.fillContextForm(context) on the correct branch when user is null, causing passkey authentication to silently fail", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "WebAuthn authenticators are exposed based solely on user existence without proper authentication state validation, potentially allowing enumeration of valid usernames and their associated WebAuthn credentials", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Authentication bypass vulnerability - the authenticate method calls context.success() without verifying user credentials when user is already set and no linked brokers exist", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Boolean.parseBoolean() returns false for null input, but the code should likely treat null as false explicitly or handle it differently since AUTH_NOTE values can be null", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/getsentry/sentry/pull/93824": { @@ -9390,6 +9648,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Breaking out of the loop when deadline is elapsed skips terminating remaining flusher processes, potentially leaking process handles and causing resource exhaustion", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "time.sleep in test_consumer.py won't actually wait because time.sleep was monkeypatched above - test timing assumptions are broken, making the test unreliable", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "flusher processes created via multiprocessing.get_context(spawn).Process are instances of SpawnProcess which on POSIX is not a subclass of multiprocessing.Process - isinstance checks will fail and process management may break", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Inconsistent metric tagging: some calls use tag 'shard' while others use 'shards' - metrics will be split across two different tag values, making aggregation and dashboards incorrect", + "path": null, + "line": null, + "source": "logicbug_added" + } ] }, "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5": { @@ -12087,6 +12371,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "When requests are authenticated with API keys or org auth tokens, organization_context.member is None and accessing .has_global_access() on None will raise AttributeError", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Importing OptimizedCursorPaginator which does not exist in the sentry codebase - this import will fail at runtime with ImportError", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Negative offset passed to queryset slicing will cause runtime error - Python slice indices must be non-negative", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Math.floor/ceil applied to potentially non-numeric values without type checking will raise TypeError", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/97529": { @@ -12885,6 +13195,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Race condition in BuildIndex: multiple goroutines can concurrently build the same index for the same key without synchronization, wasting resources and potentially causing data corruption", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Race condition: s.search.TotalDocs() iterates the cache without synchronization while the event watcher goroutine concurrently modifies it - concurrent map read/write causes panic in Go", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Hard-coded port assignment may cause port conflicts in concurrent test execution", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Delete function checks for ResourceVersion < 0 but the error message says 'update must include the previous version' which is incorrect for a delete operation", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/getsentry/sentry/pull/80168": { @@ -13646,6 +13982,20 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "MetricAlertDetectorHandler inherits from StatefulDetectorHandler but the class body only contains pass, failing to implement required abstract methods - will raise NotImplementedError at runtime", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Method docstring says it returns a list of DetectorEvaluationResult but the implementation now returns a dict keyed by DetectorGroupKey - documentation and return type are inconsistent", + "path": null, + "line": null, + "source": "logicbug_added" + } ] }, "https://github.com/getsentry/sentry/pull/80528": { @@ -14177,6 +14527,20 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Function modifies the config variable to include display values but then returns the original monitor.config instead of the modified config, discarding the changes", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Slice operation `previous_checkins[:failure_issue_threshold]` occurs before reverse operation, taking most recent N check-ins instead of oldest N check-ins, resulting in examining wrong subset for threshold evaluation", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/getsentry/sentry/pull/77754": { @@ -14785,6 +15149,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Mutable default argument in dataclass field - using a mutable object as default value for timestamp field shares state across instances, causing subtle data corruption bugs", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "to_dict() returns a datetime object for the queued field - if this dict is passed as task kwargs to apply_async(), JSON serialization will fail since datetime is not JSON-serializable by default", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Method name typo: test_from_dict_inalid_data is missing a letter and should be test_from_dict_invalid_data - the test may not be discovered by test runners that filter by name pattern", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Method named 'empty_array' but tests an empty dict - test name does not match what is being tested, making the test suite misleading and harder to maintain", + "path": null, + "line": null, + "source": "logicbug_added" + } ] }, "https://github.com/getsentry/sentry/pull/95633": { @@ -15642,6 +16032,14 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "queue.shutdown() with 'immediate=False' parameter may not exist in the standard Python queue.Queue module - calling a non-existent method will raise AttributeError at runtime", + "path": null, + "line": null, + "source": "logicbug_added" + } ] }, "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2": { @@ -17047,6 +17445,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "OptimizedCursorPaginator negative-offset branch slices QuerySet with a negative start index - Django QuerySets do not support negative indexing and will raise AssertionError", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "BasePaginator negative-offset branch slices QuerySet with a negative start index - Django raises AssertionError when negative slice indices are used on QuerySets", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "OptimizedCursorPaginator.get_item_key applies floor and ceil to a datetime key when order_by is -datetime - math.floor and math.ceil raise TypeError when given a datetime object instead of a number", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Span with span_id='b'*16 has parent_span_id=None but is_segment_span=True, while other spans reference 'b'*16 as their parent, creating inconsistent parent-child relationship", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Dictionary comprehension uses span.payload as key, but span.payload is bytes which is not hashable and cannot be used as a Redis hash key", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3": { @@ -18246,6 +18676,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "sample_rate = 0.0 is falsy in Python - the condition if client_sample_rate evaluates to False for 0.0, silently skipping the zero sample rate case", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Python's built-in hash() is non-deterministic across processes due to PYTHONHASHSEED - cache keys built with hash() will not match across workers, causing cache misses in distributed environments", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Upsampling eligibility check passes the outer dataset instead of the scoped_dataset actually used by the query - when both datasets differ, the eligibility check is performed on wrong data and may incorrectly enable or disable upsampling", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "The condition `if client_sample_rate:` will evaluate to False for the valid sample rate of 0.0, preventing zero sample rates from being set", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/103633": { @@ -19119,6 +19575,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Asymmetric cache trust logic in Check operation: cached permission grants are returned immediately without re-verification, but cache misses always go to the source. An expired or revoked permission may continue to grant access until cache expiry", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "NoopCache implementation always returns cache miss, causing authorization checks to bypass caching entirely and potentially leading to performance degradation that could facilitate DoS attacks on authorization services", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "NoopCache.Get method takes a receiver pointer when it should be a value receiver since the struct has no fields", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Cache key mismatch: setting cache with action 'dashboards:read' but checking with verb 'get' which likely maps to a different action", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/getsentry/sentry/pull/67876": { @@ -20057,6 +20539,26 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Potential null reference when github_authenticated_user state is not set - accessing state without checking existence can raise KeyError or AttributeError", + "path": "src/sentry/integrations/github/webhooks.py", + "line": null, + "source": "logicbug_added" + }, + { + "text": "OAuth state uses static pipeline.signature instead of per-request random nonce, allowing CSRF attacks on the OAuth callback flow", + "path": "src/sentry/integrations/github/webhooks.py", + "line": null, + "source": "logicbug_added" + }, + { + "text": "Missing key existence check before accessing integration.metadata[sender][login] - accessing nested dict key 'sender' without guard will raise KeyError when sender is absent", + "path": "src/sentry/integrations/github/webhooks.py", + "line": null, + "source": "logicbug_added" + } ] }, "https://github.com/keycloak/keycloak/pull/32918": { @@ -20681,6 +21183,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Recursive caching call uses session instead of delegate - the cache method calls session's method which re-enters caching, causing infinite recursion or incorrect cache behavior", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Cleanup reference uses incorrect alias - should be 'idp-alias-' + i but uses 'alias', resulting in cleanup targeting wrong identity provider configuration", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Stream.of(values()).map(LoginFilter::getFilter).reduce(Predicate::and).get() will throw NoSuchElementException if values() returns an empty array", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Potential null pointer exception when this.organization is null but this.organization.getId() is called", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/94942": { @@ -21462,6 +21990,26 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Dead code: NewInMemoryDB().RunCommands and db.QueryFramesInto always return \"not implemented\" - these exported methods are unusable; either implement them or remove them to avoid confusion", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Function always returns false regardless of feature flag state due to inverted logic and incorrect control flow", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Missing return statement after error assignment - execution continues after logging error and setting rsp.Error", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/90939": { @@ -21962,6 +22510,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Critical cache corruption: entryPointAssetsCache is unconditionally overwritten with the fetch result even when the fetch returns an error - a failed fetch will replace valid cached data with an error value", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Incomplete double-checked locking for caching: cache is checked without the lock, then locked to set, but no second check after acquiring lock - concurrent goroutines can both see nil and both fetch the assets", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Race condition: after releasing RLock and before acquiring Lock, another goroutine could modify entryPointAssetsCache, causing the cached value 'ret' to become stale or the cache state to change", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Potential null pointer dereference: 'ret' captures entryPointAssetsCache under RLock but could be nil, and is returned without validation when cfg.Env != setting.Dev", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/80329": { @@ -22998,6 +23572,20 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Ticker is never stopped, causing a goroutine leak when the function returns", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Using log.Error() instead of log.Debug() or log.Info() for normal operational logging during cleanup process", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/90045": { @@ -24468,6 +25056,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Bug: recordLegacyDuration is called when storage operation fails but the correct function to call for storage failures is recordStorageDuration - wrong metric is being recorded on error path", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Context created with d.Log instead of the log variable that was initialized with additional context values - log entries will be missing the name, kind, and method fields", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Inconsistency: metrics recorded using name instead of options.Kind - other methods in this file use options.Kind for metric dimensions, causing this method to emit metrics with a different label format", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Variable 'l' is declared as nil and never assigned a value, but is used to initialize legacyStoreMock", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Variable 's' is declared as nil and never assigned a value, but is used to initialize storageMock", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/106778": { @@ -25739,6 +26359,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "RuleActionsButtons is invoked with only promRule argument but SilenceGrafanaRuleDrawer inside requires a Grafana Ruler rule - passing only a Prometheus rule will cause missing prop error or incorrect drawer behavior", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "GrafanaRuleListItem rendered in a list is missing the required React key prop - missing key causes React reconciliation issues when list order changes, leading to incorrect DOM updates", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Redundant type check 'rule && rule.type === PromRuleType.Alerting' after prometheusRuleType.grafana.alertingRule(rule) already verified this condition", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Function calls prometheusRuleType.grafana.rule() but uses result in boolean context without handling potential undefined/null return values", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/107534": { @@ -26332,6 +26978,20 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Expected expression includes regex pattern '20|10' but mock data returns string values that would be sorted as ['1', '10', '2', '20', '3'], making the first two values '1' and '10' instead of the expected '20' and '10'", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Template variables are applied to filtered queries but the original query objects may be mutated, potentially affecting other parts of the application that reference the same query objects", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/79265": { @@ -27549,6 +28209,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Race condition: multiple concurrent requests can simultaneously pass the device count check and create devices beyond the limit - there is no atomic check-and-increment to prevent concurrent overwrites", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Type error: dbSession.Exec(args...) spreads a []interface{} where the first element is the query string, but Exec expects the query as a separate first argument - this will panic at runtime", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Anonymous authentication now fails entirely when ErrDeviceLimitReached - previously device tagging was asynchronous and non-blocking, now it blocks authentication for users over the device limit", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Error from TagDevice is returned directly when it's ErrDeviceLimitReached, but other errors are only logged as warnings and authentication continues, creating inconsistent error handling where some failures block authentication while others silently proceed", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "SQL injection vulnerability in updateDevice function - query string is concatenated with user-controlled parameters and passed to Exec without proper parameter binding", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9": { @@ -28296,6 +28988,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Thread-safety issue with lazy @loaded_locales - multiple threads can simultaneously check and set the instance variable without synchronization, causing duplicate locale loading or data corruption", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Race condition in ensure_loaded! method - @loaded_locales is initialized and checked in separate operations without synchronization", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Object.send bypasses method visibility (private/protected) and can invoke any method including dangerous ones", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Inconsistent locale loading logic - translate method duplicates the loaded check from ensure_loaded! but doesn't use ensure_loaded! method", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/grafana/grafana/pull/76186": { @@ -28994,6 +29712,26 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "ContextualLoggerMiddleware methods (QueryData, CallResource, CheckHealth, CollectMetrics) will panic when passed a nil request - dereferencing nil pointer before nil check causes runtime panic", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "FromContext method creates a new TestLogger instance instead of returning the current instance, losing all accumulated test data", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Type assertion to *log.ConcreteLogger without checking if the returned logger is nil, which could cause a panic when accessing methods on a nil pointer", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10": { @@ -30779,6 +31517,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "NoMethodError in EmbeddableHost before_validation callback - calling a method on a nil object before validation will raise NoMethodError and prevent record from being saved", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Migration uses raw SQL to insert embeddable_hosts rows - any existing values containing http:// or https:// scheme prefixes will be duplicated with different formats, creating data inconsistency", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "EmbeddableHostsController update and destroy methods do not validate record existence - EmbeddableHost.where(id: params[:id]).first returns nil when record not found, calling update or destroy on nil raises NoMethodError", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "record_for_host compares lower(host) = ? but does not normalize the parameter case - mixed-case referer hosts like Example.COM will fail to match even though they are equivalent", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Potential NoMethodError on nil - value may be nil from hash lookup or safe navigation, calling methods on nil will raise exception", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7": { @@ -33373,6 +34143,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "findMembers() is now asynchronous and its result is not awaited - the controller returns before member data is available, causing the response to contain empty or stale member information", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Capping next offset at user_count produces empty page when total equals limit - offset equals total, causing the next page to return 0 results when there should be a final page", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "HTTP method mismatch: remove_member test uses PUT but the remove_member controller action expects DELETE - test will always fail with 404 or 405", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "find_group method can return nil from Group.find_by, then guardian.ensure_can_see!(nil) will be called on a nil object", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Potential NoMethodError on nil - value may be nil from hash lookup or safe navigation, and calling methods on nil will raise an exception", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3": { @@ -34261,6 +35063,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "BlockedEmail.should_block_email? has side effects during a read operation - it updates statistics even when only checking whether an email should be blocked, causing unexpected DB writes on reads", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Password length validation uses incorrect condition that allows nil passwords to pass when @password_required is false", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Race condition in should_block? method - multiple concurrent requests for the same email could create inconsistent match_count values due to read-modify-write operation without proper locking", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Method doesn't handle nil or empty email values, will cause NoMethodError when calling =~ on nil", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Missing assignment operator (=) for action_type attribute in Fabricator definition", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5": { @@ -35975,6 +36809,26 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "include_website_name method is missing the required ? suffix - Rails serializers expect include_ conditional methods to end with ? otherwise the attribute is always serialized", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Logic error in subdomain comparison - when website domain is 'example.com' and current hostname is 'forums.example.com', the test expects 'example.com/user' but the logic should likely treat the subdomain as different from the parent domain", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "URI parsing is called twice on the same website string without caching, and the second call lacks error handling that could cause unhandled exceptions", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4": { @@ -38050,6 +38904,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "postMessage targetOrigin should be the origin (scheme+host+port), not the full referrer URL - using the full URL causes the message to be silently dropped by the browser", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "X-Frame-Options set to ALLOWALL completely disables clickjacking protection - referer header validation can be spoofed, leaving the endpoint vulnerable to iframe-based attacks", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "ERB block closes with \"end if\" which is invalid Ruby ERB syntax - this will raise a SyntaxError at render time; it should be \"end\" to close the if block", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Race condition in Redis throttling logic - another process can set key between setnx and expire calls, causing expire to be lost and key to persist indefinitely", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Transaction block does not handle the case where PostCreator.create fails but doesn't raise an exception", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1": { @@ -39435,6 +40321,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "The downsize method is defined twice - the second definition overrides the first, causing dead code and potentially breaking behavior that depended on the original two-argument signature", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Hardcoding maxSizeKB = 10 * 1024 ignores Discourse.SiteSettings max_type_size_kb - client-side limit can diverge from server-side configuration, allowing uploads that exceed the server limit", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Passing 80% as dimensions to gifsicle fails for animated GIFs - the animated path requires absolute pixel dimensions, not percentages, causing animated thumbnail generation to fail", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Image downsizing loop can run indefinitely if OptimizedImage.downsize fails to reduce file size, creating an infinite loop condition", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "tempfile.size is called without checking if tempfile is nil, which will raise NoMethodError if file download failed", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2": { @@ -40521,6 +41439,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Typo in property name: 'stopNotificiationsText' should be 'stopNotificationsText' (missing 'n' in Notifications) - the property will never be found causing silent undefined access", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Race condition: TopicUser.change is called without checking if the current post number is actually newer than the existing last_emailed_post_number", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Logic assumes TopicUser record exists but find_by can return nil, leading to NoMethodError on nil object", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Potential variable name typo: code appears to reference 'topi' instead of 'topic_id' in MessageBus.publish call", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Route 't/:topic_id/summary' will never be reached because it's defined after 't/:slug/:topic_id/summary' route, and Rails matches routes in order", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/22532": { @@ -41801,6 +42751,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "updateManyByCredentialId called with empty data object {} - Prisma @updatedAt decorator only fires when the record has actual field changes, so the updatedAt timestamp will not be updated on cache invalidation", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "sed command uses macOS-specific syntax with empty string after -i flag, will fail on Linux systems", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Race condition potential - if cache statuses are updated between the getCacheStatusByCredentialIds call and the map creation, the cacheStatusMap may contain stale data", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "findUniqueOrThrow method uses SelectedCalendarWhereInput instead of SelectedCalendarWhereUniqueInput, which will cause a type error since findUniqueOrThrow requires unique identifiers", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "deleteCacheMutation is called with credentialId but the mutation expects a different parameter structure based on the trpc endpoint name 'deleteCache'", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/8330": { @@ -42877,6 +43859,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Comparison using === instead of isSame() for Dayjs objects will always be false since they are different object instances", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Variable i is initialized to -1 but the code is incomplete - this suggests a loop that may have off-by-one errors when incrementing/accessing array elements", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Assertion expects results for plus2DateString but the time range query spans from plus1DateString to plus2DateString, potentially missing or incorrectly filtering results", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "startTime calculation duplicates the same dayjs computation twice instead of reusing the result, causing unnecessary work and potential inconsistency", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "endTime calculation duplicates the same dayjs computation twice instead of reusing the result, causing unnecessary work and potential inconsistency", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/14943": { @@ -43616,6 +44630,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "scheduleSMSReminders.ts deletion logic incorrectly deletes non-SMS workflow reminders (Email, WhatsApp) that share the same workflow - the filter condition is too broad and removes reminders of unintended types", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "retryCount reads a possibly stale value with reminder.retryCount + 1 - concurrent requests can both increment from the same base, losing increments; use atomic increment via Prisma update", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "OR condition deletes SMS reminders with retryCount > 1, conflicting with retry logic that increments retryCount on failures", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Using lte (less than or equal) with current date will delete SMS reminders scheduled for exactly the current time, preventing them from being sent if the handler runs at the exact scheduled moment", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Duplicate retry count increment logic - retryCount is incremented both in the 'else' block for failed SMS and in the catch block for exceptions, potentially double-counting failures", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/22345": { @@ -44347,6 +45393,32 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Logic error: fetching userIdsFromOrg only when teamsFromOrg.length > 0 excludes org-level members for organizations without child teams - org members not in any team will be incorrectly excluded from results", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Logic flaw in condition combining - if both authConditions and filterConditions are falsy, returns NOTHING_CONDITION which will exclude all results", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Constructor accepts InsightsBookingServicePublicOptions but assigns to InsightsBookingServiceOptions type field, causing type mismatch", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Missing await for async cleanup operation - test may complete before cleanup finishes, potentially causing interference with subsequent tests", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/11059": { @@ -46562,6 +47634,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "parseRefreshTokenResponse returns a Zod safeParse result object ({ success, data, error }) not the credential object - callers that persist this directly will store the wrapper object instead of the token data", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "When sync endpoint is used, res is a fetch Response object and res.data does not exist - accessing res?.data will return undefined and token.access_token will be undefined, silently breaking token refresh", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "refreshFunction helper returns a fetch Response object when the sync endpoint path is used, but callers expect a credential object with access_token - res.data does not exist on Response, returning undefined and breaking token refresh silently", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Invalid Zod schema syntax - `z.string().toString()` creates a string literal 'string' as a key, not a dynamic string key pattern", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Incorrect expiry date calculation - adding seconds to milliseconds timestamp", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/7232": { @@ -48050,6 +49154,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "deleteScheduledEmailReminder and deleteScheduledSMSReminder are async functions called without await inside forEach loops - reminder deletions fire-and-forget, errors are silently lost, and cleanup may be incomplete", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "forEach loop calls deleteScheduledEmailReminder and deleteScheduledSMSReminder without awaiting them, causing potential race conditions or silent failures", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Race condition in onChange handler - form.getValues() may not reflect current input value due to asynchronous form updates", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Database delete operation is not awaited but function continues execution, potentially causing race conditions", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "deleteScheduledEmailReminder is called with a third parameter 'true' but deleteScheduledSMSReminder is called without it, suggesting inconsistent API usage", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/14740": { @@ -49504,6 +50640,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Case sensitivity bypass in email blacklist: comparison does not normalize email case, allowing BLOCKED@example.com to bypass a blacklist entry for blocked@example.com", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Email sender called with original guests list - existing attendees included in the input are treated as new invitees, sending duplicate invitations to guests who are already attending", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "uniqueGuests filters out existing attendees and blacklisted emails but does not deduplicate within the input itself - createMany can insert duplicate guest records if the same email appears multiple times", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Incorrect validation condition checks if array length is 0, but initial state contains an empty string [\"\"], so length will be 1, not 0", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Logic error using AND (&&) instead of OR (||) to check if user is team admin OR owner - currently requires user to be BOTH admin AND owner, which is incorrect", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/10600": { @@ -50913,6 +52081,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Race condition in backup code validation: backup codes are decrypted and mutated in memory before being written back - two concurrent login requests can both read the same unused code and both succeed, allowing code reuse", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Backup code validation is case-sensitive due to indexOf() - users entering uppercase hex characters will fail validation even though backup codes should be case-insensitive", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Error message says 'backup code login' but this endpoint disables 2FA - the message incorrectly describes the action, confusing users about what operation is being performed", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Race condition: isDisabling state is not reset if the API call fails or throws an exception", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Race condition: clicking download button and immediately awaiting the download event without ensuring the UI is ready for interaction", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/10967": { @@ -52759,6 +53959,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Logic inversion in organization creation: slug property is conditionally set when IS_TEAM_BILLING_ENABLED is true instead of false - organizations created without billing enabled will be missing the slug field", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Calendar interface now requires createEvent(event, credentialId) but implementations like Lark and Office365 still declare createEvent(event) without credentialId - TypeScript interface mismatch causes runtime errors", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "String.split() always returns an array, so the condition 'Array.isArray(calendar)' will always be true when mainHostDestinationCalendar exists, making the else branch unreachable", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Array destructuring assumes evt.destinationCalendar has at least one element, but fallback array is empty which makes mainHostDestinationCalendar undefined", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Variable 'calendar' is assigned a string literal 'calendar' when mainHostDestinationCalendar is falsy, but then checked with Array.isArray(), creating a type inconsistency", + "path": null, + "line": null, + "source": "extracted" + } ] }, "https://github.com/calcom/cal.com/pull/8087": { @@ -54179,6 +55411,38 @@ "line": null, "source": "extracted" } + ], + "codesheriff": [ + { + "text": "Critical: forEach with async callbacks causes fire-and-forget execution - calendar and video event deletions run concurrently without coordination, errors are silently swallowed, and the function may return before operations complete", + "path": null, + "line": null, + "source": "logicbug_added" + }, + { + "text": "Using async callback in forEach - the async functions will not be awaited and will execute concurrently without proper error handling or completion tracking", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "forEach with async callback creates fire-and-forget promises that are not awaited, potentially causing race conditions and unhandled promise rejections", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "reduce function can concatenate undefined values into the array when getAvailability() returns undefined, creating an array with undefined elements", + "path": null, + "line": null, + "source": "extracted" + }, + { + "text": "Template literal displays [object Object] instead of meaningful app identifier when paymentApp is an object", + "path": null, + "line": null, + "source": "extracted" + } ] } } \ No newline at end of file diff --git a/offline/results/anthropic_claude-sonnet-4-5-20250929/evaluations.json b/offline/results/anthropic_claude-sonnet-4-5-20250929/evaluations.json index 244bef5..5b7485d 100644 --- a/offline/results/anthropic_claude-sonnet-4-5-20250929/evaluations.json +++ b/offline/results/anthropic_claude-sonnet-4-5-20250929/evaluations.json @@ -1963,6 +1963,55 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak__greptile-v4-1__PR37429__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR37429__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The translation is in Italian instead of Lithuanian. This should be translated to Lithuanian to match the file's locale (messages_lt.properties).", + "severity": "Medium", + "matched_candidate": "Translation file messages_lt.properties contains Italian text instead of Lithuanian - the locale file has wrong language content, causing Lithuanian users to see Italian interface text", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: the messages_lt.properties file (Lithuanian locale) contains Italian text instead of Lithuanian translations. The golden comment notes the translation is in Italian when it should be Lithuanian, and the candidate issue describes the same problem - that the Lithuanian locale file has Italian content instead of the correct language. This is clearly the same underlying bug." + }, + { + "golden_comment": "The totpStep1 value uses Traditional Chinese terms in the Simplified Chinese file (zh_CN), which is likely incorrect for this locale. Please verify the locale\u2011appropriate translation.", + "severity": "Medium", + "matched_candidate": "Simplified Chinese locale file zh_CN uses Traditional Chinese terms in totpStep1 - incorrect character set for the locale, showing Traditional Chinese to Simplified Chinese users", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: the zh_CN (Simplified Chinese) locale file contains Traditional Chinese characters in the totpStep1 value, which is incorrect for that locale. The candidate issue is more explicit about the character set mismatch, while the golden comment asks for verification, but both are flagging the same underlying problem of using Traditional Chinese terms in a Simplified Chinese locale file." + }, + { + "golden_comment": "The method name 'santizeAnchors' should be 'sanitizeAnchors' (missing 'i').", + "severity": "Low", + "matched_candidate": "Method named 'santizeAnchors' should be 'sanitizeAnchors' - missing 'i' in the method name is a typo that may prevent callers from finding the method and cause runtime errors", + "confidence": 1.0, + "reasoning": "Both the golden comment and candidate issue identify the exact same typo: the method name 'santizeAnchors' is missing the letter 'i' and should be 'sanitizeAnchors'. The candidate provides additional context about potential consequences, but the core issue being identified is identical." + } + ], + "false_positives": [ + { + "candidate": "Regular expression in replaceAll uses unescaped backslashes that will be interpreted as escape sequences, causing incorrect pattern matching" + } + ], + "false_negatives": [ + { + "golden_comment": "The anchor sanitization logic has a potential issue where it consumes English matcher groups without proper validation. If the translated text has more anchor tags than the English text, this could lead to incorrect validation results.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 4, + "total_golden": 4, + "tp": 3, + "fp": 1, + "fn": 1, + "errors_count": 0, + "precision": 0.75, + "recall": 0.75, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/keycloak/keycloak/pull/37429" } }, "https://github.com/keycloak/keycloak/pull/37634": { @@ -3900,6 +3949,58 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak__greptile-v4-1__PR37634__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR37634__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Wrong parameter in null check (grantType vs. rawTokenId)", + "severity": "Critical", + "matched_candidate": "Wrong parameter in null check: method checks grantType for null instead of rawTokenId, causing the null guard to be ineffective and allowing NullPointerException when rawTokenId is null", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same bug: the null check is performed on the wrong parameter (grantType instead of rawTokenId). The candidate provides more detail about the consequences (ineffective null guard, potential NullPointerException), but the core issue is identical." + }, + { + "golden_comment": "In isAccessTokenId, the substring for the grant shortcut and the equality check look inverted: the grant shortcut occupies indices 4\u20135 (substring(4,6)), and a match should return true (combined with UUID check), not false.", + "severity": "High", + "matched_candidate": "In isAccessTokenId method, the substring check and equality check for grant shortcut appear inverted - the logic always returns false for valid access tokens", + "confidence": 0.95, + "reasoning": "Both comments identify the same bug in the isAccessTokenId method: the logic is inverted such that valid access tokens incorrectly return false. The golden comment provides more technical detail about the substring indices (4-5, substring(4,6)) and mentions the UUID check, while the candidate is more concise, but both are pointing to the exact same logical inversion issue that causes the method to fail for valid tokens." + }, + { + "golden_comment": " Catching generic RuntimeException is too broad. The implementation throws IllegalArgumentException specifically - catch that instead for more precise testing.", + "severity": "Low", + "matched_candidate": "Catching generic RuntimeException is too broad - the implementation throws IllegalArgumentException specifically; catching only RuntimeException hides the specific exception type and makes error handling less precise", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: the test is catching the generic RuntimeException when it should catch the more specific IllegalArgumentException that the implementation actually throws. They both emphasize that catching the generic exception is too broad and that the specific exception type should be caught instead for more precise error handling/testing." + } + ], + "false_positives": [ + { + "candidate": "Variable 'accessToken' is used but 'token' is the parameter name in scope - appears to be using an undefined variable" + }, + { + "candidate": "Map inversion creates duplicate values when multiple keys map to the same value, with 'UNKNOWN' constant creating circular mapping that overwrites other entries" + } + ], + "false_negatives": [ + { + "golden_comment": "Javadoc mentions \"usually like 3-letters shortcut\" but some implementations use 2-letter shortcuts (\"ac\", \"cc\", \"rt\", \"te\", \"pc\", \"ci\", \"ro\"). Consider updating documentation to reflect actual usage pattern.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 4, + "tp": 3, + "fp": 2, + "fn": 1, + "errors_count": 0, + "precision": 0.6, + "recall": 0.75, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/keycloak/keycloak/pull/37634" } }, "https://github.com/keycloak/keycloak/pull/38446": { @@ -5432,6 +5533,47 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak__greptile-v4-1__PR38446__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR38446__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Unsafe raw List deserialization without type safety. Calling Optional.get() directly on the Optional returned by RecoveryAuthnCodesUtils.getCredential(user) without checking isPresent() can lead to a NoSuchElementException if the Optional is empty.", + "severity": "Medium", + "matched_candidate": "Null pointer exception vulnerability - calling .get() on Optional without checking if present, could crash authentication flow when no recovery credential exists", + "confidence": 0.92, + "reasoning": "Both issues identify the same core problem: calling .get() on an Optional without checking isPresent() first, which can throw NoSuchElementException. The golden comment mentions this occurs with RecoveryAuthnCodesUtils.getCredential(user), and the candidate describes the same vulnerability in the authentication flow when no recovery credential exists. While the golden comment also mentions an unrelated issue about 'Unsafe raw List deserialization', the primary shared concern is the unsafe Optional.get() call, making this a match." + } + ], + "false_positives": [ + { + "candidate": "Unsafe cast from AuthenticatorFactory to ConfigurableAuthenticatorFactory without type checking" + }, + { + "candidate": "Unsafe cast from FormActionFactory to ConfigurableAuthenticatorFactory without type checking" + }, + { + "candidate": "Type confusion vulnerability in getConfigurableAuthenticatorFactory method - unsafe casting of different factory types without proper validation could lead to ClassCastException or bypass of authentication checks" + } + ], + "false_negatives": [ + { + "golden_comment": "After creating the RecoveryAuthnCodesCredentialModel, consider setting its id from the stored credential (e.g., myUser.recoveryCodes.getId()); otherwise getId() will be null and downstream removal by id (e.g., removeStoredCredentialById in the authenticator flow) may not work.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 1, + "fp": 3, + "fn": 1, + "errors_count": 0, + "precision": 0.2, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/keycloak/keycloak/pull/38446" } }, "https://github.com/keycloak/keycloak/pull/36882": { @@ -6649,6 +6791,32 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak__greptile-v4-1__PR36882__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR36882__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Incorrect method call for exit codes. The picocli.exit() method calls System.exit() directly, which is problematic:", + "severity": "Medium", + "matched_candidate": "Using picocli.exit() instead of CommandLine.ExitCode or proper exit mechanism - calls static method on picocli class rather than intended exit functionality", + "confidence": 0.92, + "reasoning": "Both comments identify the same core issue: incorrect usage of picocli.exit() method. The golden comment points out that picocli.exit() calls System.exit() directly which is problematic, while the candidate notes it's calling a static method on the picocli class rather than the intended exit functionality. Both are flagging the same problematic method call (picocli.exit()) as being incorrect, though they emphasize slightly different aspects of why it's wrong. The underlying issue being identified is identical - the wrong exit mechanism is being used." + } + ], + "false_positives": [], + "false_negatives": [], + "errors": [], + "total_candidates": 1, + "total_golden": 1, + "tp": 1, + "fp": 0, + "fn": 0, + "errors_count": 0, + "precision": 1.0, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/keycloak/keycloak/pull/36882" } }, "https://github.com/keycloak/keycloak/pull/36880": { @@ -8337,6 +8505,50 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak__greptile-v4-1__PR36880__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR36880__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Inconsistent feature flag bug causing orphaned permissions. The AdminPermissions event listener, responsible for cleaning up permissions upon role, client, or group removal, is incorrectly guarded by the ADMIN_FINE_GRAINED_AUTHZ (V1) feature flag. This is inconsistent with other methods in the class that use ADMIN_FINE_GRAINED_AUTHZ_V2. Consequently, if ADMIN_FINE_GRAINED_AUTHZ_V2 is enabled but V1 is not, the permission cleanup logic will not execute, leading to orphaned permission data. Cleanup should occur regardless of which fine-grained authorization version is enabled.", + "severity": "High", + "matched_candidate": "Feature flag inconsistency causes orphaned permissions: AdminPermissions event listener cleans up permissions when feature is disabled but another code path creates permissions without checking the flag", + "confidence": 0.85, + "reasoning": "Both issues identify the same core problem: the AdminPermissions event listener has a feature flag inconsistency that leads to orphaned permissions. The golden comment specifies that the listener uses ADMIN_FINE_GRAINED_AUTHZ (V1) while other methods use V2, causing cleanup to fail when only V2 is enabled. The candidate describes this as 'cleans up permissions when feature is disabled but another code path creates permissions without checking the flag' - which is essentially the same issue from a slightly different angle. Both identify that the feature flag inconsistency in the cleanup logic causes orphaned permissions. The candidate's description is less precise about which specific flags are involved (V1 vs V2), but it correctly identifies the fundamental problem of feature flag mismatch between creation and cleanup paths." + }, + { + "golden_comment": "In hasPermission(ClientModel client, String scope), the resource lookup uses findByName(server, client.getId(), server.getId()), but AdminPermissionsSchema.getOrCreateResource creates per-client resources with the owner set to resourceServer.getClientId(), so this lookup will never find those resources and will always fall back to the 'all-clients' resource, effectively ignoring client-specific permissions.", + "severity": "High", + "matched_candidate": "hasPermission uses findByName with wrong arguments - wrong resource name convention causes resource lookup to fail, making permission checks incorrectly return false for valid permissions", + "confidence": 0.95, + "reasoning": "Both issues identify the same core problem: the hasPermission method's findByName call uses incorrect arguments/parameters that don't match the resource naming convention used when creating resources. The golden comment specifically mentions the mismatch between using client.getId() in the lookup versus resourceServer.getClientId() in creation, while the candidate describes this as 'wrong resource name convention'. Both conclude this causes the lookup to fail and permissions to be incorrectly evaluated. The candidate's description of 'return false for valid permissions' aligns with the golden's 'fall back to all-clients resource, effectively ignoring client-specific permissions'. These are semantically equivalent descriptions of the same bug." + }, + { + "golden_comment": "In getClientsWithPermission(String scope), iterating resourceStore.findByType(server, AdminPermissionsSchema.CLIENTS_RESOURCE_TYPE) and returning resource.getName() will only ever consider the type-level 'Clients' resource (per-client resources have no type) and return its name, while AvailableRoleMappingResource#getRoleIdsWithPermissions expects actual client IDs to pass to realm.getClientById, which can lead to incorrect behavior or a null client and subsequent failures.", + "severity": "High", + "matched_candidate": "In getClientsWithPermission, iterating CLIENTS_RESOURCE_TYPE resources and returning resource IDs returns the wrong set - should return client IDs associated with those resources, not the resource IDs themselves", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: the getClientsWithPermission method is returning resource IDs/names instead of actual client IDs. The golden comment provides more detail about why this happens (type-level vs per-client resources) and the downstream impact (null client failures in AvailableRoleMappingResource), while the candidate is more concise. However, they both point to the same fundamental bug - the method returns the wrong type of identifier (resource IDs instead of client IDs)." + } + ], + "false_positives": [ + { + "candidate": "Race condition in lazy initialization where multiple threads can simultaneously check 'clientPermissions != null' and both create new instances, causing the first instance to be lost" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 3, + "tp": 3, + "fp": 1, + "fn": 0, + "errors_count": 0, + "precision": 0.6, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/keycloak/keycloak/pull/36880" } }, "https://github.com/keycloak/keycloak/pull/37038": { @@ -9916,6 +10128,50 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak__greptile-v4-1__PR37038__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR37038__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Incorrect permission check in canManage() method", + "severity": "High", + "matched_candidate": "Incorrect permission check in canManage() method - the method checks the wrong permission scope, allowing users who should not have manage access to pass the authorization check", + "confidence": 1.0, + "reasoning": "Both the golden comment and candidate issue identify the same underlying problem: the canManage() method has an incorrect permission check. The candidate provides more detail about the consequences (wrong permission scope, unauthorized access), but the core issue being flagged is identical - the permission check in canManage() is wrong." + } + ], + "false_positives": [ + { + "candidate": "In getGroupIdsWithViewPermission, hasPermission is called with groupResource.getId() for both the resource and scope arguments - the scope parameter should be different from the resource ID, causing incorrect permission evaluation" + }, + { + "candidate": "In getGroupIdsWithViewPermission, hasPermission is called with groupResource.getId() for both resource and scope parameters - scope should differ from resource ID, causing incorrect permission evaluation and potentially granting wrong access" + }, + { + "candidate": "Authorization bypass in getGroupIdsWithViewPermission() - returns empty set when user has view permission, which may grant access to all groups instead of restricting access" + }, + { + "candidate": "Incomplete constant name 'VIEW_MEMBE' in the GROUPS ResourceType definition - should be 'VIEW_MEMBERS'" + } + ], + "false_negatives": [ + { + "golden_comment": "In getGroupIdsWithViewPermission, hasPermission is called with groupResource.getId() and the same groupResource.getId() is added to granted, but hasPermission resolves resources by name (treating the argument as a group id) and the GroupPermissionEvaluator contract says this method returns group IDs that are later used as UserModel.GROUPS and in getUsersCount group filters. This mismatch means per-group VIEW_MEMBERS/MANAGE_MEMBERS permissions may not yield the expected group IDs for filtering and counts, and evaluation may effectively only look at the type-level 'all-groups' resource; consider revisiting whether this should operate on the underlying group ids (resource names) instead so it aligns with the JPA queries and the interface contract.", + "severity": "High" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 1, + "fp": 4, + "fn": 1, + "errors_count": 0, + "precision": 0.2, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/keycloak/keycloak/pull/37038" } }, "https://github.com/keycloak/keycloak/pull/33832": { @@ -11498,6 +11754,49 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak__greptile-v4-1__PR33832__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR33832__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Returns wrong provider (default keystore instead of BouncyCastle)", + "severity": "High", + "matched_candidate": "Method returns wrong provider - returns default keystore provider instead of BouncyCastle provider, causing callers to use incorrect cryptographic implementation", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: the method returns the wrong provider (default keystore instead of BouncyCastle). The candidate provides more detail about the consequences (callers using incorrect cryptographic implementation), but the core problem is identical." + }, + { + "golden_comment": "Dead code exists where ASN1Encoder instances are created and written to, but their results are immediately discarded. The actual encoding is performed by new ASN1Encoder instances created in the subsequent return statement, rendering the earlier operations useless.", + "severity": "Low", + "matched_candidate": "Dead code: ASN1Encoder instances are created and written to but their results are immediately discarded - the actual encoding is performed elsewhere, making these encoder operations unnecessary and confusing", + "confidence": 0.98, + "reasoning": "Both comments identify the same issue: ASN1Encoder instances are created and written to, but their results are discarded because the actual encoding happens in new instances created in the return statement. The candidate uses slightly different wording ('performed elsewhere' vs 'in the subsequent return statement') but describes the identical problem of dead code where encoder operations are unnecessary." + } + ], + "false_positives": [ + { + "candidate": "KeyPairGenerator.getInstance(\"EC\").genKeyPair() creates a key pair with default parameters, but ECDSA algorithms require specific curve parameters (P-256, P-384, P-521) that may not match the default curve" + }, + { + "candidate": "String concatenation creates trailing comma and space that are never removed, resulting in malformed output" + }, + { + "candidate": "Method appears to call itself recursively instead of delegating to a wrapped or parent object, which can cause infinite recursion and StackOverflowError" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 2, + "fp": 3, + "fn": 0, + "errors_count": 0, + "precision": 0.4, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/keycloak/keycloak/pull/33832" } }, "https://github.com/keycloak/keycloak/pull/40940": { @@ -12964,6 +13263,49 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak__greptile-v4-1__PR40940__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR40940__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Returning null from getSubGroupsCount() violates the GroupModel contract (Javadoc says it never returns null) and may lead to NPEs in callers that expect a non-null count.", + "severity": "Critical", + "matched_candidate": "Returning null from getSubGroupsCount() violates the GroupModel contract - Javadoc declares the method never returns null but this implementation returns null, causing NullPointerException in callers that invoke methods on the result", + "confidence": 1.0, + "reasoning": "Both the golden comment and candidate issue identify the exact same problem: the getSubGroupsCount() method returns null, which violates its contract (Javadoc states it never returns null) and can cause NullPointerExceptions in calling code. The candidate issue provides slightly more detail about the mechanism (callers invoking methods on the result), but the core issue is identical." + }, + { + "golden_comment": "The reader thread isn\u2019t waited for; flipping deletedAll to true and asserting immediately can race and miss exceptions added just after the flag change, making this test flaky.", + "severity": "Medium", + "matched_candidate": "Reader thread is not awaited before assertion - flipping deletedAll to true and immediately asserting races with the reader thread adding exceptions, causing flaky test failures", + "confidence": 1.0, + "reasoning": "Both comments identify the identical race condition: the reader thread is not properly synchronized/waited for before setting deletedAll to true and asserting, which creates a race where exceptions added by the reader thread just after the flag flip can be missed, causing flaky test behavior. The wording differs but the underlying issue, root cause, and consequence are the same." + } + ], + "false_positives": [ + { + "candidate": "Method appears to call itself recursively instead of delegating to a wrapped or parent object, causing potential infinite recursion and StackOverflowError" + }, + { + "candidate": "Response.Status.Family.SUCCESSFUL includes status codes 200-299, but group creation should specifically check for 201 CREATED status code" + }, + { + "candidate": "Response object is used in try-with-resources but the groupUuid extraction happens inside the try block where response might be closed" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 2, + "fp": 3, + "fn": 0, + "errors_count": 0, + "precision": 0.4, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/keycloak/keycloak/pull/40940" } }, "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1": { @@ -14435,6 +14777,49 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak-greptile__greptile-v4-1__PR1__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak-greptile__greptile-v4-1__PR1__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "ConditionalPasskeysEnabled() called without UserModel parameter", + "severity": "Medium", + "matched_candidate": "ConditionalPasskeysEnabled() is called without the required UserModel parameter - the method signature requires a user argument but is invoked without one, causing a compilation error or NullPointerException", + "confidence": 1.0, + "reasoning": "Both the golden comment and candidate issue identify the exact same problem: the ConditionalPasskeysEnabled() method is being called without providing the required UserModel parameter. The candidate provides more detail about potential consequences (compilation error or NullPointerException), but the core issue - missing the UserModel parameter in the method call - is identical." + }, + { + "golden_comment": "With isConditionalPasskeysEnabled(UserModel user) requiring user != null, authenticate(...) will not call webauthnAuth.fillContextForm(context) on the initial login page where context.getUser() is still null, so conditional passkey UI will not be set up for first-time passkey login. Consider whether this should also be enabled when no user has been selected yet so ID-less passkey authentication on the initial login form continues to work.", + "severity": "Medium", + "matched_candidate": "With isConditionalPasskeysEnabled(UserModel user) requiring user != null, authenticate() will not call webauthnAuth.fillContextForm(context) on the correct branch when user is null, causing passkey authentication to silently fail", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: the `isConditionalPasskeysEnabled(UserModel user)` method requires a non-null user, which prevents `webauthnAuth.fillContextForm(context)` from being called in `authenticate()` when `context.getUser()` is null (i.e., on the initial login page). The golden comment focuses on the consequence for conditional passkey UI setup and ID-less passkey authentication, while the candidate describes it as 'passkey authentication silently failing'. Despite slightly different emphasis on the consequences, both are pointing to the exact same bug: the null user check prevents proper passkey setup on initial login." + } + ], + "false_positives": [ + { + "candidate": "WebAuthn authenticators are exposed based solely on user existence without proper authentication state validation, potentially allowing enumeration of valid usernames and their associated WebAuthn credentials" + }, + { + "candidate": "Authentication bypass vulnerability - the authenticate method calls context.success() without verifying user credentials when user is already set and no linked brokers exist" + }, + { + "candidate": "Boolean.parseBoolean() returns false for null input, but the code should likely treat null as false explicitly or handle it differently since AUTH_NOTE values can be null" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 2, + "fp": 3, + "fn": 0, + "errors_count": 0, + "precision": 0.4, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1" } }, "https://github.com/getsentry/sentry/pull/93824": { @@ -16494,234 +16879,286 @@ "tool": "greptile-v4-1", "repo_name": "sentry__sentry__greptile-v4-1__PR93824__20260406", "pr_url": "https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR93824__20260406/pull/1" - } - }, - "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5": { - "gemini": { - "skipped": false, - "true_positives": [], - "false_positives": [ - { - "candidate": "Loop variable `id` shadows the built-in `id()` function and should be renamed to something more specific like `event_id`" - } - ], - "false_negatives": [ - { - "golden_comment": "Breaking changes in error response format", - "severity": "Medium" - }, - { - "golden_comment": "Detector validator uses wrong key when updating type", - "severity": "Medium" - }, - { - "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", - "severity": "Low" - } - ], - "errors": [], - "total_candidates": 1, - "total_golden": 3, - "tp": 0, - "fp": 1, - "fn": 3, - "errors_count": 0, - "precision": 0.0, - "recall": 0.0, - "tool": "gemini", - "repo_name": "sentry__sentry-greptile__gemini__PR5__20260128", - "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__gemini__PR5__20260128/pull/1" }, - "greptile": { - "skipped": false, - "true_positives": [], - "false_positives": [ - { - "candidate": "Mutating input parameter by sorting in place may cause unexpected side effects" - }, - { - "candidate": "The validation logic for `age` and `timestamp` fields is incorrect - both validators check `self.initial_data` which runs before individual field validation, causing incorrect validation failures. When DRF validates fields, each validator runs independently, so checking for the other field in `initial_data` won't work as expected. The validators should run at the serializer level using `validate()` method instead." - } - ], - "false_negatives": [ - { - "golden_comment": "Breaking changes in error response format", - "severity": "Medium" - }, - { - "golden_comment": "Detector validator uses wrong key when updating type", - "severity": "Medium" - }, - { - "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", - "severity": "Low" - } - ], - "errors": [], - "total_candidates": 2, - "total_golden": 3, - "tp": 0, - "fp": 2, - "fn": 3, - "errors_count": 0, - "precision": 0.0, - "recall": 0.0, - "tool": "greptile", - "repo_name": "sentry__sentry-greptile__greptile__PR5__20260122", - "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__greptile__PR5__20260122/pull/1" - }, - "claude": { - "skipped": false, - "true_positives": [], - "false_positives": [], - "false_negatives": [ - { - "golden_comment": "Breaking changes in error response format", - "severity": "Medium" - }, - { - "golden_comment": "Detector validator uses wrong key when updating type", - "severity": "Medium" - }, - { - "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", - "severity": "Low" - } - ], - "errors": [], - "total_candidates": 0, - "total_golden": 3, - "tp": 0, - "fp": 0, - "fn": 3, - "errors_count": 0, - "precision": 0.0, - "recall": 0.0, - "tool": "claude", - "repo_name": "sentry__sentry-greptile__claude__PR5__20260127", - "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__claude__PR5__20260127/pull/1" - }, - "qodo": { + "codesheriff": { "skipped": false, "true_positives": [ { - "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", - "severity": "Low", - "matched_candidate": "fetch_error_details zips error_ids with events.values() from nodestore.backend.get_multi(...) where dict iteration order may not match input order and may drop missing keys, causing wrong event payload association with error ids", - "confidence": 0.98, - "reasoning": "Both comments identify the same core issue: the code uses zip(error_ids, events.values()) which incorrectly assumes that the dictionary values from get_multi will be in the same order as the input error_ids list. Both note that dictionary iteration order may not match input order and that missing keys can cause misalignment, resulting in event data being paired with wrong error IDs. The candidate issue is more verbose but describes the identical bug." - } - ], - "false_positives": [ - { - "candidate": "browser_reporting_collector logs request.data via logger.info which can include full URLs, user agents, and potentially other sensitive report contents, creating privacy/PII logging concerns" + "golden_comment": "Inconsistent metric tagging with 'shard' and 'shards'", + "severity": "Medium", + "matched_candidate": "Inconsistent metric tagging: some calls use tag 'shard' while others use 'shards' - metrics will be split across two different tag values, making aggregation and dashboards incorrect", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: inconsistent use of 'shard' vs 'shards' as metric tags. The candidate provides more detail about the consequences (split metrics, aggregation problems, dashboard issues), but the core problem being identified is identical." }, { - "candidate": "Mutual-exclusion validation for age vs timestamp uses self.initial_data.get(...) truthiness checks, which can behave incorrectly for valid falsy values (e.g., age=0) instead of checking key presence" + "golden_comment": "Because flusher processes are created via multiprocessing.get_context('spawn').Process, they are instances of multiprocessing.context.SpawnProcess, which on POSIX is not a subclass of multiprocessing.Process, so this isinstance check will always be false and hung processes won't be killed here.", + "severity": "High", + "matched_candidate": "flusher processes created via multiprocessing.get_context(spawn).Process are instances of SpawnProcess which on POSIX is not a subclass of multiprocessing.Process - isinstance checks will fail and process management may break", + "confidence": 0.95, + "reasoning": "Both comments identify the exact same issue: flusher processes created via multiprocessing.get_context('spawn').Process are instances of SpawnProcess, which on POSIX is not a subclass of multiprocessing.Process, causing isinstance checks to fail. The golden comment specifically mentions that hung processes won't be killed, while the candidate mentions process management may break - these are semantically equivalent consequences of the same root cause." }, { - "candidate": "get_merged_pr_single_issue_template calls PRCommentWorkflow._truncate_title(...) but the truncation helper is defined on CommitContextIntegration, creating an incorrect class reference that may break at runtime" - } - ], - "false_negatives": [ - { - "golden_comment": "Breaking changes in error response format", - "severity": "Medium" + "golden_comment": "Sleep in test_consumer.py won\u2019t actually wait because time.sleep was monkeypatched above; consider restoring sleep or using a different sync to ensure the flusher has time to process.", + "severity": "Medium", + "matched_candidate": "time.sleep in test_consumer.py won't actually wait because time.sleep was monkeypatched above - test timing assumptions are broken, making the test unreliable", + "confidence": 0.95, + "reasoning": "Both comments identify the exact same issue: time.sleep was monkeypatched earlier in the code, so when it's called in test_consumer.py, it won't actually wait/sleep as intended. The golden comment suggests solutions (restoring sleep or using different sync), while the candidate focuses on the consequence (broken test timing/unreliability), but they're pointing to the same underlying problem." }, { - "golden_comment": "Detector validator uses wrong key when updating type", - "severity": "Medium" + "golden_comment": "Breaking out of the loop when the deadline has elapsed can skip terminating remaining flusher processes, potentially leaving them running after shutdown; consider ensuring termination is attempted even if the deadline is exceeded.", + "severity": "Medium", + "matched_candidate": "Breaking out of the loop when deadline is elapsed skips terminating remaining flusher processes, potentially leaking process handles and causing resource exhaustion", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: breaking out of the loop when the deadline elapses causes remaining flusher processes to not be terminated. The golden comment focuses on processes 'potentially leaving them running after shutdown' while the candidate mentions 'leaking process handles and causing resource exhaustion.' These are semantically equivalent - unterminated processes will continue running and leak resources. Both identify the same bug and recommend ensuring termination is attempted for all processes." } ], - "errors": [], - "total_candidates": 4, - "total_golden": 3, - "tp": 1, - "fp": 3, - "fn": 2, - "errors_count": 0, - "precision": 0.25, - "recall": 0.3333333333333333, - "tool": "qodo", - "repo_name": "sentry__sentry-greptile__qodo__PR5__20260123", - "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__qodo__PR5__20260123/pull/1" - }, - "graphite": { - "skipped": false, - "true_positives": [], "false_positives": [], "false_negatives": [ { - "golden_comment": "Breaking changes in error response format", - "severity": "Medium" - }, - { - "golden_comment": "Detector validator uses wrong key when updating type", - "severity": "Medium" - }, - { - "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", + "golden_comment": "Fixed sleep in tests can be flaky; wait on condition instead", "severity": "Low" } ], "errors": [], - "total_candidates": 0, - "total_golden": 3, - "tp": 0, + "total_candidates": 4, + "total_golden": 5, + "tp": 4, "fp": 0, - "fn": 3, - "errors_count": 0, - "precision": 0.0, - "recall": 0.0, - "tool": "graphite", - "repo_name": "sentry__sentry-greptile__graphite__PR5__20260123", - "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__graphite__PR5__20260123/pull/1" - }, - "bugbot": { - "skipped": false, - "true_positives": [ - { - "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", - "severity": "Low", - "matched_candidate": "fetch_error_details function uses zip(error_ids, events.values()) to pair error IDs with event data, but nodestore.backend.get_multi(node_ids) returns a dict keyed by node_id (not error_id), and dict values are not guaranteed to be in the same order as the input list", - "confidence": 0.95, - "reasoning": "Both comments identify the same core issue: the code incorrectly uses zip(error_ids, events.values()) which assumes the dictionary values will be in the same order as error_ids, but since get_multi returns a dict, there's no guarantee the values will align with the input order. Both mention that this can cause event data to be paired with wrong IDs. The candidate provides slightly more technical detail about the dict being keyed by node_id vs error_id, but fundamentally describes the same bug." - } - ], - "false_positives": [ - { - "candidate": "TableWidgetVisualization receives hardcoded empty data (columns=[], tableData={data: [], meta: {fields: {}, units: {}}}) when use-table-widget-visualization feature flag is enabled, instead of actual results from result.data and result.meta" - } - ], - "false_negatives": [ - { - "golden_comment": "Breaking changes in error response format", - "severity": "Medium" - }, - { - "golden_comment": "Detector validator uses wrong key when updating type", - "severity": "Medium" - } - ], - "errors": [], - "total_candidates": 3, - "total_golden": 3, - "tp": 1, - "fp": 1, - "fn": 2, + "fn": 1, "errors_count": 0, - "precision": 0.3333333333333333, - "recall": 0.3333333333333333, - "tool": "bugbot", - "repo_name": "sentry__sentry-greptile__bugbot__PR5__20260122", - "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__bugbot__PR5__20260122/pull/1" - }, - "copilot": { + "precision": 1.0, + "recall": 0.8, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/getsentry/sentry/pull/93824" + } + }, + "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5": { + "gemini": { "skipped": false, "true_positives": [], "false_positives": [ { - "candidate": "TableWidgetVisualization renders with empty data and columns when feature flag is enabled instead of passing actual result.data, result.meta, and derived columns from fields and fieldAliases" + "candidate": "Loop variable `id` shadows the built-in `id()` function and should be renamed to something more specific like `event_id`" + } + ], + "false_negatives": [ + { + "golden_comment": "Breaking changes in error response format", + "severity": "Medium" + }, + { + "golden_comment": "Detector validator uses wrong key when updating type", + "severity": "Medium" + }, + { + "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 1, + "total_golden": 3, + "tp": 0, + "fp": 1, + "fn": 3, + "errors_count": 0, + "precision": 0.0, + "recall": 0.0, + "tool": "gemini", + "repo_name": "sentry__sentry-greptile__gemini__PR5__20260128", + "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__gemini__PR5__20260128/pull/1" + }, + "greptile": { + "skipped": false, + "true_positives": [], + "false_positives": [ + { + "candidate": "Mutating input parameter by sorting in place may cause unexpected side effects" + }, + { + "candidate": "The validation logic for `age` and `timestamp` fields is incorrect - both validators check `self.initial_data` which runs before individual field validation, causing incorrect validation failures. When DRF validates fields, each validator runs independently, so checking for the other field in `initial_data` won't work as expected. The validators should run at the serializer level using `validate()` method instead." + } + ], + "false_negatives": [ + { + "golden_comment": "Breaking changes in error response format", + "severity": "Medium" + }, + { + "golden_comment": "Detector validator uses wrong key when updating type", + "severity": "Medium" + }, + { + "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 2, + "total_golden": 3, + "tp": 0, + "fp": 2, + "fn": 3, + "errors_count": 0, + "precision": 0.0, + "recall": 0.0, + "tool": "greptile", + "repo_name": "sentry__sentry-greptile__greptile__PR5__20260122", + "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__greptile__PR5__20260122/pull/1" + }, + "claude": { + "skipped": false, + "true_positives": [], + "false_positives": [], + "false_negatives": [ + { + "golden_comment": "Breaking changes in error response format", + "severity": "Medium" + }, + { + "golden_comment": "Detector validator uses wrong key when updating type", + "severity": "Medium" + }, + { + "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 0, + "total_golden": 3, + "tp": 0, + "fp": 0, + "fn": 3, + "errors_count": 0, + "precision": 0.0, + "recall": 0.0, + "tool": "claude", + "repo_name": "sentry__sentry-greptile__claude__PR5__20260127", + "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__claude__PR5__20260127/pull/1" + }, + "qodo": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", + "severity": "Low", + "matched_candidate": "fetch_error_details zips error_ids with events.values() from nodestore.backend.get_multi(...) where dict iteration order may not match input order and may drop missing keys, causing wrong event payload association with error ids", + "confidence": 0.98, + "reasoning": "Both comments identify the same core issue: the code uses zip(error_ids, events.values()) which incorrectly assumes that the dictionary values from get_multi will be in the same order as the input error_ids list. Both note that dictionary iteration order may not match input order and that missing keys can cause misalignment, resulting in event data being paired with wrong error IDs. The candidate issue is more verbose but describes the identical bug." + } + ], + "false_positives": [ + { + "candidate": "browser_reporting_collector logs request.data via logger.info which can include full URLs, user agents, and potentially other sensitive report contents, creating privacy/PII logging concerns" + }, + { + "candidate": "Mutual-exclusion validation for age vs timestamp uses self.initial_data.get(...) truthiness checks, which can behave incorrectly for valid falsy values (e.g., age=0) instead of checking key presence" + }, + { + "candidate": "get_merged_pr_single_issue_template calls PRCommentWorkflow._truncate_title(...) but the truncation helper is defined on CommitContextIntegration, creating an incorrect class reference that may break at runtime" + } + ], + "false_negatives": [ + { + "golden_comment": "Breaking changes in error response format", + "severity": "Medium" + }, + { + "golden_comment": "Detector validator uses wrong key when updating type", + "severity": "Medium" + } + ], + "errors": [], + "total_candidates": 4, + "total_golden": 3, + "tp": 1, + "fp": 3, + "fn": 2, + "errors_count": 0, + "precision": 0.25, + "recall": 0.3333333333333333, + "tool": "qodo", + "repo_name": "sentry__sentry-greptile__qodo__PR5__20260123", + "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__qodo__PR5__20260123/pull/1" + }, + "graphite": { + "skipped": false, + "true_positives": [], + "false_positives": [], + "false_negatives": [ + { + "golden_comment": "Breaking changes in error response format", + "severity": "Medium" + }, + { + "golden_comment": "Detector validator uses wrong key when updating type", + "severity": "Medium" + }, + { + "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 0, + "total_golden": 3, + "tp": 0, + "fp": 0, + "fn": 3, + "errors_count": 0, + "precision": 0.0, + "recall": 0.0, + "tool": "graphite", + "repo_name": "sentry__sentry-greptile__graphite__PR5__20260123", + "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__graphite__PR5__20260123/pull/1" + }, + "bugbot": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).", + "severity": "Low", + "matched_candidate": "fetch_error_details function uses zip(error_ids, events.values()) to pair error IDs with event data, but nodestore.backend.get_multi(node_ids) returns a dict keyed by node_id (not error_id), and dict values are not guaranteed to be in the same order as the input list", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: the code incorrectly uses zip(error_ids, events.values()) which assumes the dictionary values will be in the same order as error_ids, but since get_multi returns a dict, there's no guarantee the values will align with the input order. Both mention that this can cause event data to be paired with wrong IDs. The candidate provides slightly more technical detail about the dict being keyed by node_id vs error_id, but fundamentally describes the same bug." + } + ], + "false_positives": [ + { + "candidate": "TableWidgetVisualization receives hardcoded empty data (columns=[], tableData={data: [], meta: {fields: {}, units: {}}}) when use-table-widget-visualization feature flag is enabled, instead of actual results from result.data and result.meta" + } + ], + "false_negatives": [ + { + "golden_comment": "Breaking changes in error response format", + "severity": "Medium" + }, + { + "golden_comment": "Detector validator uses wrong key when updating type", + "severity": "Medium" + } + ], + "errors": [], + "total_candidates": 3, + "total_golden": 3, + "tp": 1, + "fp": 1, + "fn": 2, + "errors_count": 0, + "precision": 0.3333333333333333, + "recall": 0.3333333333333333, + "tool": "bugbot", + "repo_name": "sentry__sentry-greptile__bugbot__PR5__20260122", + "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__bugbot__PR5__20260122/pull/1" + }, + "copilot": { + "skipped": false, + "true_positives": [], + "false_positives": [ + { + "candidate": "TableWidgetVisualization renders with empty data and columns when feature flag is enabled instead of passing actual result.data, result.meta, and derived columns from fields and fieldAliases" } ], "false_negatives": [ @@ -20448,6 +20885,53 @@ "tool": "greptile-v4-1", "repo_name": "sentry__sentry-greptile__greptile-v4-1__PR1__20260406", "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__greptile-v4-1__PR1__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Importing non-existent OptimizedCursorPaginator", + "severity": "Low", + "matched_candidate": "Importing OptimizedCursorPaginator which does not exist in the sentry codebase - this import will fail at runtime with ImportError", + "confidence": 1.0, + "reasoning": "Both comments identify the same issue: the code is importing 'OptimizedCursorPaginator' which does not exist in the codebase. The golden comment states it's 'non-existent' while the candidate provides more detail about the runtime ImportError consequence, but they are describing the exact same underlying problem." + }, + { + "golden_comment": "Django querysets do not support negative slicing", + "severity": "High", + "matched_candidate": "Negative offset passed to queryset slicing will cause runtime error - Python slice indices must be non-negative", + "confidence": 0.95, + "reasoning": "Both comments identify the same underlying issue: attempting to use negative indices/offsets when slicing Django querysets, which is not supported and will cause a runtime error. The golden comment states querysets don't support negative slicing, while the candidate provides more detail about the consequence (runtime error with non-negative requirement), but they refer to the same problem." + }, + { + "golden_comment": "When requests are authenticated with API keys or org auth tokens (which have user_id=None), organization_context.member is None. Line 71 attempts to access organization_context.member.has_global_access without checking if member is None, causing an AttributeError crash when optimized_pagination=true is used, even though the request passed all permission checks with valid org:write scope.", + "severity": "High", + "matched_candidate": "When requests are authenticated with API keys or org auth tokens, organization_context.member is None and accessing .has_global_access() on None will raise AttributeError", + "confidence": 0.95, + "reasoning": "Both comments identify the exact same issue: when organization_context.member is None (due to API key or org auth token authentication), attempting to access has_global_access causes an AttributeError. The golden comment provides more context about the specific line (71) and the optimized_pagination flag, but the core bug being identified is identical - a None check is missing before accessing the member attribute." + }, + { + "golden_comment": "get_item_key assumes a numeric key, but the paginator is used with order_by=-datetime in the audit logs endpoint; calling math.floor/ceil on a datetime will raise a TypeError.", + "severity": "High", + "matched_candidate": "Math.floor/ceil applied to potentially non-numeric values without type checking will raise TypeError", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: math.floor/ceil operations are being applied to values that may not be numeric (specifically datetime objects), which will cause a TypeError. The golden comment provides more specific context about the paginator and audit logs endpoint, while the candidate is more general, but they both point to the exact same bug - attempting mathematical operations on non-numeric types without proper type checking." + } + ], + "false_positives": [], + "false_negatives": [], + "errors": [], + "total_candidates": 4, + "total_golden": 4, + "tp": 4, + "fp": 0, + "fn": 0, + "errors_count": 0, + "precision": 1.0, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1" } }, "https://github.com/grafana/grafana/pull/97529": { @@ -21906,6 +22390,46 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR97529__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR97529__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "A race condition in BuildIndex allows multiple goroutines to concurrently build the same expensive index for the same key. This is caused by moving the b.cacheMu lock from protecting the entire function to only protecting the final cache assignment. ", + "severity": "High", + "matched_candidate": "Race condition in BuildIndex: multiple goroutines can concurrently build the same index for the same key without synchronization, wasting resources and potentially causing data corruption", + "confidence": 0.95, + "reasoning": "Both comments identify the same race condition in BuildIndex where multiple goroutines can concurrently build the same index for the same key. The golden comment provides more specific detail about the cause (moving the cacheMu lock), while the candidate mentions potential consequences (resource waste and data corruption). However, they both describe the same core synchronization issue." + }, + { + "golden_comment": "Calling s.search.TotalDocs() here may race with concurrent index creation: TotalDocs iterates b.cache without synchronization, and the event watcher goroutine started just above could trigger BuildIndex writes concurrently, potentially causing a concurrent map read/write panic.", + "severity": "High", + "matched_candidate": "Race condition: s.search.TotalDocs() iterates the cache without synchronization while the event watcher goroutine concurrently modifies it - concurrent map read/write causes panic in Go", + "confidence": 1.0, + "reasoning": "Both comments identify the same race condition issue: s.search.TotalDocs() accesses the cache without synchronization while the event watcher goroutine can concurrently modify it through BuildIndex, leading to a potential concurrent map read/write panic. The candidate issue captures all key elements: the unsynchronized cache iteration, the concurrent modification by the event watcher goroutine, and the resulting panic." + } + ], + "false_positives": [ + { + "candidate": "Hard-coded port assignment may cause port conflicts in concurrent test execution" + }, + { + "candidate": "Delete function checks for ResourceVersion < 0 but the error message says 'update must include the previous version' which is incorrect for a delete operation" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 4, + "total_golden": 2, + "tp": 2, + "fp": 2, + "fn": 0, + "errors_count": 0, + "precision": 0.5, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/97529" } }, "https://github.com/getsentry/sentry/pull/80168": { @@ -23356,6 +23880,39 @@ "tool": "greptile-v4-1", "repo_name": "sentry__sentry__greptile-v4-1__PR80168__20260406", "pr_url": "https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR80168__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "MetricAlertDetectorHandler inherits from StatefulDetectorHandler but only contains pass, failing to implement its required abstract methods: counter_names (property), get_dedupe_value(), get_group_key_values(), and build_occurrence_and_event_data(). This will cause a TypeError at runtime when the class is instantiated.", + "severity": "High", + "matched_candidate": "MetricAlertDetectorHandler inherits from StatefulDetectorHandler but the class body only contains pass, failing to implement required abstract methods - will raise NotImplementedError at runtime", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: MetricAlertDetectorHandler inherits from StatefulDetectorHandler but only contains 'pass' and fails to implement required abstract methods, which will cause runtime errors. The golden comment specifies TypeError and lists the specific methods (counter_names, get_dedupe_value, get_group_key_values, build_occurrence_and_event_data), while the candidate mentions NotImplementedError. Despite the different error types mentioned, they're describing the same fundamental problem - missing abstract method implementations that will fail at runtime." + }, + { + "golden_comment": "Docstring says this returns a list of DetectorEvaluationResult, but the method now returns a dict keyed by DetectorGroupKey. Consider updating the docstring to match the new return type.", + "severity": "Low", + "matched_candidate": "Method docstring says it returns a list of DetectorEvaluationResult but the implementation now returns a dict keyed by DetectorGroupKey - documentation and return type are inconsistent", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: the method's docstring claims it returns a list of DetectorEvaluationResult, but the actual implementation returns a dict keyed by DetectorGroupKey. The candidate issue describes this as an inconsistency between documentation and return type, which is semantically identical to the golden comment's suggestion to update the docstring to match the new return type." + } + ], + "false_positives": [], + "false_negatives": [], + "errors": [], + "total_candidates": 2, + "total_golden": 2, + "tp": 2, + "fp": 0, + "fn": 0, + "errors_count": 0, + "precision": 1.0, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/getsentry/sentry/pull/80168" } }, "https://github.com/getsentry/sentry/pull/80528": { @@ -24700,6 +25257,41 @@ "tool": "greptile-v4-1", "repo_name": "sentry__sentry__greptile-v4-1__PR80528__20260406", "pr_url": "https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR80528__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The function modifies the config variable to include display values but then returns the original monitor.config instead of the modified version.", + "severity": "High", + "matched_candidate": "Function modifies the config variable to include display values but then returns the original monitor.config instead of the modified config, discarding the changes", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: the function modifies a local 'config' variable by adding display values, but then returns 'monitor.config' (the original, unmodified version) instead of returning the modified 'config' variable, causing the changes to be lost. The wording is slightly different but they describe the identical bug." + } + ], + "false_positives": [ + { + "candidate": "Slice operation `previous_checkins[:failure_issue_threshold]` occurs before reverse operation, taking most recent N check-ins instead of oldest N check-ins, resulting in examining wrong subset for threshold evaluation" + } + ], + "false_negatives": [ + { + "golden_comment": "The code fetches MonitorCheckIn objects by ID when the required data already exists in previous_checkins. This creates an unnecessary database query.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 2, + "total_golden": 2, + "tp": 1, + "fp": 1, + "fn": 1, + "errors_count": 0, + "precision": 0.5, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/getsentry/sentry/pull/80528" } }, "https://github.com/getsentry/sentry/pull/77754": { @@ -26413,6 +27005,53 @@ "tool": "greptile-v4-1", "repo_name": "sentry__sentry__greptile-v4-1__PR77754__20260406", "pr_url": "https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR77754__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Shared mutable default in dataclass timestamp", + "severity": "Medium", + "matched_candidate": "Mutable default argument in dataclass field - using a mutable object as default value for timestamp field shares state across instances, causing subtle data corruption bugs", + "confidence": 0.98, + "reasoning": "Both comments identify the same core issue: a mutable default value in a dataclass field (specifically the 'timestamp' field) that will be shared across instances. The golden comment uses the phrase 'shared mutable default' while the candidate uses 'mutable default argument' and explains the consequence ('shares state across instances, causing subtle data corruption bugs'). These are semantically identical - they both point to the same bug where using a mutable object as a default value in a dataclass causes unintended state sharing." + }, + { + "golden_comment": "The method name has a typo: test_from_dict_inalid_data should be test_from_dict_invalid_data.", + "severity": "Low", + "matched_candidate": "Method name typo: test_from_dict_inalid_data is missing a letter and should be test_from_dict_invalid_data - the test may not be discovered by test runners that filter by name pattern", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same typo in the method name: 'inalid' should be 'invalid' in 'test_from_dict_inalid_data'. The candidate provides additional context about potential test discovery issues, but the core issue being flagged is identical." + }, + { + "golden_comment": "Method name says 'empty_array' but tests empty dict - consider renaming to 'test_from_dict_empty_dict' for clarity.", + "severity": "Low", + "matched_candidate": "Method named 'empty_array' but tests an empty dict - test name does not match what is being tested, making the test suite misleading and harder to maintain", + "confidence": 0.98, + "reasoning": "Both comments identify the exact same issue: the test method is named 'empty_array' but it actually tests an empty dictionary, not an empty array. The golden comment suggests renaming to 'test_from_dict_empty_dict' while the candidate describes it as misleading and harder to maintain. Despite different wording and emphasis, they point to the identical underlying problem - a mismatch between the method name and what it actually tests." + }, + { + "golden_comment": "to_dict() returns a datetime for queued; if this dict is passed in task kwargs (e.g., via apply_async), JSON serialization may fail depending on the serializer, which can cause enqueue errors.", + "severity": "Medium", + "matched_candidate": "to_dict() returns a datetime object for the queued field - if this dict is passed as task kwargs to apply_async(), JSON serialization will fail since datetime is not JSON-serializable by default", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: the to_dict() method returns a datetime object for the 'queued' field, which causes JSON serialization problems when the dict is passed as task kwargs to apply_async(). The golden comment mentions this 'may fail depending on the serializer' while the candidate states it 'will fail since datetime is not JSON-serializable by default'. Despite slightly different certainty levels in the wording, they describe the exact same underlying bug and its consequences." + } + ], + "false_positives": [], + "false_negatives": [], + "errors": [], + "total_candidates": 4, + "total_golden": 4, + "tp": 4, + "fp": 0, + "fn": 0, + "errors_count": 0, + "precision": 1.0, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/getsentry/sentry/pull/77754" } }, "https://github.com/getsentry/sentry/pull/95633": { @@ -28074,6 +28713,41 @@ "tool": "greptile-v4-1", "repo_name": "sentry__sentry__greptile-v4-1__PR95633__20260406", "pr_url": "https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR95633__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The queue.shutdown() method with 'immediate=False' parameter may not exist in the standard Python queue module. This could cause AttributeError at runtime. Verify the correct API or implement a custom shutdown mechanism.", + "severity": "High", + "matched_candidate": "queue.shutdown() with 'immediate=False' parameter may not exist in the standard Python queue.Queue module - calling a non-existent method will raise AttributeError at runtime", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: the queue.shutdown() method with 'immediate=False' parameter does not exist in the standard Python queue module and will cause an AttributeError at runtime. The candidate issue uses slightly different wording but describes the identical problem, root cause, and consequence." + } + ], + "false_positives": [], + "false_negatives": [ + { + "golden_comment": "The magic number 50 for max_wait is used repeatedly throughout the tests. Consider extracting this as a named constant to improve maintainability.", + "severity": "Low" + }, + { + "golden_comment": "The test test_thread_queue_parallel_error_handling has a docstring that doesn't match the test implementation.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 1, + "total_golden": 3, + "tp": 1, + "fp": 0, + "fn": 2, + "errors_count": 0, + "precision": 1.0, + "recall": 0.3333333333333333, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/getsentry/sentry/pull/95633" } }, "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2": { @@ -30038,6 +30712,56 @@ "tool": "greptile-v4-1", "repo_name": "sentry__sentry-greptile__greptile-v4-1__PR2__20260406", "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__greptile-v4-1__PR2__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "OptimizedCursorPaginator negative-offset branch slices QuerySet with a negative start index", + "severity": "Critical", + "matched_candidate": "OptimizedCursorPaginator negative-offset branch slices QuerySet with a negative start index - Django QuerySets do not support negative indexing and will raise AssertionError", + "confidence": 1.0, + "reasoning": "Both the golden comment and candidate issue identify the exact same problem: the OptimizedCursorPaginator's negative-offset branch slices a QuerySet with a negative start index. The candidate issue adds helpful context about why this is problematic (Django QuerySets don't support negative indexing and will raise AssertionError), but the core issue being identified is identical." + }, + { + "golden_comment": "BasePaginator negative-offset branch slices QuerySet with a negative start index", + "severity": "High", + "matched_candidate": "OptimizedCursorPaginator negative-offset branch slices QuerySet with a negative start index - Django QuerySets do not support negative indexing and will raise AssertionError", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: a negative-offset branch that slices a QuerySet with a negative start index. While the golden comment refers to 'BasePaginator' and the candidate refers to 'OptimizedCursorPaginator', they are describing the same underlying bug - attempting to use negative indexing on Django QuerySets, which is not supported. The candidate provides additional context about the AssertionError that results, but this is the same issue." + }, + { + "golden_comment": "OptimizedCursorPaginator.get_item_key uses floor/ceil on a datetime key (order_by='-datetime'), causing TypeError.", + "severity": "High", + "matched_candidate": "OptimizedCursorPaginator.get_item_key applies floor and ceil to a datetime key when order_by is -datetime - math.floor and math.ceil raise TypeError when given a datetime object instead of a number", + "confidence": 1.0, + "reasoning": "Both the golden comment and candidate issue identify the exact same bug: OptimizedCursorPaginator.get_item_key incorrectly uses floor/ceil operations on a datetime key (when order_by='-datetime'), which causes a TypeError because math.floor and math.ceil cannot operate on datetime objects. The candidate provides slightly more detail about why the TypeError occurs, but both describe the same underlying issue." + } + ], + "false_positives": [ + { + "candidate": "BasePaginator negative-offset branch slices QuerySet with a negative start index - Django raises AssertionError when negative slice indices are used on QuerySets" + }, + { + "candidate": "Span with span_id='b'*16 has parent_span_id=None but is_segment_span=True, while other spans reference 'b'*16 as their parent, creating inconsistent parent-child relationship" + }, + { + "candidate": "Dictionary comprehension uses span.payload as key, but span.payload is bytes which is not hashable and cannot be used as a Redis hash key" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 3, + "tp": 3, + "fp": 3, + "fn": 0, + "errors_count": 0, + "precision": 0.6, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2" } }, "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3": { @@ -31857,6 +32581,46 @@ "tool": "greptile-v4-1", "repo_name": "sentry__sentry-greptile__greptile-v4-1__PR3__20260406", "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__greptile-v4-1__PR3__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "sample_rate = 0.0 is falsy and skipped", + "severity": "Low", + "matched_candidate": "sample_rate = 0.0 is falsy in Python - the condition if client_sample_rate evaluates to False for 0.0, silently skipping the zero sample rate case", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: that `sample_rate = 0.0` is falsy in Python, causing it to be incorrectly skipped in a conditional check. The candidate provides more technical detail about the specific condition (`if client_sample_rate`) but describes the same underlying bug where 0.0 evaluates to False and gets silently skipped." + }, + { + "golden_comment": "Using Python\u2019s built-in hash() to build cache keys is non-deterministic across processes (hash randomization), so keys won\u2019t match across workers and invalidate_upsampling_cache may fail to delete them. Use a deterministic serialization of project_ids for the cache key.", + "severity": "Low", + "matched_candidate": "Python's built-in hash() is non-deterministic across processes due to PYTHONHASHSEED - cache keys built with hash() will not match across workers, causing cache misses in distributed environments", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: Python's hash() function is non-deterministic across processes due to hash randomization (PYTHONHASHSEED), which causes cache keys to not match across workers. The golden comment mentions the specific consequence of invalidate_upsampling_cache failing to delete keys, while the candidate mentions cache misses in distributed environments. These are different manifestations of the same underlying problem - non-deterministic hash() causing cache key mismatches across processes." + }, + { + "golden_comment": "The upsampling eligibility check passes the outer dataset instead of the actual dataset used by scoped_dataset. In paths where the query ultimately runs against discover (e.g., dashboard split) while the original dataset is metrics, upsampling may be skipped even when all projects are allowlisted.", + "severity": "Medium", + "matched_candidate": "Upsampling eligibility check passes the outer dataset instead of the scoped_dataset actually used by the query - when both datasets differ, the eligibility check is performed on wrong data and may incorrectly enable or disable upsampling", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: the upsampling eligibility check uses the outer dataset instead of the scoped_dataset that is actually used by the query. The golden comment provides a specific example scenario (dashboard split with discover vs metrics), while the candidate describes the general case, but both are pointing to the same bug where the wrong dataset is used for the eligibility check, potentially causing incorrect upsampling behavior." + } + ], + "false_positives": [], + "false_negatives": [], + "errors": [], + "total_candidates": 4, + "total_golden": 3, + "tp": 3, + "fp": 0, + "fn": 0, + "errors_count": 0, + "precision": 0.75, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3" } }, "https://github.com/grafana/grafana/pull/103633": { @@ -33370,6 +34134,47 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR103633__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR103633__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The Check operation exhibits asymmetric cache trust logic: cached permission grants are trusted and returned immediately, but cached denials from the same permission cache are ignored, leading to a fresh database lookup. This allows stale cached grants to provide access to revoked resources, posing a security risk. ", + "severity": "High", + "matched_candidate": "Asymmetric cache trust logic in Check operation: cached permission grants are returned immediately without re-verification, but cache misses always go to the source. An expired or revoked permission may continue to grant access until cache expiry", + "confidence": 0.95, + "reasoning": "Both issues identify the same core problem: asymmetric cache handling in the Check operation where cached grants are trusted immediately while denials/misses require fresh lookups, creating a security vulnerability where revoked permissions can still grant access through stale cache entries. The candidate describes 'cache misses always go to the source' which is semantically equivalent to the golden's 'cached denials are ignored, leading to fresh database lookup.' Both highlight the same security risk of stale grants providing unauthorized access." + } + ], + "false_positives": [ + { + "candidate": "NoopCache implementation always returns cache miss, causing authorization checks to bypass caching entirely and potentially leading to performance degradation that could facilitate DoS attacks on authorization services" + }, + { + "candidate": "NoopCache.Get method takes a receiver pointer when it should be a value receiver since the struct has no fields" + }, + { + "candidate": "Cache key mismatch: setting cache with action 'dashboards:read' but checking with verb 'get' which likely maps to a different action" + } + ], + "false_negatives": [ + { + "golden_comment": "The test comment says the cached permissions 'allow access', but the map stores false for dashboards:uid:dash1, so checkPermission will still treat this scope as not allowed.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 4, + "total_golden": 2, + "tp": 1, + "fp": 3, + "fn": 1, + "errors_count": 0, + "precision": 0.25, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/103633" } }, "https://github.com/getsentry/sentry/pull/67876": { @@ -35096,6 +35901,46 @@ "tool": "greptile-v4-1", "repo_name": "sentry__sentry__greptile-v4-1__PR67876__20260406", "pr_url": "https://github.com/code-review-benchmark/sentry__sentry__greptile-v4-1__PR67876__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Null reference if github_authenticated_user state is missing", + "severity": "Medium", + "matched_candidate": "Potential null reference when github_authenticated_user state is not set - accessing state without checking existence can raise KeyError or AttributeError", + "confidence": 0.95, + "reasoning": "Both comments identify the same underlying issue: accessing the github_authenticated_user state without verifying its existence first, which could lead to a null/missing reference error. The candidate provides more technical detail (KeyError/AttributeError) but describes the identical problem as the golden comment." + }, + { + "golden_comment": "OAuth state uses pipeline.signature (static) instead of a per-request random value", + "severity": "Medium", + "matched_candidate": "OAuth state uses static pipeline.signature instead of per-request random nonce, allowing CSRF attacks on the OAuth callback flow", + "confidence": 1.0, + "reasoning": "Both comments identify the same core issue: the OAuth state parameter is using a static value (pipeline.signature) instead of a per-request random value. The candidate issue provides additional context about the security implication (CSRF attacks on OAuth callback), but this is the natural consequence of the problem described in the golden comment. They are pointing to the exact same bug." + }, + { + "golden_comment": "The code attempts to access integration.metadata[sender][login] without checking for the existence of the sender key. This causes a KeyError for integrations where the sender metadata was not set during creation", + "severity": "High", + "matched_candidate": "Missing key existence check before accessing integration.metadata[sender][login] - accessing nested dict key 'sender' without guard will raise KeyError when sender is absent", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: accessing `integration.metadata[sender][login]` without first checking if the 'sender' key exists in the metadata dictionary, which will cause a KeyError when the sender key is missing. The candidate uses slightly different wording ('guard', 'absent') compared to the golden comment ('checking for existence', 'not set during creation'), but they describe the identical underlying bug." + } + ], + "false_positives": [], + "false_negatives": [], + "errors": [], + "total_candidates": 3, + "total_golden": 3, + "tp": 3, + "fp": 0, + "fn": 0, + "errors_count": 0, + "precision": 1.0, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/getsentry/sentry/pull/67876" } }, "https://github.com/keycloak/keycloak/pull/32918": { @@ -36479,6 +37324,46 @@ "tool": "greptile-v4-1", "repo_name": "keycloak__keycloak__greptile-v4-1__PR32918__20260406", "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__greptile-v4-1__PR32918__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Recursive caching call using session instead of delegate", + "severity": "Critical", + "matched_candidate": "Recursive caching call uses session instead of delegate - the cache method calls session's method which re-enters caching, causing infinite recursion or incorrect cache behavior", + "confidence": 1.0, + "reasoning": "Both comments identify the same core issue: a recursive caching problem where the cache method incorrectly calls the session's method instead of using the delegate, leading to re-entry into the caching logic. The candidate provides more detail about the consequences (infinite recursion or incorrect cache behavior), but the fundamental problem being identified is identical." + }, + { + "golden_comment": "Cleanup reference uses incorrect alias - should be 'idp-alias-' + i instead of 'alias'.", + "severity": "Medium", + "matched_candidate": "Cleanup reference uses incorrect alias - should be 'idp-alias-' + i but uses 'alias', resulting in cleanup targeting wrong identity provider configuration", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same bug: the cleanup code uses the incorrect alias 'alias' when it should use 'idp-alias-' + i. The candidate provides slightly more detail about the consequence (cleanup targeting wrong identity provider), but the core issue being identified is identical." + } + ], + "false_positives": [ + { + "candidate": "Stream.of(values()).map(LoginFilter::getFilter).reduce(Predicate::and).get() will throw NoSuchElementException if values() returns an empty array" + }, + { + "candidate": "Potential null pointer exception when this.organization is null but this.organization.getId() is called" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 4, + "total_golden": 2, + "tp": 2, + "fp": 2, + "fn": 0, + "errors_count": 0, + "precision": 0.5, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/keycloak/keycloak/pull/32918" } }, "https://github.com/grafana/grafana/pull/94942": { @@ -37897,6 +38782,43 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR94942__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR94942__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The enableSqlExpressions function has flawed logic that always returns false, effectively disabling SQL expressions unconditionally:", + "severity": "Critical", + "matched_candidate": "Function always returns false regardless of feature flag state due to inverted logic and incorrect control flow", + "confidence": 1.0, + "reasoning": "Both comments identify the same critical bug: the enableSqlExpressions function has logic errors that cause it to always return false, preventing SQL expressions from being enabled. The golden comment states it 'always returns false, effectively disabling SQL expressions unconditionally' while the candidate describes 'always returns false regardless of feature flag state due to inverted logic and incorrect control flow'. Both are describing the identical malfunction - the function cannot enable SQL expressions due to flawed logic." + }, + { + "golden_comment": "Several methods such as NewInMemoryDB().RunCommands and db.QueryFramesInto return 'not implemented'.", + "severity": "High", + "matched_candidate": "Dead code: NewInMemoryDB().RunCommands and db.QueryFramesInto always return \"not implemented\" - these exported methods are unusable; either implement them or remove them to avoid confusion", + "confidence": 1.0, + "reasoning": "Both comments identify the same underlying issue: the methods NewInMemoryDB().RunCommands and db.QueryFramesInto return 'not implemented'. The candidate provides additional context about these being exported methods that are unusable and suggests either implementing or removing them, but the core issue being flagged is identical." + } + ], + "false_positives": [ + { + "candidate": "Missing return statement after error assignment - execution continues after logging error and setting rsp.Error" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 3, + "total_golden": 2, + "tp": 2, + "fp": 1, + "fn": 0, + "errors_count": 0, + "precision": 0.6666666666666666, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/94942" } }, "https://github.com/grafana/grafana/pull/90939": { @@ -39220,6 +40142,46 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR90939__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR90939__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The GetWebAssets function implements an incomplete double-checked locking pattern for caching web assets. The function first checks if the cache is populated using a read lock (RLock), and if the cache is empty, it acquires a write lock to populate it. However, it fails to re-check whether the cache was populated by another goroutine while waiting to acquire the write lock.", + "severity": "Medium", + "matched_candidate": "Incomplete double-checked locking for caching: cache is checked without the lock, then locked to set, but no second check after acquiring lock - concurrent goroutines can both see nil and both fetch the assets", + "confidence": 0.95, + "reasoning": "Both comments identify the same double-checked locking bug in the GetWebAssets function. The golden comment describes checking cache with read lock, then acquiring write lock without re-checking if another goroutine populated the cache. The candidate describes the same issue: checking cache without lock (or with read lock), acquiring write lock, but missing the second check after lock acquisition. Both point to the race condition where multiple goroutines can simultaneously fetch assets because the cache isn't re-verified after acquiring the write lock." + }, + { + "golden_comment": "In addition to the missing double-check, the function has a critical flaw in its error handling: it unconditionally assigns the fetch result to the cache (line 69: entryPointAssetsCache = result) regardless of whether the fetch succeeded or failed. When an error occurs during asset fetching, result is nil, and this nil value overwrites any previously valid cache entry.", + "severity": "High", + "matched_candidate": "Critical cache corruption: entryPointAssetsCache is unconditionally overwritten with the fetch result even when the fetch returns an error - a failed fetch will replace valid cached data with an error value", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same critical bug: the cache variable `entryPointAssetsCache` is unconditionally overwritten with the fetch result regardless of success or failure. Both note that when an error occurs, the result is nil/error value, which replaces any previously valid cached data. The golden comment mentions line 69 specifically and notes the missing double-check as additional context, while the candidate focuses solely on the cache corruption issue, but they are describing the same underlying flaw." + } + ], + "false_positives": [ + { + "candidate": "Race condition: after releasing RLock and before acquiring Lock, another goroutine could modify entryPointAssetsCache, causing the cached value 'ret' to become stale or the cache state to change" + }, + { + "candidate": "Potential null pointer dereference: 'ret' captures entryPointAssetsCache under RLock but could be nil, and is returned without validation when cfg.Env != setting.Dev" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 4, + "total_golden": 2, + "tp": 2, + "fp": 2, + "fn": 0, + "errors_count": 0, + "precision": 0.5, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/90939" } }, "https://github.com/grafana/grafana/pull/80329": { @@ -40631,6 +41593,36 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR80329__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR80329__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The code uses Error log level for what appears to be debugging information. This will pollute error logs in production. Consider using Debug or Info level instead.", + "severity": "Low", + "matched_candidate": "Using log.Error() instead of log.Debug() or log.Info() for normal operational logging during cleanup process", + "confidence": 0.95, + "reasoning": "Both comments identify the same issue: inappropriate use of Error log level for non-error situations (debugging/operational information). The golden comment mentions 'debugging information' while the candidate mentions 'normal operational logging during cleanup process' - both are describing non-error scenarios being logged at Error level. Both suggest using Debug or Info level instead. This is the same underlying problem with semantically equivalent solutions." + } + ], + "false_positives": [ + { + "candidate": "Ticker is never stopped, causing a goroutine leak when the function returns" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 2, + "total_golden": 1, + "tp": 1, + "fp": 1, + "fn": 0, + "errors_count": 0, + "precision": 0.5, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/80329" } }, "https://github.com/grafana/grafana/pull/90045": { @@ -42548,6 +43540,53 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR90045__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR90045__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The context is being created with d.Log instead of the log variable that was initialized with additional context values (name, kind, method). This means those values won't be propagated to the logging context.", + "severity": "Medium", + "matched_candidate": "Context created with d.Log instead of the log variable that was initialized with additional context values - log entries will be missing the name, kind, and method fields", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: the context is being created with d.Log instead of the log variable that contains additional context values (name, kind, method), resulting in those values not being propagated to the logging context. The candidate uses slightly different wording but describes the identical problem and consequence." + }, + { + "golden_comment": "Bug: calling recordLegacyDuration when storage operation fails should be recordStorageDuration.", + "severity": "High", + "matched_candidate": "Bug: recordLegacyDuration is called when storage operation fails but the correct function to call for storage failures is recordStorageDuration - wrong metric is being recorded on error path", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same bug: when a storage operation fails, the code incorrectly calls recordLegacyDuration instead of recordStorageDuration. The candidate provides more detail about the consequence (wrong metric being recorded on error path), but the core issue is identical." + }, + { + "golden_comment": "Inconsistency: using name instead of options.Kind for metrics recording differs from other methods.", + "severity": "Medium", + "matched_candidate": "Inconsistency: metrics recorded using name instead of options.Kind - other methods in this file use options.Kind for metric dimensions, causing this method to emit metrics with a different label format", + "confidence": 1.0, + "reasoning": "Both comments identify the same underlying issue: the method uses 'name' for metrics recording while other methods in the file use 'options.Kind', creating an inconsistency in how metrics are recorded. The candidate provides more detail about the consequence (different label format), but the core issue is identical." + } + ], + "false_positives": [ + { + "candidate": "Variable 'l' is declared as nil and never assigned a value, but is used to initialize legacyStoreMock" + }, + { + "candidate": "Variable 's' is declared as nil and never assigned a value, but is used to initialize storageMock" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 3, + "tp": 3, + "fp": 2, + "fn": 0, + "errors_count": 0, + "precision": 0.6, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/90045" } }, "https://github.com/grafana/grafana/pull/106778": { @@ -44240,6 +45279,46 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR106778__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR106778__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The rendered GrafanaRuleListItem is missing the required key prop for React list items. This can cause rendering issues when the list order changes.", + "severity": "Medium", + "matched_candidate": "GrafanaRuleListItem rendered in a list is missing the required React key prop - missing key causes React reconciliation issues when list order changes, leading to incorrect DOM updates", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: GrafanaRuleListItem is missing the required React key prop when rendered in a list. Both explain the same consequence - that this causes rendering/reconciliation issues when the list order changes. The candidate provides slightly more technical detail about 'React reconciliation' and 'incorrect DOM updates' while the golden uses 'rendering issues', but these describe the same underlying problem." + }, + { + "golden_comment": "RuleActionsButtons is invoked with only promRule, but SilenceGrafanaRuleDrawer inside RuleActionsButtons still depends on a Grafana Ruler rule being present, so for Grafana rules coming from list views the 'Silence notifications' menu entry (now driven by Grafana Prom abilities) will toggle showSilenceDrawer without ever rendering the drawer. This means clicking 'Silence notifications' for these rules has no visible effect, even when abilities indicate silencing is allowed.", + "severity": "High", + "matched_candidate": "RuleActionsButtons is invoked with only promRule argument but SilenceGrafanaRuleDrawer inside requires a Grafana Ruler rule - passing only a Prometheus rule will cause missing prop error or incorrect drawer behavior", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: RuleActionsButtons receives only a promRule (Prometheus rule) but SilenceGrafanaRuleDrawer requires a Grafana Ruler rule. The golden comment provides more detail about the user-facing consequence (clicking 'Silence notifications' has no visible effect), while the candidate describes it more technically (missing prop error or incorrect drawer behavior). However, both are pointing to the exact same architectural problem - a component dependency mismatch where the silence drawer won't work properly when only a Prometheus rule is provided." + } + ], + "false_positives": [ + { + "candidate": "Redundant type check 'rule && rule.type === PromRuleType.Alerting' after prometheusRuleType.grafana.alertingRule(rule) already verified this condition" + }, + { + "candidate": "Function calls prometheusRuleType.grafana.rule() but uses result in boolean context without handling potential undefined/null return values" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 4, + "total_golden": 2, + "tp": 2, + "fp": 2, + "fn": 0, + "errors_count": 0, + "precision": 0.5, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/106778" } }, "https://github.com/grafana/grafana/pull/107534": { @@ -45441,6 +46520,36 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR107534__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR107534__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [], + "false_positives": [ + { + "candidate": "Expected expression includes regex pattern '20|10' but mock data returns string values that would be sorted as ['1', '10', '2', '20', '3'], making the first two values '1' and '10' instead of the expected '20' and '10'" + }, + { + "candidate": "Template variables are applied to filtered queries but the original query objects may be mutated, potentially affecting other parts of the application that reference the same query objects" + } + ], + "false_negatives": [ + { + "golden_comment": "The applyTemplateVariables method is called with request.filters as the third parameter, but this parameter is not used in the corresponding test setup.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 2, + "total_golden": 1, + "tp": 0, + "fp": 2, + "fn": 1, + "errors_count": 0, + "precision": 0.0, + "recall": 0.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/107534" } }, "https://github.com/grafana/grafana/pull/79265": { @@ -47587,6 +48696,62 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR79265__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR79265__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Race condition: Multiple concurrent requests could pass the device count check simultaneously and create devices beyond the limit. Consider using a database transaction or lock.", + "severity": "High", + "matched_candidate": "Race condition: multiple concurrent requests can simultaneously pass the device count check and create devices beyond the limit - there is no atomic check-and-increment to prevent concurrent overwrites", + "confidence": 0.95, + "reasoning": "Both comments identify the same race condition issue: multiple concurrent requests can pass the device count check simultaneously and create devices beyond the limit. The golden comment suggests using a database transaction or lock as a solution, while the candidate mentions the lack of atomic check-and-increment. Both are describing the same underlying concurrency problem with the device count validation." + }, + { + "golden_comment": "Anonymous authentication now fails entirely if anonDeviceService.TagDevice returns ErrDeviceLimitReached. Previously, device tagging was asynchronous and non-blocking. This change prevents anonymous users from authenticating when the device limit is reached.", + "severity": "Medium", + "matched_candidate": "Anonymous authentication now fails entirely when ErrDeviceLimitReached - previously device tagging was asynchronous and non-blocking, now it blocks authentication for users over the device limit", + "confidence": 0.98, + "reasoning": "Both comments identify the exact same issue: anonymous authentication now fails completely when ErrDeviceLimitReached is returned, whereas previously device tagging was asynchronous and non-blocking. The candidate uses slightly more concise wording ('blocks authentication for users over the device limit' vs 'prevents anonymous users from authenticating when the device limit is reached') but describes the identical problem - the change from non-blocking to blocking behavior that causes authentication failure." + }, + { + "golden_comment": "This call won\u2019t compile: dbSession.Exec(args...) is given a []interface{} where the first element is the query, but Exec\u2019s signature requires a first parameter of type string (not an interface{} splat).", + "severity": "Medium", + "matched_candidate": "Type error: dbSession.Exec(args...) spreads a []interface{} where the first element is the query string, but Exec expects the query as a separate first argument - this will panic at runtime", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: dbSession.Exec(args...) is being called with a []interface{} spread where the first element is the query string, but Exec's signature requires the query as a distinct first parameter of type string, not as part of the variadic spread. The golden comment says it won't compile, while the candidate says it will panic at runtime - but both are describing the same type mismatch problem with the function call." + } + ], + "false_positives": [ + { + "candidate": "Error from TagDevice is returned directly when it's ErrDeviceLimitReached, but other errors are only logged as warnings and authentication continues, creating inconsistent error handling where some failures block authentication while others silently proceed" + }, + { + "candidate": "SQL injection vulnerability in updateDevice function - query string is concatenated with user-controlled parameters and passed to Exec without proper parameter binding" + } + ], + "false_negatives": [ + { + "golden_comment": "Returning ErrDeviceLimitReached when no rows were updated is misleading; the device might not exist.", + "severity": "Low" + }, + { + "golden_comment": "Time window calculation inconsistency: Using device.UpdatedAt.UTC().Add(-anonymousDeviceExpiration) as the lower bound but device.UpdatedAt as the current time may not match the intended logic. Consider using time.Now().UTC() consistently.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 5, + "tp": 3, + "fp": 2, + "fn": 2, + "errors_count": 0, + "precision": 0.6, + "recall": 0.6, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/79265" } }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9": { @@ -49042,6 +50207,44 @@ "tool": "greptile-v4-1", "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR9__20260405", "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR9__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Thread-safety issue with lazy @loaded_locales", + "severity": "Low", + "matched_candidate": "Thread-safety issue with lazy @loaded_locales - multiple threads can simultaneously check and set the instance variable without synchronization, causing duplicate locale loading or data corruption", + "confidence": 1.0, + "reasoning": "Both the golden comment and candidate issue identify the same core problem: a thread-safety issue with the lazy initialization of @loaded_locales. The candidate provides more technical detail about the consequences (duplicate locale loading, data corruption) and the mechanism (simultaneous check-and-set without synchronization), but these are elaborations on the same fundamental issue identified in the golden comment." + } + ], + "false_positives": [ + { + "candidate": "Object.send bypasses method visibility (private/protected) and can invoke any method including dangerous ones" + }, + { + "candidate": "Inconsistent locale loading logic - translate method duplicates the loaded check from ensure_loaded! but doesn't use ensure_loaded! method" + } + ], + "false_negatives": [ + { + "golden_comment": "Consider normalizing the input locale (e.g., to a symbol) when checking/loading here to avoid double-loading if the same locale is passed as a String vs Symbol (also applies to other locations in the PR).", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 4, + "total_golden": 2, + "tp": 1, + "fp": 2, + "fn": 1, + "errors_count": 0, + "precision": 0.25, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9" } }, "https://github.com/grafana/grafana/pull/76186": { @@ -50467,6 +51670,44 @@ "tool": "greptile-v4-1", "repo_name": "grafana__grafana__greptile-v4-1__PR76186__20260405", "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__greptile-v4-1__PR76186__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The ContextualLoggerMiddleware methods (QueryData, CallResource, CheckHealth, CollectMetrics) panic when a nil request is received. This occurs because they directly access req.PluginContext (via the instrumentContext function) without first checking if req is nil. This is a regression, as previous middleware layers gracefully handled nil requests.", + "severity": "High", + "matched_candidate": "ContextualLoggerMiddleware methods (QueryData, CallResource, CheckHealth, CollectMetrics) will panic when passed a nil request - dereferencing nil pointer before nil check causes runtime panic", + "confidence": 0.95, + "reasoning": "Both the golden comment and candidate issue identify the exact same problem: the ContextualLoggerMiddleware methods will panic when receiving a nil request because they dereference the request pointer (accessing req.PluginContext) before checking if it's nil. The golden comment provides additional context about it being a regression and mentions the instrumentContext function, but the core issue - nil pointer dereference causing panic in these specific methods - is identical." + } + ], + "false_positives": [ + { + "candidate": "FromContext method creates a new TestLogger instance instead of returning the current instance, losing all accumulated test data" + }, + { + "candidate": "Type assertion to *log.ConcreteLogger without checking if the returned logger is nil, which could cause a panic when accessing methods on a nil pointer" + } + ], + "false_negatives": [ + { + "golden_comment": "The traceID is no longer logged for plugin requests. During a refactoring, the tracing import and the logic to extract and add traceID from the context to log parameters were removed from the LoggerMiddleware. The newly introduced ContextualLoggerMiddleware does not add this information, resulting in missing traceID in plugin request logs and impacting debugging and request tracing capabilities.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 3, + "total_golden": 2, + "tp": 1, + "fp": 2, + "fn": 1, + "errors_count": 0, + "precision": 0.3333333333333333, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/grafana/grafana/pull/76186" } }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10": { @@ -52744,6 +53985,57 @@ "tool": "greptile-v4-1", "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR10__20260405", "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR10__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "NoMethodError before_validation in EmbeddableHost", + "severity": "Critical", + "matched_candidate": "NoMethodError in EmbeddableHost before_validation callback - calling a method on a nil object before validation will raise NoMethodError and prevent record from being saved", + "confidence": 0.95, + "reasoning": "Both comments identify the same underlying issue: a NoMethodError occurring in the EmbeddableHost class during the before_validation callback. The candidate provides more detail about the cause (calling a method on a nil object) and consequence (preventing record save), but these are elaborations on the same core problem identified in the golden comment." + }, + { + "golden_comment": "The update and destroy methods in Admin::EmbeddableHostsController do not validate the existence of the EmbeddableHost record retrieved by ID. If EmbeddableHost.where(id: params[:id]).first returns nil (i.e., the host does not exist), attempting to call methods on the nil object (e.g., save_host or destroy) will result in a NoMethodError.", + "severity": "Medium", + "matched_candidate": "EmbeddableHostsController update and destroy methods do not validate record existence - EmbeddableHost.where(id: params[:id]).first returns nil when record not found, calling update or destroy on nil raises NoMethodError", + "confidence": 1.0, + "reasoning": "Both the golden comment and candidate issue identify the exact same problem: the update and destroy methods in Admin::EmbeddableHostsController fail to validate whether EmbeddableHost.where(id: params[:id]).first returns nil, which would cause a NoMethodError when attempting to call methods (save_host/destroy or update/destroy) on the nil object. The wording is slightly different but they describe the identical underlying issue with the same root cause, affected methods, and consequence." + }, + { + "golden_comment": "record_for_host compares lower(host) = ? but does not normalize the parameter\u2019s case, so mixed\u2011case referer hosts may fail to match even though comparison intends to be case\u2011insensitive.", + "severity": "Medium", + "matched_candidate": "record_for_host compares lower(host) = ? but does not normalize the parameter case - mixed-case referer hosts like Example.COM will fail to match even though they are equivalent", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: the `record_for_host` function uses `lower(host)` in a SQL comparison but fails to normalize the input parameter to lowercase, causing mixed-case hostnames to fail matching despite the intent for case-insensitive comparison. The candidate even provides a concrete example (Example.COM) to illustrate the same problem described in the golden comment." + }, + { + "golden_comment": "Because this migration inserts embeddable_hosts rows with raw SQL, any existing embeddable_hosts values that include http:// or /https:// or path segments won\u2019t go through the EmbeddableHost model\u2019s normalization, so the new host lookup (which compares only the bare host) may fail for migrated data. Consider ensuring that migrated hosts are normalized to the same format as newly created EmbeddableHost records so existing embedding configurations keep working.", + "severity": "High", + "matched_candidate": "Migration uses raw SQL to insert embeddable_hosts rows - any existing values containing http:// or https:// scheme prefixes will be duplicated with different formats, creating data inconsistency", + "confidence": 0.85, + "reasoning": "Both comments identify the same core issue: the migration uses raw SQL to insert embeddable_hosts rows, and existing values with http:// or https:// schemes won't be properly normalized. The golden comment focuses on the consequence that lookups may fail due to format mismatches, while the candidate focuses on data duplication/inconsistency. However, both are describing the same underlying problem - the lack of normalization in the migration for existing data with scheme prefixes, leading to data integrity issues." + } + ], + "false_positives": [ + { + "candidate": "Potential NoMethodError on nil - value may be nil from hash lookup or safe navigation, calling methods on nil will raise exception" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 4, + "tp": 4, + "fp": 1, + "fn": 0, + "errors_count": 0, + "precision": 0.8, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10" } }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7": { @@ -54587,6 +55879,37 @@ "tool": "greptile-v4-1", "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR7__20260405", "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR7__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [], + "false_positives": [], + "false_negatives": [ + { + "golden_comment": "In .topic-meta-data h5 a, the original code had color: scale-color($primary, $lightness: 30%) but was changed to dark-light-choose(scale-color($primary, $lightness: 70%), scale-color($secondary, $lightness: 30%)). The lightness for the light theme changed from 30% to 70%, which is a dramatic inversion", + "severity": "Low" + }, + { + "golden_comment": "This change for desktop/user.css changes $primary from 30% to 50% for the light theme; most other changes preserve the original $primary value and move the complement to $secondary for dark. Consider reviewing this (also applies to a similar .name change in the mobile variant).", + "severity": "Low" + }, + { + "golden_comment": "In topic-post.css the original code used $lightness: 70% but the replacement uses $lightness: 30% for the light theme. This makes the text significantly darker than intended.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 0, + "total_golden": 3, + "tp": 0, + "fp": 0, + "fn": 3, + "errors_count": 0, + "precision": 0.0, + "recall": 0.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7" } }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8": { @@ -56529,210 +57852,257 @@ "tool": "greptile-v4-1", "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR8__20260405", "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR8__20260405/pull/1" - } - }, - "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3": { - "gemini": { - "skipped": false, - "true_positives": [], - "false_positives": [ - { - "candidate": "Race condition when updating match_count: concurrent requests for the same email could read the same value, increment in memory, and overwrite each other's changes, leading to incorrect match_count" - }, - { - "candidate": "Missing newline at the end of the file, which violates Ruby style guide convention" - } - ], - "false_negatives": [ - { - "golden_comment": "BlockedEmail.should_block_email? method has side effects during a read operation - it updates statistics even when just checking if an email should be blocked. This could cause race conditions in concurrent environments and makes the method name misleading.", - "severity": "Medium" - }, - { - "golden_comment": "Regex pattern @(#{domains}) only matches domain suffixes, not full domains. evil.example.com would match whitelist entry example.com.", - "severity": "Medium" - } - ], - "errors": [], - "total_candidates": 2, - "total_golden": 2, - "tp": 0, - "fp": 2, - "fn": 2, - "errors_count": 0, - "precision": 0.0, - "recall": 0.0, - "tool": "gemini", - "repo_name": "discourse__discourse-graphite__gemini__PR3__20260128", - "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__gemini__PR3__20260128/pull/1" }, - "claude": { + "codesheriff": { "skipped": false, "true_positives": [ { - "golden_comment": "BlockedEmail.should_block_email? method has side effects during a read operation - it updates statistics even when just checking if an email should be blocked. This could cause race conditions in concurrent environments and makes the method name misleading.", - "severity": "Medium", - "matched_candidate": "BlockedEmail.should_block? has validation side effects - increments counters during validation which should be side-effect-free", + "golden_comment": " The findMembers() call is now asynchronous and unhandled. The controller may not have member data immediately available, creating a race condition.", + "severity": "High", + "matched_candidate": "findMembers() is now asynchronous and its result is not awaited - the controller returns before member data is available, causing the response to contain empty or stale member information", "confidence": 0.95, - "reasoning": "Both comments identify the same core issue: the BlockedEmail validation/checking method has side effects (incrementing/updating counters/statistics) during what should be a read-only operation. The golden comment refers to 'should_block_email?' while the candidate refers to 'should_block?' - this is likely the same method or a minor naming variation. Both highlight that validation/checking operations should be side-effect-free, and both mention the problematic counter increments. The golden comment additionally mentions race conditions and misleading naming, but the fundamental issue being identified is identical." - } - ], - "false_positives": [ - { - "candidate": "BlockedEmail.should_block? uses .save instead of .save!, silently swallowing validation errors" - }, - { - "candidate": "Statistics pollution in BlockedEmail - failed user creations still increment match_count making statistics inaccurate" - }, - { - "candidate": "Transaction issues in BlockedEmail - stats are saved even if user creation fails after validation due to other validation errors" - }, - { - "candidate": "Missing newline at end of file in lib/validators/email_validator.rb" - }, - { - "candidate": "Inconsistent use of 'and' keyword instead of '&&' operator in lib/validators/email_validator.rb line 13" - } - ], - "false_negatives": [ - { - "golden_comment": "Regex pattern @(#{domains}) only matches domain suffixes, not full domains. evil.example.com would match whitelist entry example.com.", - "severity": "Medium" - } - ], - "errors": [], - "total_candidates": 7, - "total_golden": 2, - "tp": 1, - "fp": 5, - "fn": 1, - "errors_count": 0, - "precision": 0.14285714285714285, - "recall": 0.5, - "tool": "claude", - "repo_name": "discourse__discourse-graphite__claude__PR3__20260127", - "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__claude__PR3__20260127/pull/1" - }, - "graphite": { - "skipped": false, - "true_positives": [], - "false_positives": [], - "false_negatives": [ - { - "golden_comment": "BlockedEmail.should_block_email? method has side effects during a read operation - it updates statistics even when just checking if an email should be blocked. This could cause race conditions in concurrent environments and makes the method name misleading.", - "severity": "Medium" + "reasoning": "Both comments identify the same core issue: findMembers() has become asynchronous and is not being properly awaited/handled. The golden comment describes this as creating a race condition where member data may not be immediately available. The candidate issue describes the same problem - the result is not awaited, causing the controller to return before data is available, resulting in empty or stale information. These are semantically identical issues describing the same asynchronous handling bug with slightly different emphasis on the consequences." }, { - "golden_comment": "Regex pattern @(#{domains}) only matches domain suffixes, not full domains. evil.example.com would match whitelist entry example.com.", - "severity": "Medium" - } - ], - "errors": [], - "total_candidates": 0, - "total_golden": 2, - "tp": 0, - "fp": 0, - "fn": 2, - "errors_count": 0, - "precision": 0.0, - "recall": 0.0, - "tool": "graphite", - "repo_name": "discourse__discourse-graphite__graphite__PR3__20260123", - "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__graphite__PR3__20260123/pull/1" - }, - "qodo": { - "skipped": false, - "true_positives": [ - { - "golden_comment": "BlockedEmail.should_block_email? method has side effects during a read operation - it updates statistics even when just checking if an email should be blocked. This could cause race conditions in concurrent environments and makes the method name misleading.", + "golden_comment": "In the next action, capping the next offset at user_count can produce an empty page (e.g., total equal to limit results in offset == total, showing 2/2 with no members). This can cause confusing UX on the last page.", "severity": "Medium", - "matched_candidate": "Block statistics updates use read-modify-write pattern that can lose increments under concurrent signups due to race condition", - "confidence": 0.92, - "reasoning": "Both issues identify the same underlying problem: race conditions caused by the read-modify-write pattern in the BlockedEmail statistics update mechanism. The golden comment focuses on the side effects during a read operation causing race conditions, while the candidate specifically identifies the read-modify-write pattern that causes lost increments under concurrent access. These are describing the same technical issue from slightly different angles - the golden emphasizes the architectural problem (side effects in a read method), while the candidate emphasizes the specific concurrency bug (lost updates). Both point to the same code problem." - } - ], - "false_positives": [ - { - "candidate": "users_controller#create returns errors and values (including email) in JSON response on failure, potentially exposing sensitive validation messages/fields" - }, - { - "candidate": "BlockedEmail.should_block? lacks atomic update or locking mechanism for match_count and last_match_at" - }, - { - "candidate": "Email uniqueness validation may allow duplicates differing only by case without normalization or case-insensitive uniqueness enforcement" - }, - { - "candidate": "Missing database index to enforce case-insensitive email uniqueness" - }, - { - "candidate": "Client-side rejectedEmails cache doesn't normalize emails, missing equivalent emails with different casing/whitespace" + "matched_candidate": "Capping next offset at user_count produces empty page when total equals limit - offset equals total, causing the next page to return 0 results when there should be a final page", + "confidence": 0.95, + "reasoning": "Both comments identify the same issue: when the next offset is capped at user_count, it can result in an empty page when total equals limit. The golden comment mentions this causes 'confusing UX on the last page' while the candidate describes it as 'offset equals total, causing the next page to return 0 results when there should be a final page'. Both are describing the identical bug - the capping logic creates an empty final page scenario." }, { - "candidate": "Client-side rejectedEmails cache can grow unbounded during repeated signup attempts without clearing mechanism" - } - ], - "false_negatives": [ - { - "golden_comment": "Regex pattern @(#{domains}) only matches domain suffixes, not full domains. evil.example.com would match whitelist entry example.com.", - "severity": "Medium" + "golden_comment": "HTTP method mismatch in .remove_member - test uses PUT but remove_member action expects DELETE", + "severity": "Medium", + "matched_candidate": "HTTP method mismatch: remove_member test uses PUT but the remove_member controller action expects DELETE - test will always fail with 404 or 405", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: the test for remove_member uses HTTP PUT method while the controller action expects DELETE method. The candidate provides slightly more detail about the consequence (404 or 405 error), but the core issue - HTTP method mismatch between test and controller - is identical." } ], - "errors": [], - "total_candidates": 7, - "total_golden": 2, - "tp": 1, - "fp": 6, - "fn": 1, - "errors_count": 0, - "precision": 0.14285714285714285, - "recall": 0.5, - "tool": "qodo", - "repo_name": "discourse__discourse-graphite__qodo__PR3__20260123", - "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo__PR3__20260123/pull/1" - }, - "copilot": { - "skipped": false, - "true_positives": [], "false_positives": [ { - "candidate": "Race condition in read-modify-write operation (lines 14-16) can result in inaccurate match_count values when multiple requests check the same email simultaneously - the operation is not atomic so concurrent calls could overwrite each other's increments" + "candidate": "find_group method can return nil from Group.find_by, then guardian.ensure_can_see!(nil) will be called on a nil object" }, { - "candidate": "Calling save without error handling means failures will be silently ignored, which could lead to inconsistent data" - } - ], - "false_negatives": [ - { - "golden_comment": "BlockedEmail.should_block_email? method has side effects during a read operation - it updates statistics even when just checking if an email should be blocked. This could cause race conditions in concurrent environments and makes the method name misleading.", - "severity": "Medium" - }, - { - "golden_comment": "Regex pattern @(#{domains}) only matches domain suffixes, not full domains. evil.example.com would match whitelist entry example.com.", - "severity": "Medium" + "candidate": "Potential NoMethodError on nil - value may be nil from hash lookup or safe navigation, and calling methods on nil will raise an exception" } ], + "false_negatives": [], "errors": [], - "total_candidates": 2, - "total_golden": 2, - "tp": 0, + "total_candidates": 5, + "total_golden": 3, + "tp": 3, "fp": 2, - "fn": 2, + "fn": 0, "errors_count": 0, - "precision": 0.0, - "recall": 0.0, - "tool": "copilot", - "repo_name": "discourse__discourse-graphite__copilot__PR3__20260122", - "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__copilot__PR3__20260122/pull/1" - }, - "bugbot": { + "precision": 0.6, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8" + } + }, + "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3": { + "gemini": { "skipped": false, "true_positives": [], "false_positives": [ { - "candidate": "When an email is added to rejectedEmails after a server-side validation failure, the client-side validation displays 'Please enter a valid email address' which implies an email format error, but the actual rejection reason could be that the email is blocked, already taken, or from a disallowed domain" + "candidate": "Race condition when updating match_count: concurrent requests for the same email could read the same value, increment in memory, and overwrite each other's changes, leading to incorrect match_count" }, { - "candidate": "BlockedEmail.should_block? performs a case-sensitive database query, so blocking 'spam@example.com' won't block 'spam@EXAMPLE.COM' or 'SPAM@example.com', even though email addresses are case-insensitive" + "candidate": "Missing newline at the end of the file, which violates Ruby style guide convention" + } + ], + "false_negatives": [ + { + "golden_comment": "BlockedEmail.should_block_email? method has side effects during a read operation - it updates statistics even when just checking if an email should be blocked. This could cause race conditions in concurrent environments and makes the method name misleading.", + "severity": "Medium" + }, + { + "golden_comment": "Regex pattern @(#{domains}) only matches domain suffixes, not full domains. evil.example.com would match whitelist entry example.com.", + "severity": "Medium" + } + ], + "errors": [], + "total_candidates": 2, + "total_golden": 2, + "tp": 0, + "fp": 2, + "fn": 2, + "errors_count": 0, + "precision": 0.0, + "recall": 0.0, + "tool": "gemini", + "repo_name": "discourse__discourse-graphite__gemini__PR3__20260128", + "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__gemini__PR3__20260128/pull/1" + }, + "claude": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "BlockedEmail.should_block_email? method has side effects during a read operation - it updates statistics even when just checking if an email should be blocked. This could cause race conditions in concurrent environments and makes the method name misleading.", + "severity": "Medium", + "matched_candidate": "BlockedEmail.should_block? has validation side effects - increments counters during validation which should be side-effect-free", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: the BlockedEmail validation/checking method has side effects (incrementing/updating counters/statistics) during what should be a read-only operation. The golden comment refers to 'should_block_email?' while the candidate refers to 'should_block?' - this is likely the same method or a minor naming variation. Both highlight that validation/checking operations should be side-effect-free, and both mention the problematic counter increments. The golden comment additionally mentions race conditions and misleading naming, but the fundamental issue being identified is identical." + } + ], + "false_positives": [ + { + "candidate": "BlockedEmail.should_block? uses .save instead of .save!, silently swallowing validation errors" + }, + { + "candidate": "Statistics pollution in BlockedEmail - failed user creations still increment match_count making statistics inaccurate" + }, + { + "candidate": "Transaction issues in BlockedEmail - stats are saved even if user creation fails after validation due to other validation errors" + }, + { + "candidate": "Missing newline at end of file in lib/validators/email_validator.rb" + }, + { + "candidate": "Inconsistent use of 'and' keyword instead of '&&' operator in lib/validators/email_validator.rb line 13" + } + ], + "false_negatives": [ + { + "golden_comment": "Regex pattern @(#{domains}) only matches domain suffixes, not full domains. evil.example.com would match whitelist entry example.com.", + "severity": "Medium" + } + ], + "errors": [], + "total_candidates": 7, + "total_golden": 2, + "tp": 1, + "fp": 5, + "fn": 1, + "errors_count": 0, + "precision": 0.14285714285714285, + "recall": 0.5, + "tool": "claude", + "repo_name": "discourse__discourse-graphite__claude__PR3__20260127", + "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__claude__PR3__20260127/pull/1" + }, + "graphite": { + "skipped": false, + "true_positives": [], + "false_positives": [], + "false_negatives": [ + { + "golden_comment": "BlockedEmail.should_block_email? method has side effects during a read operation - it updates statistics even when just checking if an email should be blocked. This could cause race conditions in concurrent environments and makes the method name misleading.", + "severity": "Medium" + }, + { + "golden_comment": "Regex pattern @(#{domains}) only matches domain suffixes, not full domains. evil.example.com would match whitelist entry example.com.", + "severity": "Medium" + } + ], + "errors": [], + "total_candidates": 0, + "total_golden": 2, + "tp": 0, + "fp": 0, + "fn": 2, + "errors_count": 0, + "precision": 0.0, + "recall": 0.0, + "tool": "graphite", + "repo_name": "discourse__discourse-graphite__graphite__PR3__20260123", + "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__graphite__PR3__20260123/pull/1" + }, + "qodo": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "BlockedEmail.should_block_email? method has side effects during a read operation - it updates statistics even when just checking if an email should be blocked. This could cause race conditions in concurrent environments and makes the method name misleading.", + "severity": "Medium", + "matched_candidate": "Block statistics updates use read-modify-write pattern that can lose increments under concurrent signups due to race condition", + "confidence": 0.92, + "reasoning": "Both issues identify the same underlying problem: race conditions caused by the read-modify-write pattern in the BlockedEmail statistics update mechanism. The golden comment focuses on the side effects during a read operation causing race conditions, while the candidate specifically identifies the read-modify-write pattern that causes lost increments under concurrent access. These are describing the same technical issue from slightly different angles - the golden emphasizes the architectural problem (side effects in a read method), while the candidate emphasizes the specific concurrency bug (lost updates). Both point to the same code problem." + } + ], + "false_positives": [ + { + "candidate": "users_controller#create returns errors and values (including email) in JSON response on failure, potentially exposing sensitive validation messages/fields" + }, + { + "candidate": "BlockedEmail.should_block? lacks atomic update or locking mechanism for match_count and last_match_at" + }, + { + "candidate": "Email uniqueness validation may allow duplicates differing only by case without normalization or case-insensitive uniqueness enforcement" + }, + { + "candidate": "Missing database index to enforce case-insensitive email uniqueness" + }, + { + "candidate": "Client-side rejectedEmails cache doesn't normalize emails, missing equivalent emails with different casing/whitespace" + }, + { + "candidate": "Client-side rejectedEmails cache can grow unbounded during repeated signup attempts without clearing mechanism" + } + ], + "false_negatives": [ + { + "golden_comment": "Regex pattern @(#{domains}) only matches domain suffixes, not full domains. evil.example.com would match whitelist entry example.com.", + "severity": "Medium" + } + ], + "errors": [], + "total_candidates": 7, + "total_golden": 2, + "tp": 1, + "fp": 6, + "fn": 1, + "errors_count": 0, + "precision": 0.14285714285714285, + "recall": 0.5, + "tool": "qodo", + "repo_name": "discourse__discourse-graphite__qodo__PR3__20260123", + "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__qodo__PR3__20260123/pull/1" + }, + "copilot": { + "skipped": false, + "true_positives": [], + "false_positives": [ + { + "candidate": "Race condition in read-modify-write operation (lines 14-16) can result in inaccurate match_count values when multiple requests check the same email simultaneously - the operation is not atomic so concurrent calls could overwrite each other's increments" + }, + { + "candidate": "Calling save without error handling means failures will be silently ignored, which could lead to inconsistent data" + } + ], + "false_negatives": [ + { + "golden_comment": "BlockedEmail.should_block_email? method has side effects during a read operation - it updates statistics even when just checking if an email should be blocked. This could cause race conditions in concurrent environments and makes the method name misleading.", + "severity": "Medium" + }, + { + "golden_comment": "Regex pattern @(#{domains}) only matches domain suffixes, not full domains. evil.example.com would match whitelist entry example.com.", + "severity": "Medium" + } + ], + "errors": [], + "total_candidates": 2, + "total_golden": 2, + "tp": 0, + "fp": 2, + "fn": 2, + "errors_count": 0, + "precision": 0.0, + "recall": 0.0, + "tool": "copilot", + "repo_name": "discourse__discourse-graphite__copilot__PR3__20260122", + "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__copilot__PR3__20260122/pull/1" + }, + "bugbot": { + "skipped": false, + "true_positives": [], + "false_positives": [ + { + "candidate": "When an email is added to rejectedEmails after a server-side validation failure, the client-side validation displays 'Please enter a valid email address' which implies an email format error, but the actual rejection reason could be that the email is blocked, already taken, or from a disallowed domain" + }, + { + "candidate": "BlockedEmail.should_block? performs a case-sensitive database query, so blocking 'spam@example.com' won't block 'spam@EXAMPLE.COM' or 'SPAM@example.com', even though email addresses are case-insensitive" } ], "false_negatives": [ @@ -58030,6 +59400,50 @@ "tool": "greptile-v4-1", "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR3__20260405", "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR3__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "BlockedEmail.should_block_email? method has side effects during a read operation - it updates statistics even when just checking if an email should be blocked. This could cause race conditions in concurrent environments and makes the method name misleading.", + "severity": "Medium", + "matched_candidate": "BlockedEmail.should_block_email? has side effects during a read operation - it updates statistics even when only checking whether an email should be blocked, causing unexpected DB writes on reads", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: the `BlockedEmail.should_block_email?` method performs unexpected side effects (updating statistics) during what appears to be a read-only operation. The golden comment additionally mentions race conditions and misleading method naming, while the candidate focuses on unexpected DB writes, but they are describing the same fundamental problem - a query method that inappropriately modifies data." + } + ], + "false_positives": [ + { + "candidate": "Password length validation uses incorrect condition that allows nil passwords to pass when @password_required is false" + }, + { + "candidate": "Race condition in should_block? method - multiple concurrent requests for the same email could create inconsistent match_count values due to read-modify-write operation without proper locking" + }, + { + "candidate": "Method doesn't handle nil or empty email values, will cause NoMethodError when calling =~ on nil" + }, + { + "candidate": "Missing assignment operator (=) for action_type attribute in Fabricator definition" + } + ], + "false_negatives": [ + { + "golden_comment": "Regex pattern @(#{domains}) only matches domain suffixes, not full domains. evil.example.com would match whitelist entry example.com.", + "severity": "Medium" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 1, + "fp": 4, + "fn": 1, + "errors_count": 0, + "precision": 0.2, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3" } }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5": { @@ -59525,6 +60939,33 @@ "tool": "greptile-v4-1", "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR5__20260405", "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR5__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [], + "false_positives": [], + "false_negatives": [ + { + "golden_comment": "Mixing float: left with flexbox causes layout issues. Further this PR removes the float-based right alignment for .d-header .panel, which may cause the login panel in the non-Ember/noscript header (where .panel is nested inside .row and not a flex item) to stack under the title instead of remaining right-aligned.", + "severity": "Low" + }, + { + "golden_comment": "-ms-align-items never existed in any version of IE/Edge; the correct legacy property is -ms-flex-align.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 0, + "total_golden": 2, + "tp": 0, + "fp": 0, + "fn": 2, + "errors_count": 0, + "precision": 0.0, + "recall": 0.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5" } }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6": { @@ -60871,6 +62312,39 @@ "tool": "greptile-v4-1", "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR6__20260405", "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR6__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The include_website_name method is missing the required ? suffix. Rails serializers expect include_ methods to end with ? for conditional attribute inclusion, a convention followed by other methods in this serializer. Without it, the website_name attribute may not be conditionally included as intended. Additionally, the '.' << website_host string concatenation should be replaced with '.' + website_host or '.#{website_host}' to avoid mutating string literals, which can lead to issues.", + "severity": "Medium", + "matched_candidate": "include_website_name method is missing the required ? suffix - Rails serializers expect include_ conditional methods to end with ? otherwise the attribute is always serialized", + "confidence": 0.95, + "reasoning": "Both the golden comment and candidate issue identify the same primary problem: the include_website_name method is missing the required '?' suffix that Rails serializers expect for conditional attribute inclusion methods. While the golden comment additionally mentions a string concatenation issue ('.' << website_host), the candidate focuses solely on the missing '?' suffix. Since they both identify the same core bug regarding the method naming convention, this is a match. The candidate's description is more concise but captures the essential issue." + } + ], + "false_positives": [ + { + "candidate": "Logic error in subdomain comparison - when website domain is 'example.com' and current hostname is 'forums.example.com', the test expects 'example.com/user' but the logic should likely treat the subdomain as different from the parent domain" + }, + { + "candidate": "URI parsing is called twice on the same website string without caching, and the second call lacks error handling that could cause unhandled exceptions" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 3, + "total_golden": 1, + "tp": 1, + "fp": 2, + "fn": 0, + "errors_count": 0, + "precision": 0.3333333333333333, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6" } }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4": { @@ -63582,6 +65056,66 @@ "tool": "greptile-v4-1", "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR4__20260405", "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR4__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "postMessage targetOrigin should be the origin (scheme+host+port), not the full referrer URL; using the full URL will cause the message to be dropped and prevent resizing.", + "severity": "Medium", + "matched_candidate": "postMessage targetOrigin should be the origin (scheme+host+port), not the full referrer URL - using the full URL causes the message to be silently dropped by the browser", + "confidence": 0.98, + "reasoning": "Both comments identify the exact same issue: postMessage is using the full referrer URL as targetOrigin instead of just the origin (scheme+host+port), which causes the message to be silently dropped/fail. The candidate mentions 'silently dropped by the browser' while the golden says 'dropped and prevent resizing' - both describe the same consequence with slightly different detail levels." + }, + { + "golden_comment": "The code sets X-Frame-Options: ALLOWALL which completely disables clickjacking protection. The referer validation can be bypassed (referer headers are easily spoofed), and the fallback to empty string for nil referer masks validation failures.", + "severity": "Medium", + "matched_candidate": "X-Frame-Options set to ALLOWALL completely disables clickjacking protection - referer header validation can be spoofed, leaving the endpoint vulnerable to iframe-based attacks", + "confidence": 0.95, + "reasoning": "Both comments identify the same core security issue: X-Frame-Options is set to ALLOWALL which disables clickjacking protection, and both mention that referer header validation is insufficient because it can be spoofed/bypassed. The golden comment additionally mentions the fallback to empty string for nil referer, but the candidate captures the primary vulnerability. They are pointing to the same underlying security flaw." + }, + { + "golden_comment": "The ERB block closes with end if, which is invalid Ruby/ERB and will raise at render; it should just be end to close the if block.", + "severity": "Medium", + "matched_candidate": "ERB block closes with \"end if\" which is invalid Ruby ERB syntax - this will raise a SyntaxError at render time; it should be \"end\" to close the if block", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: the ERB block incorrectly uses 'end if' instead of 'end' to close the if block, which is invalid Ruby/ERB syntax and will cause a render error. The candidate provides slightly more detail by explicitly mentioning 'SyntaxError' but the core issue is identical." + } + ], + "false_positives": [ + { + "candidate": "Race condition in Redis throttling logic - another process can set key between setnx and expire calls, causing expire to be lost and key to persist indefinitely" + }, + { + "candidate": "Transaction block does not handle the case where PostCreator.create fails but doesn't raise an exception" + } + ], + "false_negatives": [ + { + "golden_comment": "SSRF vulnerability using open(url) without validation", + "severity": "Critical" + }, + { + "golden_comment": "The current origin validation using indexOf is insufficient and can be bypassed. An attacker could use a malicious domain like evil-discourseUrl.com to pass this check.", + "severity": "Medium" + }, + { + "golden_comment": "The TopicEmbed.import method is susceptible to a NoMethodError if the contents parameter is nil when attempting to append a string, and an XSS vulnerability due to unescaped url interpolation in the generated HTML.", + "severity": "Medium" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 6, + "tp": 3, + "fp": 2, + "fn": 3, + "errors_count": 0, + "precision": 0.6, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4" } }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1": { @@ -65404,6 +66938,53 @@ "tool": "greptile-v4-1", "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR1__20260405", "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR1__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The downsize method is defined twice. The second definition, which expects a single dimensions string parameter, overrides the first, which expected separate max_width and max_height parameters. This makes the original method unreachable and breaks existing code that calls it with separate width and height arguments.", + "severity": "Medium", + "matched_candidate": "The downsize method is defined twice - the second definition overrides the first, causing dead code and potentially breaking behavior that depended on the original two-argument signature", + "confidence": 0.98, + "reasoning": "Both comments identify the exact same issue: the downsize method is defined twice, with the second definition overriding the first. Both note that this creates dead/unreachable code and breaks existing functionality that depends on the original two-parameter signature (max_width and max_height vs dimensions string). The candidate uses slightly different terminology ('dead code' vs 'unreachable', 'two-argument signature' vs 'separate width and height arguments') but describes the identical underlying problem." + }, + { + "golden_comment": "Hardcoding maxSizeKB = 10 * 1024 ignores Discourse.SiteSettings['max_' + type + '_size_kb'], so the client-side limit can diverge from server-side and per-type settings (also applies to the 413 handler below).", + "severity": "Low", + "matched_candidate": "Hardcoding maxSizeKB = 10 * 1024 ignores Discourse.SiteSettings max_type_size_kb - client-side limit can diverge from server-side configuration, allowing uploads that exceed the server limit", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: hardcoding maxSizeKB = 10 * 1024 ignores the Discourse.SiteSettings configuration for max size per type, which can cause the client-side limit to diverge from the server-side limit. The golden comment mentions the specific setting name format 'max_' + type + '_size_kb' and also notes the 413 handler is affected, while the candidate uses 'max_type_size_kb' and focuses on uploads exceeding server limits. Despite minor wording differences, both identify the same fundamental problem of hardcoded values bypassing dynamic settings." + }, + { + "golden_comment": "Passing 80% as the dimensions can fail for animated GIFs when allow_animated_thumbnails is true, since the animated path uses gifsicle --resize-fit which expects WxH geometry, not a percentage; downsizing would then silently fail.", + "severity": "Medium", + "matched_candidate": "Passing 80% as dimensions to gifsicle fails for animated GIFs - the animated path requires absolute pixel dimensions, not percentages, causing animated thumbnail generation to fail", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: passing percentage dimensions (80%) to gifsicle fails for animated GIFs because the animated path requires absolute pixel dimensions (WxH geometry) rather than percentages. The golden comment mentions '--resize-fit' and 'allow_animated_thumbnails' flag as context, while the candidate is more concise, but both describe the identical bug - percentage dimensions causing animated thumbnail generation to fail." + } + ], + "false_positives": [ + { + "candidate": "Image downsizing loop can run indefinitely if OptimizedImage.downsize fails to reduce file size, creating an infinite loop condition" + }, + { + "candidate": "tempfile.size is called without checking if tempfile is nil, which will raise NoMethodError if file download failed" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 3, + "tp": 3, + "fp": 2, + "fn": 0, + "errors_count": 0, + "precision": 0.6, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1" } }, "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2": { @@ -67019,6 +68600,49 @@ "tool": "greptile-v4-1", "repo_name": "discourse__discourse-graphite__greptile-v4-1__PR2__20260405", "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__greptile-v4-1__PR2__20260405/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "logic: Potential nil pointer exception - if no TopicUser record exists, tu will be nil and calling methods on it will crash", + "severity": "High", + "matched_candidate": "Logic assumes TopicUser record exists but find_by can return nil, leading to NoMethodError on nil object", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: the `find_by` method can return nil when no TopicUser record exists, and calling methods on this nil object will cause a crash (nil pointer exception/NoMethodError). The candidate uses Ruby-specific terminology (NoMethodError) while the golden uses more general terms (nil pointer exception), but they describe the identical problem." + }, + { + "golden_comment": "Typo in property name: 'stopNotificiationsText' should be 'stopNotificationsText' (missing 'n' in 'Notifications')", + "severity": "Low", + "matched_candidate": "Typo in property name: 'stopNotificiationsText' should be 'stopNotificationsText' (missing 'n' in Notifications) - the property will never be found causing silent undefined access", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same typo: 'stopNotificiationsText' should be 'stopNotificationsText' (missing 'n' in 'Notifications'). The candidate issue provides additional context about the consequence (silent undefined access), but the core issue being identified is identical." + } + ], + "false_positives": [ + { + "candidate": "Race condition: TopicUser.change is called without checking if the current post number is actually newer than the existing last_emailed_post_number" + }, + { + "candidate": "Potential variable name typo: code appears to reference 'topi' instead of 'topic_id' in MessageBus.publish call" + }, + { + "candidate": "Route 't/:topic_id/summary' will never be reached because it's defined after 't/:slug/:topic_id/summary' route, and Rails matches routes in order" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 2, + "fp": 3, + "fn": 0, + "errors_count": 0, + "precision": 0.4, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2" } }, "https://github.com/calcom/cal.com/pull/22532": { @@ -68720,6 +70344,49 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR22532__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR22532__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The updateManyByCredentialId call uses an empty data object, which prevents Prisma's @updatedAt decorator from updating the updatedAt timestamp. This results in inaccurate cache status tracking, as the timestamp isn't updated when the cache is refreshed. To fix this, explicitly set the updatedAt field.", + "severity": "Medium", + "matched_candidate": "updateManyByCredentialId called with empty data object {} - Prisma @updatedAt decorator only fires when the record has actual field changes, so the updatedAt timestamp will not be updated on cache invalidation", + "confidence": 0.98, + "reasoning": "Both the golden comment and candidate issue identify the exact same problem: the updateManyByCredentialId call uses an empty data object {}, which prevents Prisma's @updatedAt decorator from updating the updatedAt timestamp because Prisma only triggers @updatedAt when there are actual field changes. Both describe the same consequence (timestamp not being updated) and the same root cause (empty data object with Prisma's @updatedAt behavior). The golden comment additionally suggests a fix (explicitly set the updatedAt field), but the core issue identification is identical." + }, + { + "golden_comment": "logic: macOS-specific sed syntax with empty string after -i flag will fail on Linux systems", + "severity": "Low", + "matched_candidate": "sed command uses macOS-specific syntax with empty string after -i flag, will fail on Linux systems", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: the sed command uses macOS-specific syntax (empty string after -i flag) that will fail on Linux systems. The candidate issue is semantically identical to the golden comment, just with slightly different wording." + } + ], + "false_positives": [ + { + "candidate": "Race condition potential - if cache statuses are updated between the getCacheStatusByCredentialIds call and the map creation, the cacheStatusMap may contain stale data" + }, + { + "candidate": "findUniqueOrThrow method uses SelectedCalendarWhereInput instead of SelectedCalendarWhereUniqueInput, which will cause a type error since findUniqueOrThrow requires unique identifiers" + }, + { + "candidate": "deleteCacheMutation is called with credentialId but the mutation expects a different parameter structure based on the trpc endpoint name 'deleteCache'" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 2, + "fp": 3, + "fn": 0, + "errors_count": 0, + "precision": 0.4, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/22532" } }, "https://github.com/calcom/cal.com/pull/8330": { @@ -70318,6 +71985,50 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR8330__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR8330__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Using === for dayjs object comparison will always return false as it compares object references, not values. Use .isSame() method instead: dayjs(date.start).add(utcOffset, 'minutes').isSame(dayjs(date.end).add(utcOffset, minutes))", + "severity": "Medium", + "matched_candidate": "Comparison using === instead of isSame() for Dayjs objects will always be false since they are different object instances", + "confidence": 0.95, + "reasoning": "Both comments identify the same underlying issue: using === to compare dayjs objects compares object references rather than values, which will always return false. The golden comment additionally suggests using the .isSame() method as a solution, while the candidate explains why the comparison fails (different object instances). Despite slightly different wording and level of detail, they both point to the exact same bug." + } + ], + "false_positives": [ + { + "candidate": "Variable i is initialized to -1 but the code is incomplete - this suggests a loop that may have off-by-one errors when incrementing/accessing array elements" + }, + { + "candidate": "Assertion expects results for plus2DateString but the time range query spans from plus1DateString to plus2DateString, potentially missing or incorrectly filtering results" + }, + { + "candidate": "startTime calculation duplicates the same dayjs computation twice instead of reusing the result, causing unnecessary work and potential inconsistency" + }, + { + "candidate": "endTime calculation duplicates the same dayjs computation twice instead of reusing the result, causing unnecessary work and potential inconsistency" + } + ], + "false_negatives": [ + { + "golden_comment": "Incorrect end time calculation using slotStartTime instead of slotEndTime", + "severity": "Medium" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 1, + "fp": 4, + "fn": 1, + "errors_count": 0, + "precision": 0.2, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/8330" } }, "https://github.com/calcom/cal.com/pull/14943": { @@ -71757,6 +73468,49 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR14943__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR14943__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can lose increments under concurrency; consider an atomic increment via Prisma (increment: 1) to avoid race conditions (also applies to the similar update in the catch block).", + "severity": "High", + "matched_candidate": "retryCount reads a possibly stale value with reminder.retryCount + 1 - concurrent requests can both increment from the same base, losing increments; use atomic increment via Prisma update", + "confidence": 0.95, + "reasoning": "Both comments identify the same race condition issue: using `reminder.retryCount + 1` can lead to lost increments under concurrent execution because it reads a potentially stale value. Both suggest the same solution: using Prisma's atomic increment operation. The golden comment additionally mentions the catch block has the same issue, but the core problem identified is identical." + }, + { + "golden_comment": "The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workflow reminders (e.g., Email, WhatsApp) that have retryCount > 1. This occurs because the retryCount condition within the OR clause for deletion lacks a method: WorkflowMethods.SMS filter, causing it to apply to all reminder types instead of only SMS reminders, which is the intended scope of this function.", + "severity": "High", + "matched_candidate": "scheduleSMSReminders.ts deletion logic incorrectly deletes non-SMS workflow reminders (Email, WhatsApp) that share the same workflow - the filter condition is too broad and removes reminders of unintended types", + "confidence": 0.95, + "reasoning": "Both the golden comment and candidate issue identify the same core problem: the deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workflow reminders (Email, WhatsApp). The golden comment specifically attributes this to the retryCount condition lacking a 'method: WorkflowMethods.SMS' filter, while the candidate describes it as a 'too broad filter condition'. Both are describing the same bug - that the deletion filter applies to all reminder types instead of being restricted to SMS reminders only. The candidate's phrasing 'removes reminders of unintended types' directly aligns with the golden comment's statement that it 'deletes non-SMS workflow reminders'. This is semantically the same issue despite slightly different technical descriptions." + } + ], + "false_positives": [ + { + "candidate": "OR condition deletes SMS reminders with retryCount > 1, conflicting with retry logic that increments retryCount on failures" + }, + { + "candidate": "Using lte (less than or equal) with current date will delete SMS reminders scheduled for exactly the current time, preventing them from being sent if the handler runs at the exact scheduled moment" + }, + { + "candidate": "Duplicate retry count increment logic - retryCount is incremented both in the 'else' block for failed SMS and in the catch block for exceptions, potentially double-counting failures" + } + ], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 2, + "fp": 3, + "fn": 0, + "errors_count": 0, + "precision": 0.4, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/14943" } }, "https://github.com/calcom/cal.com/pull/22345": { @@ -73193,6 +74947,47 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR22345__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR22345__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Fetching userIdsFromOrg only when teamsFromOrg.length > 0 can exclude org-level members for orgs without child teams; consider deriving from teamIds (which includes orgId) or removing the guard so org-only orgs still include member user bookings.", + "severity": "Medium", + "matched_candidate": "Logic error: fetching userIdsFromOrg only when teamsFromOrg.length > 0 excludes org-level members for organizations without child teams - org members not in any team will be incorrectly excluded from results", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: the conditional check `teamsFromOrg.length > 0` before fetching `userIdsFromOrg` causes org-level members to be excluded when an organization has no child teams. The golden comment suggests considering alternatives like deriving from teamIds or removing the guard, while the candidate describes the consequence (org members not in teams being excluded). Despite different wording and level of detail in the solution, they point to the exact same bug." + } + ], + "false_positives": [ + { + "candidate": "Logic flaw in condition combining - if both authConditions and filterConditions are falsy, returns NOTHING_CONDITION which will exclude all results" + }, + { + "candidate": "Constructor accepts InsightsBookingServicePublicOptions but assigns to InsightsBookingServiceOptions type field, causing type mismatch" + }, + { + "candidate": "Missing await for async cleanup operation - test may complete before cleanup finishes, potentially causing interference with subsequent tests" + } + ], + "false_negatives": [ + { + "golden_comment": "In getBaseConditions(), the else if (filterConditions) and final else branches are unreachable. This is because getAuthorizationConditions() always returns a non-null Prisma.Sql object, making authConditions always truthy, which means only the first two if/else if conditions are ever evaluated.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 4, + "total_golden": 2, + "tp": 1, + "fp": 3, + "fn": 1, + "errors_count": 0, + "precision": 0.25, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/22345" } }, "https://github.com/calcom/cal.com/pull/11059": { @@ -75854,6 +77649,62 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR11059__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR11059__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Invalid Zod schema syntax. Computed property keys like [z.string().toString()] are not valid in Zod object schemas and will cause runtime errors. ", + "severity": "High", + "matched_candidate": "Invalid Zod schema syntax - `z.string().toString()` creates a string literal 'string' as a key, not a dynamic string key pattern", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: the invalid use of `z.string().toString()` as a computed property key in a Zod schema. The golden comment focuses on the fact that computed property keys are not valid in Zod schemas and will cause runtime errors. The candidate issue explains the specific problem with this syntax - that `toString()` creates a literal string 'string' rather than a dynamic key pattern. Both are describing the same underlying bug from slightly different angles - the invalid syntax and its consequences." + }, + { + "golden_comment": "parseRefreshTokenResponse returns a Zod safeParse result ({ success, data, error }), not the credential key object. Persisting that as key stores the wrapper instead of the token payload; we should store the parsed data or use schema parse.", + "severity": "High", + "matched_candidate": "parseRefreshTokenResponse returns a Zod safeParse result object ({ success, data, error }) not the credential object - callers that persist this directly will store the wrapper object instead of the token data", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: parseRefreshTokenResponse returns a Zod safeParse result object (with success, data, error properties) rather than the credential/token object itself. Both note that persisting this result directly will incorrectly store the wrapper object instead of the actual token data. The golden comment additionally suggests solutions (store parsed data or use schema parse), but the core issue identified is identical." + }, + { + "golden_comment": "When APP_CREDENTIAL_SHARING_ENABLED and CALCOM_CREDENTIAL_SYNC_ENDPOINT are set, the refreshFunction helper returns the fetch Response, but several callers (for example GoogleCalendarService.refreshAccessToken expecting res.data, and HubspotCalendarService.refreshAccessToken expecting a HubspotToken) assume it returns the integration-specific token object. That mismatch will cause runtime errors in the sync-enabled path unless the return type or those call sites are adjusted.", + "severity": "High", + "matched_candidate": "When sync endpoint is used, res is a fetch Response object and res.data does not exist - accessing res?.data will return undefined and token.access_token will be undefined, silently breaking token refresh", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: when the sync endpoint is enabled (APP_CREDENTIAL_SHARING_ENABLED and CALCOM_CREDENTIAL_SYNC_ENDPOINT are set), the refreshFunction returns a fetch Response object instead of the expected token object. The golden comment explains that callers like GoogleCalendarService and HubspotCalendarService expect specific token objects (res.data or HubspotToken), while the candidate focuses on the specific consequence that res.data will be undefined, causing token.access_token to be undefined. Both are describing the same type mismatch problem - the candidate is just more specific about one manifestation of the issue." + }, + { + "golden_comment": "When the sync endpoint path is used, res is a fetch Response and has no .data; res?.data will be undefined and token.access_token will throw at runtime. This relies on a consistent return shape from refreshOAuthTokens, which isn\u2019t guaranteed currently.", + "severity": "High", + "matched_candidate": "refreshFunction helper returns a fetch Response object when the sync endpoint path is used, but callers expect a credential object with access_token - res.data does not exist on Response, returning undefined and breaking token refresh silently", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: when the sync endpoint path is used, the refreshFunction returns a fetch Response object instead of a credential object. This causes res.data to be undefined (since Response objects don't have a .data property), which then breaks the token.access_token access. The golden comment mentions 'res?.data will be undefined and token.access_token will throw at runtime' while the candidate says 'res.data does not exist on Response, returning undefined and breaking token refresh'. Both are describing the same runtime error from inconsistent return shapes." + } + ], + "false_positives": [ + { + "candidate": "Incorrect expiry date calculation - adding seconds to milliseconds timestamp" + } + ], + "false_negatives": [ + { + "golden_comment": "The parseRefreshTokenResponse function incorrectly sets refresh_token to the hardcoded string 'refresh_token' when it's missing from the OAuth refresh token response. This invalidates the token, breaking subsequent token refreshes and causing authentication failures.", + "severity": "High" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 5, + "tp": 4, + "fp": 1, + "fn": 1, + "errors_count": 0, + "precision": 0.8, + "recall": 0.8, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/11059" } }, "https://github.com/calcom/cal.com/pull/7232": { @@ -77589,6 +79440,47 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR7232__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR7232__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Asynchronous functions deleteScheduledEmailReminder and deleteScheduledSMSReminder are called without await inside forEach loops. This occurs during booking rescheduling/cancellation, and workflow/workflow step deletion/updates. Consequently, scheduled workflow reminders may not be reliably cancelled, potentially leaving them active.", + "severity": "Medium", + "matched_candidate": "deleteScheduledEmailReminder and deleteScheduledSMSReminder are async functions called without await inside forEach loops - reminder deletions fire-and-forget, errors are silently lost, and cleanup may be incomplete", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: async functions deleteScheduledEmailReminder and deleteScheduledSMSReminder are called without await inside forEach loops. The golden comment emphasizes the consequence of unreliable cancellation during booking operations and workflow updates. The candidate issue describes the same problem using 'fire-and-forget' terminology and mentions silent error loss and incomplete cleanup, which are direct consequences of the missing await. Both are describing the identical underlying bug with slightly different emphasis on the consequences." + } + ], + "false_positives": [ + { + "candidate": "Race condition in onChange handler - form.getValues() may not reflect current input value due to asynchronous form updates" + }, + { + "candidate": "Database delete operation is not awaited but function continues execution, potentially causing race conditions" + }, + { + "candidate": "deleteScheduledEmailReminder is called with a third parameter 'true' but deleteScheduledSMSReminder is called without it, suggesting inconsistent API usage" + } + ], + "false_negatives": [ + { + "golden_comment": "When immediateDelete is true, the deleteScheduledEmailReminder function cancels the SendGrid email but fails to delete the corresponding WorkflowReminder record from the database. This creates orphaned database entries and is inconsistent with the immediateDelete: false path, which marks the record as cancelled. The SendGrid DELETE API call is also omitted in this path.", + "severity": "High" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 1, + "fp": 3, + "fn": 1, + "errors_count": 0, + "precision": 0.2, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/7232" } }, "https://github.com/calcom/cal.com/pull/14740": { @@ -79885,6 +81777,60 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR14740__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR14740__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Case sensitivity bypass in email blacklist", + "severity": "High", + "matched_candidate": "Case sensitivity bypass in email blacklist: comparison does not normalize email case, allowing BLOCKED@example.com to bypass a blacklist entry for blocked@example.com", + "confidence": 1.0, + "reasoning": "Both the golden comment and candidate issue identify the exact same security vulnerability: the email blacklist can be bypassed due to case sensitivity in the comparison logic. The candidate provides more detail by giving a concrete example (BLOCKED@example.com vs blocked@example.com), but the core issue is identical - the lack of case normalization in email comparison allows blacklist bypass." + }, + { + "golden_comment": "The logic for checking team admin/owner permissions is incorrect. This condition uses AND (&&) which requires both isTeamAdmin AND isTeamOwner to be true, but it should use OR (||) since a user needs to be either an admin OR an owner to have permission.", + "severity": "Critical", + "matched_candidate": "Logic error using AND (&&) instead of OR (||) to check if user is team admin OR owner - currently requires user to be BOTH admin AND owner, which is incorrect", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same logical error: the use of AND (&&) instead of OR (||) when checking team permissions. Both explain that the current code incorrectly requires a user to be BOTH admin AND owner, when it should allow either role. The candidate issue is semantically identical to the golden comment, just more concisely worded." + }, + { + "golden_comment": "This calls the email sender with the original guests, so existing attendees included in the input will be treated as new when sending notifications, leading to incorrect emails.", + "severity": "Medium", + "matched_candidate": "Email sender called with original guests list - existing attendees included in the input are treated as new invitees, sending duplicate invitations to guests who are already attending", + "confidence": 0.98, + "reasoning": "Both comments identify the same underlying issue: the email sender is called with the original guests list, causing existing attendees to be treated as new invitees, which results in incorrect/duplicate email notifications. The candidate issue provides slightly more detail by explicitly mentioning 'duplicate invitations,' but the core problem is identical." + }, + { + "golden_comment": "uniqueGuests filters out existing attendees and blacklisted emails but does not deduplicate duplicates within the input; createMany can insert duplicate attendee rows if the client submits repeated emails.", + "severity": "Medium", + "matched_candidate": "uniqueGuests filters out existing attendees and blacklisted emails but does not deduplicate within the input itself - createMany can insert duplicate guest records if the same email appears multiple times", + "confidence": 0.98, + "reasoning": "Both comments identify the exact same issue: the uniqueGuests filter removes existing attendees and blacklisted emails but fails to deduplicate emails within the input array itself, which allows createMany to insert duplicate records when the same email appears multiple times in the input. The wording is slightly different ('attendee rows' vs 'guest records', 'repeated emails' vs 'same email appears multiple times') but the underlying bug and its consequence are identical." + }, + { + "golden_comment": "Starting with an array containing an empty string may cause validation issues. Consider starting with an empty array [] and handling the empty state in the MultiEmail component instead.", + "severity": "Low", + "matched_candidate": "Incorrect validation condition checks if array length is 0, but initial state contains an empty string [\"\"], so length will be 1, not 0", + "confidence": 0.95, + "reasoning": "Both comments identify the same underlying issue: the initial state is set to an array containing an empty string [''] rather than an empty array [], which causes problems with validation logic that checks for array length. The golden comment suggests this may cause validation issues and recommends using an empty array instead. The candidate issue specifically points out that validation checking for length === 0 will fail because the array with an empty string has length 1. These are describing the same problem from slightly different angles - the root cause (wrong initial state) and its consequence (validation failure)." + } + ], + "false_positives": [], + "false_negatives": [], + "errors": [], + "total_candidates": 5, + "total_golden": 5, + "tp": 5, + "fp": 0, + "fn": 0, + "errors_count": 0, + "precision": 1.0, + "recall": 1.0, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/14740" } }, "https://github.com/calcom/cal.com/pull/10600": { @@ -81994,6 +83940,58 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR10600__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR10600__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Error message mentions 'backup code login' but this is a disable endpoint, not login", + "severity": "Low", + "matched_candidate": "Error message says 'backup code login' but this endpoint disables 2FA - the message incorrectly describes the action, confusing users about what operation is being performed", + "confidence": 1.0, + "reasoning": "Both comments identify the same issue: the error message incorrectly mentions 'backup code login' when the endpoint is actually for disabling 2FA (not login). The candidate provides more detail about user confusion, but the core problem - the mismatch between the error message text and the actual endpoint functionality - is identical." + }, + { + "golden_comment": "Backup code validation is case-sensitive due to the use of indexOf(). This causes validation to fail if a user enters uppercase hex characters, as backup codes should be case-insensitive for a better user experience.", + "severity": "Medium", + "matched_candidate": "Backup code validation is case-sensitive due to indexOf() - users entering uppercase hex characters will fail validation even though backup codes should be case-insensitive", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: backup code validation is case-sensitive because of indexOf() usage, causing validation failures when users enter uppercase hex characters, when it should be case-insensitive. The wording is nearly identical and they describe the same root cause, symptom, and expected behavior." + }, + { + "golden_comment": "Because backupCodes are decrypted and mutated in memory before being written back, two concurrent login requests using the same backupCode could both pass this check and update, so a single backup code may effectively be accepted more than once if used concurrently, weakening the intended one-time-use semantics.", + "severity": "High", + "matched_candidate": "Race condition in backup code validation: backup codes are decrypted and mutated in memory before being written back - two concurrent login requests can both read the same unused code and both succeed, allowing code reuse", + "confidence": 1.0, + "reasoning": "Both the golden comment and candidate issue identify the exact same race condition vulnerability: backup codes are decrypted and mutated in memory before being written back to storage, allowing two concurrent login requests to both read the same unused backup code and both succeed validation. This breaks the one-time-use semantics of backup codes. The candidate uses slightly different wording ('code reuse' vs 'accepted more than once') but describes the identical underlying issue with the same root cause and consequence." + } + ], + "false_positives": [ + { + "candidate": "Race condition: isDisabling state is not reset if the API call fails or throws an exception" + }, + { + "candidate": "Race condition: clicking download button and immediately awaiting the download event without ensuring the UI is ready for interaction" + } + ], + "false_negatives": [ + { + "golden_comment": "The exported function TwoFactor handles backup codes and is in BackupCode.tsx. Inconsistent naming.", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 4, + "tp": 3, + "fp": 2, + "fn": 1, + "errors_count": 0, + "precision": 0.6, + "recall": 0.75, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/10600" } }, "https://github.com/calcom/cal.com/pull/10967": { @@ -84419,6 +86417,55 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR10967__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR10967__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "Potential null reference if mainHostDestinationCalendar is undefined if evt.destinationCalendar is null or an empty array ", + "severity": "High", + "matched_candidate": "Array destructuring assumes evt.destinationCalendar has at least one element, but fallback array is empty which makes mainHostDestinationCalendar undefined", + "confidence": 0.95, + "reasoning": "Both comments identify the same underlying issue: mainHostDestinationCalendar can become undefined when evt.destinationCalendar is null or empty. The golden comment mentions 'null or an empty array' while the candidate mentions 'fallback array is empty' - both are describing the same destructuring problem where the fallback `|| []` results in an empty array, making the destructured variable undefined." + }, + { + "golden_comment": "Logic inversion in organization creation: The slug property is now conditionally set when IS_TEAM_BILLING_ENABLED is true, instead of when it's false as originally intended. This change, combined with requestedSlug still being set when IS_TEAM_BILLING_ENABLED is true, results in both properties being set when billing is enabled, and neither when disabled", + "severity": "Medium", + "matched_candidate": "Logic inversion in organization creation: slug property is conditionally set when IS_TEAM_BILLING_ENABLED is true instead of false - organizations created without billing enabled will be missing the slug field", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: the logic for setting the slug property has been inverted. The golden comment provides more detail about the consequences (both properties set when billing enabled, neither when disabled), while the candidate focuses on one consequence (missing slug when billing disabled). However, they both correctly identify the same bug - the conditional logic for slug assignment is backwards relative to the original intent." + }, + { + "golden_comment": "The Calendar interface now requires createEvent(event, credentialId), but some implementations (e.g., Lark/Office365) still declare createEvent(event) only\u2014this breaks the interface contract (also applies to other locations in the PR).", + "severity": "Low", + "matched_candidate": "Calendar interface now requires createEvent(event, credentialId) but implementations like Lark and Office365 still declare createEvent(event) without credentialId - TypeScript interface mismatch causes runtime errors", + "confidence": 1.0, + "reasoning": "Both comments identify the exact same issue: the Calendar interface has been updated to require createEvent(event, credentialId) with two parameters, but some implementations (specifically mentioning Lark and Office365) still only implement createEvent(event) with one parameter. This is an interface contract violation/mismatch. The candidate adds context about TypeScript and runtime errors, but the core issue being identified is identical." + } + ], + "false_positives": [], + "false_negatives": [ + { + "golden_comment": "The optional chaining on mainHostDestinationCalendar?.integration is redundant since you already check mainHostDestinationCalendar in the ternary condition.", + "severity": "Low" + }, + { + "golden_comment": "Logic error: when externalCalendarId is provided, you're searching for a calendar where externalId === externalCalendarId, but this will always fail since you're looking for a calendar that matches itself. Should likely find by credentialId or use different logic.", + "severity": "High" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 5, + "tp": 3, + "fp": 0, + "fn": 2, + "errors_count": 0, + "precision": 0.6, + "recall": 0.6, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/10967" } }, "https://github.com/calcom/cal.com/pull/8087": { @@ -86133,6 +88180,44 @@ "tool": "greptile-v4-1", "repo_name": "cal_dot_com__cal.com__greptile-v4-1__PR8087__20260406", "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__greptile-v4-1__PR8087__20260406/pull/1" + }, + "codesheriff": { + "skipped": false, + "true_positives": [ + { + "golden_comment": "The code uses forEach with async callbacks, which causes asynchronous operations (e.g., calendar/video event deletions, payment refunds) to run concurrently without being awaited. This 'fire-and-forget' behavior leads to unhandled promise rejections, race conditions, and incomplete cleanup, as surrounding try-catch blocks cannot properly handle errors from these unawaited promises. Replace forEach with for...of loops or Promise.all() with map() to ensure proper sequential execution and error handling.", + "severity": "Critical", + "matched_candidate": "Critical: forEach with async callbacks causes fire-and-forget execution - calendar and video event deletions run concurrently without coordination, errors are silently swallowed, and the function may return before operations complete", + "confidence": 0.95, + "reasoning": "Both comments identify the same core issue: using forEach with async callbacks leads to fire-and-forget execution where asynchronous operations run without being awaited. Both mention that this causes errors to be silently swallowed/unhandled, operations run concurrently without proper coordination, and the function may complete before async operations finish. The golden comment additionally suggests specific solutions (for...of or Promise.all with map), but the candidate captures all the key problems: concurrent execution, missing error handling, and incomplete cleanup." + } + ], + "false_positives": [ + { + "candidate": "reduce function can concatenate undefined values into the array when getAvailability() returns undefined, creating an array with undefined elements" + }, + { + "candidate": "Template literal displays [object Object] instead of meaningful app identifier when paymentApp is an object" + } + ], + "false_negatives": [ + { + "golden_comment": "Consider adding try-catch around the await to handle import failures gracefully", + "severity": "Low" + } + ], + "errors": [], + "total_candidates": 5, + "total_golden": 2, + "tp": 1, + "fp": 2, + "fn": 1, + "errors_count": 0, + "precision": 0.2, + "recall": 0.5, + "tool": "codesheriff", + "repo_name": null, + "pr_url": "https://github.com/calcom/cal.com/pull/8087" } } } \ No newline at end of file diff --git a/offline/results/benchmark_data.json b/offline/results/benchmark_data.json index 95da859..c358724 100644 --- a/offline/results/benchmark_data.json +++ b/offline/results/benchmark_data.json @@ -1364,6 +1364,11 @@ "created_at": "2026-04-07T19:38:04Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/keycloak/keycloak/pull/37429", + "review_comments": [] } ] }, @@ -2498,6 +2503,11 @@ "created_at": "2026-04-07T19:31:41Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/keycloak/keycloak/pull/37634", + "review_comments": [] } ] }, @@ -3646,6 +3656,36 @@ "created_at": "2026-04-07T19:31:24Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/keycloak/keycloak/pull/38446", + "review_comments": [ + { + "path": "services/src/main/java/org/keycloak/forms/login/freemarker/model/RecoveryAuthnCodeInputLoginBean.java", + "line": 5, + "body": "**CodeSheriff: Users.Oc.Codesheriff Bench.Rules.Java Optional Get Without Check** [CRITICAL]\n\nCalling Optional.get() without first checking isPresent() will throw NoSuchElementException if the Optional is empty. Use isPresent(), orElse(), orElseGet(), or orElseThrow() to handle the empty case explicitly.\n\n", + "created_at": "2026-04-10T05:32:15.324Z" + }, + { + "path": "services/src/main/java/org/keycloak/forms/login/freemarker/model/RecoveryAuthnCodeInputLoginBean.java", + "line": 5, + "body": "**CodeSheriff: Users.Oc.Codesheriff Bench.Rules.Java Optional Get Without Ispresent** [CRITICAL]\n\nCalling Optional.get() without checking isPresent() will throw NoSuchElementException when the Optional is empty. Use orElse(), orElseGet(), orElseThrow(), or check isPresent() before calling get().\n\n", + "created_at": "2026-04-10T05:32:15.324Z" + }, + { + "path": "testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java", + "line": 15, + "body": "**CodeSheriff: Recovery codes are stored in plain text in credential data without any encryption or hashing** [CRITICAL]\n\nRecovery codes are stored in plain text in credential data without any encryption or hashing. An attacker with access to the credential storage can directly read and use these backup authentication codes to bypass MFA protection.\n", + "created_at": "2026-04-10T05:32:15.324Z" + }, + { + "path": "services/src/main/java/org/keycloak/forms/login/freemarker/model/RecoveryAuthnCodeInputLoginBean.java", + "line": 5, + "body": "**CodeSheriff: Calling ** [HIGH]\n\nCalling .get() on Optional without checking if it's present first. If RecoveryAuthnCodesUtils.getCredential(user) returns empty Optional, this will throw NoSuchElementException, potentially causing denial of service or revealing internal application state through stack traces.\n", + "created_at": "2026-04-10T05:32:15.324Z" + } + ] } ] }, @@ -4526,6 +4566,11 @@ "created_at": "2026-04-07T19:21:12Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/keycloak/keycloak/pull/36882", + "review_comments": [] } ] }, @@ -5684,6 +5729,24 @@ "created_at": "2026-04-07T19:32:43Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/keycloak/keycloak/pull/36880", + "review_comments": [ + { + "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java", + "line": 71, + "body": "**CodeSheriff: Users.Oc.Codesheriff Bench.Rules.Java Recursive Delegation Bug** [CRITICAL]\n\nMethod appears to call itself recursively instead of delegating to a wrapped or parent object. This can cause infinite recursion and StackOverflowError. Verify that the recursive call is intentional and has a proper base case, or change to delegate to the correct target (e.g., delegate.method() instead of this.method()).\n\n", + "created_at": "2026-04-10T05:27:38.123Z" + }, + { + "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionsV2.java", + "line": 265, + "body": "**CodeSheriff: Users.Oc.Codesheriff Bench.Rules.Java Recursive Delegation Bug** [CRITICAL]\n\nMethod appears to call itself recursively instead of delegating to a wrapped or parent object. This can cause infinite recursion and StackOverflowError. Verify that the recursive call is intentional and has a proper base case, or change to delegate to the correct target (e.g., delegate.method() instead of this.method()).\n\n", + "created_at": "2026-04-10T05:27:38.123Z" + } + ] } ] }, @@ -6947,6 +7010,18 @@ "created_at": "2026-04-07T19:34:49Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/keycloak/keycloak/pull/37038", + "review_comments": [ + { + "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/UserPermissionsV2.java", + "line": 1, + "body": "**CodeSheriff: Authorization bypass due to missing closing braces in if statements** [CRITICAL]\n\nAuthorization bypass due to missing closing braces in if statements. All permission checks after the first if statement will execute unconditionally, allowing unauthorized users to perform admin operations like managing users, mapping roles, and managing group membership without proper MANAGE_USERS role verification.\n", + "created_at": "2026-04-10T05:29:04.803Z" + } + ] } ] }, @@ -8079,6 +8154,18 @@ "created_at": "2026-04-07T19:29:46Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/keycloak/keycloak/pull/33832", + "review_comments": [ + { + "path": "authz/client/src/main/java/org/keycloak/authorization/client/util/crypto/ASN1Encoder.java", + "line": 56, + "body": "**CodeSheriff: Users.Oc.Codesheriff Bench.Rules.Java Recursive Delegation Bug** [CRITICAL]\n\nMethod appears to call itself recursively instead of delegating to a wrapped or parent object. This can cause infinite recursion and StackOverflowError. Verify that the recursive call is intentional and has a proper base case, or change to delegate to the correct target (e.g., delegate.method() instead of this.method()).\n\n", + "created_at": "2026-04-10T05:25:40.104Z" + } + ] } ] }, @@ -9085,6 +9172,24 @@ "created_at": "2026-04-07T19:29:09Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/keycloak/keycloak/pull/40940", + "review_comments": [ + { + "path": "model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/GroupAdapter.java", + "line": 1, + "body": "**CodeSheriff: Users.Oc.Codesheriff Bench.Rules.Java Optional Get Without Check** [CRITICAL]\n\nCalling Optional.get() without first checking isPresent() will throw NoSuchElementException if the Optional is empty. Use isPresent(), orElse(), orElseGet(), or orElseThrow() to handle the empty case explicitly.\n\n", + "created_at": "2026-04-10T05:32:28.365Z" + }, + { + "path": "model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/GroupAdapter.java", + "line": 1, + "body": "**CodeSheriff: Users.Oc.Codesheriff Bench.Rules.Java Optional Get Without Ispresent** [CRITICAL]\n\nCalling Optional.get() without checking isPresent() will throw NoSuchElementException when the Optional is empty. Use orElse(), orElseGet(), orElseThrow(), or check isPresent() before calling get().\n\n", + "created_at": "2026-04-10T05:32:28.365Z" + } + ] } ] }, @@ -10209,6 +10314,18 @@ "created_at": "2026-04-07T19:31:17Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1", + "review_comments": [ + { + "path": "services/src/main/java/org/keycloak/authentication/authenticators/browser/AbstractUsernameFormAuthenticator.java", + "line": 1, + "body": "**CodeSheriff: Users.Oc.Codesheriff Bench.Rules.Java Hardcoded Credential Secret Pattern** [CRITICAL]\n\nHardcoded credential or API key in source. Keys checked into source control are effectively public \u2014 they leak via git history, CI logs, and forks. Move the value to an environment variable, a JNDI binding, or a secrets manager (Vault, AWS Secrets Manager, Spring Cloud Config).\n\n", + "created_at": "2026-04-10T05:24:02.210Z" + } + ] } ] }, @@ -11364,6 +11481,11 @@ "created_at": "2026-04-07T19:29:13Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/getsentry/sentry/pull/93824", + "review_comments": [] } ] }, @@ -12907,6 +13029,11 @@ "created_at": "2026-04-07T19:31:30Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5", + "review_comments": [] } ] }, @@ -14316,6 +14443,30 @@ "created_at": "2026-04-07T19:32:48Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1", + "review_comments": [ + { + "path": "src/sentry/api/paginator.py", + "line": 6, + "body": "**CodeSheriff: Users.Oc.Codesheriff Bench.Rules.Python Queryset Negative Slice** [CRITICAL]\n\nDjango QuerySets do not support negative indexing or negative slicing. Using a negative index on a QuerySet raises AssertionError. Convert to a list first or use alternative logic to access elements from the end.\n\n", + "created_at": "2026-04-10T04:37:58.622Z" + }, + { + "path": "src/sentry/api/paginator.py", + "line": 71, + "body": "**CodeSheriff: Users.Oc.Codesheriff Bench.Rules.Python Queryset Negative Slice** [CRITICAL]\n\nDjango QuerySets do not support negative indexing or negative slicing. Using a negative index on a QuerySet raises AssertionError. Convert to a list first or use alternative logic to access elements from the end.\n\n", + "created_at": "2026-04-10T04:37:58.622Z" + }, + { + "path": "src/sentry/api/paginator.py", + "line": 75, + "body": "**CodeSheriff: Users.Oc.Codesheriff Bench.Rules.Python Queryset Negative Slice** [CRITICAL]\n\nDjango QuerySets do not support negative indexing or negative slicing. Using a negative index on a QuerySet raises AssertionError. Convert to a list first or use alternative logic to access elements from the end.\n\n", + "created_at": "2026-04-10T04:37:58.622Z" + } + ] } ] }, @@ -15331,6 +15482,11 @@ "created_at": "2026-04-07T19:31:16Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/grafana/grafana/pull/97529", + "review_comments": [] } ] }, @@ -16389,6 +16545,11 @@ "created_at": "2026-04-07T19:31:42Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/getsentry/sentry/pull/80168", + "review_comments": [] } ] }, @@ -17364,6 +17525,18 @@ "created_at": "2026-04-07T19:29:29Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/getsentry/sentry/pull/80528", + "review_comments": [ + { + "path": "src/sentry/monitors/logic/mark_failed.py", + "line": 2, + "body": "**CodeSheriff: Hallucinated API: sentry.monitors.logic.incidents.try_incident_threshold** [HIGH]\n\nThe function try_incident_threshold does not exist in sentry.monitors.logic.incidents module\n", + "created_at": "2026-04-10T04:52:49.537Z" + } + ] } ] }, @@ -18346,6 +18519,18 @@ "created_at": "2026-04-07T19:30:55Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/getsentry/sentry/pull/77754", + "review_comments": [ + { + "path": "src/sentry/integrations/services/assignment_source.py", + "line": 15, + "body": "**CodeSheriff: Hallucinated API: queued: datetime = timezone.now()** [HIGH]\n\ntimezone.now() is not callable - it should be timezone.now\n", + "created_at": "2026-04-10T04:51:48.317Z" + } + ] } ] }, @@ -19449,6 +19634,11 @@ "created_at": "2026-04-07T19:31:36Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/getsentry/sentry/pull/95633", + "review_comments": [] } ] }, @@ -20822,6 +21012,30 @@ "created_at": "2026-04-07T19:30:55Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2", + "review_comments": [ + { + "path": "src/sentry/api/paginator.py", + "line": 6, + "body": "**CodeSheriff: Users.Oc.Codesheriff Bench.Rules.Python Queryset Negative Slice** [CRITICAL]\n\nDjango QuerySets do not support negative indexing or negative slicing. Using a negative index on a QuerySet raises AssertionError. Convert to a list first or use alternative logic to access elements from the end.\n\n", + "created_at": "2026-04-10T04:38:22.876Z" + }, + { + "path": "src/sentry/api/paginator.py", + "line": 71, + "body": "**CodeSheriff: Users.Oc.Codesheriff Bench.Rules.Python Queryset Negative Slice** [CRITICAL]\n\nDjango QuerySets do not support negative indexing or negative slicing. Using a negative index on a QuerySet raises AssertionError. Convert to a list first or use alternative logic to access elements from the end.\n\n", + "created_at": "2026-04-10T04:38:22.876Z" + }, + { + "path": "src/sentry/api/paginator.py", + "line": 75, + "body": "**CodeSheriff: Users.Oc.Codesheriff Bench.Rules.Python Queryset Negative Slice** [CRITICAL]\n\nDjango QuerySets do not support negative indexing or negative slicing. Using a negative index on a QuerySet raises AssertionError. Convert to a list first or use alternative logic to access elements from the end.\n\n", + "created_at": "2026-04-10T04:38:22.876Z" + } + ] } ] }, @@ -22209,6 +22423,11 @@ "created_at": "2026-04-07T19:31:12Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3", + "review_comments": [] } ] }, @@ -23255,6 +23474,11 @@ "created_at": "2026-04-07T19:28:44Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/grafana/grafana/pull/103633", + "review_comments": [] } ] }, @@ -24449,6 +24673,36 @@ "created_at": "2026-04-07T19:30:58Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/getsentry/sentry/pull/67876", + "review_comments": [ + { + "path": "tests/sentry/integrations/github/test_integration.py", + "line": 3, + "body": "**CodeSheriff: API key in source code** [CRITICAL]\n\nHardcoded credential detected \u2014 must be moved to environment variables.\n", + "created_at": "2026-04-10T04:40:39.907Z" + }, + { + "path": "src/sentry/integrations/github/integration.py", + "line": 3, + "body": "**CodeSheriff: Hallucinated API: sentry.http.safe_urlopen** [HIGH]\n\nModule sentry.http does not exist - this appears to be a hallucinated import\n", + "created_at": "2026-04-10T04:40:39.908Z" + }, + { + "path": "src/sentry/integrations/github/integration.py", + "line": 3, + "body": "**CodeSheriff: Hallucinated API: sentry.http.safe_urlread** [HIGH]\n\nModule sentry.http does not exist - this appears to be a hallucinated import\n", + "created_at": "2026-04-10T04:40:39.908Z" + }, + { + "path": "src/sentry/integrations/github/integration.py", + "line": 4, + "body": "**CodeSheriff: Hallucinated API: sentry.identity.github.get_user_info** [HIGH]\n\nFunction get_user_info does not exist in sentry.identity.github module\n", + "created_at": "2026-04-10T04:40:39.908Z" + } + ] } ] }, @@ -25447,6 +25701,18 @@ "created_at": "2026-04-07T19:33:42Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/keycloak/keycloak/pull/32918", + "review_comments": [ + { + "path": "model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/idp/InfinispanIdentityProviderStorageProvider.java", + "line": 14, + "body": "**CodeSheriff: Users.Oc.Codesheriff Bench.Rules.Java Recursive Delegation Bug** [CRITICAL]\n\nMethod appears to call itself recursively instead of delegating to a wrapped or parent object. This can cause infinite recursion and StackOverflowError. Verify that the recursive call is intentional and has a proper base case, or change to delegate to the correct target (e.g., delegate.method() instead of this.method()).\n\n", + "created_at": "2026-04-10T05:24:28.113Z" + } + ] } ] }, @@ -26384,6 +26650,11 @@ "created_at": "2026-04-07T19:28:24Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/grafana/grafana/pull/94942", + "review_comments": [] } ] }, @@ -27221,6 +27492,11 @@ "created_at": "2026-04-07T19:27:53Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/grafana/grafana/pull/90939", + "review_comments": [] } ] }, @@ -28348,6 +28624,11 @@ "created_at": "2026-04-07T19:29:11Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/grafana/grafana/pull/80329", + "review_comments": [] } ] }, @@ -29879,6 +30160,11 @@ "created_at": "2026-04-07T19:28:57Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/grafana/grafana/pull/90045", + "review_comments": [] } ] }, @@ -31076,6 +31362,11 @@ "created_at": "2026-04-07T19:31:17Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/grafana/grafana/pull/106778", + "review_comments": [] } ] }, @@ -31975,6 +32266,11 @@ "created_at": "2026-04-07T19:33:42Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/grafana/grafana/pull/107534", + "review_comments": [] } ] }, @@ -33190,6 +33486,11 @@ "created_at": "2026-04-07T19:29:14Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/grafana/grafana/pull/79265", + "review_comments": [] } ] }, @@ -34117,6 +34418,11 @@ "created_at": "2026-04-07T19:29:37Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9", + "review_comments": [] } ] }, @@ -35031,6 +35337,11 @@ "created_at": "2026-04-07T19:27:58Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/grafana/grafana/pull/76186", + "review_comments": [] } ] }, @@ -36703,6 +37014,11 @@ "created_at": "2026-04-07T19:31:16Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10", + "review_comments": [] } ] }, @@ -37867,6 +38183,11 @@ "created_at": "2026-04-07T19:28:29Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7", + "review_comments": [] } ] }, @@ -39446,6 +39767,11 @@ "created_at": "2026-04-07T19:31:09Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8", + "review_comments": [] } ] }, @@ -40598,6 +40924,11 @@ "created_at": "2026-04-07T19:29:21Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3", + "review_comments": [] } ] }, @@ -41603,6 +41934,11 @@ "created_at": "2026-04-07T19:28:31Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5", + "review_comments": [] } ] }, @@ -42634,6 +42970,11 @@ "created_at": "2026-04-07T19:29:01Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6", + "review_comments": [] } ] }, @@ -44573,6 +44914,11 @@ "created_at": "2026-04-07T19:29:25Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4", + "review_comments": [] } ] }, @@ -45858,6 +46204,11 @@ "created_at": "2026-04-07T19:29:04Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1", + "review_comments": [] } ] }, @@ -47073,6 +47424,11 @@ "created_at": "2026-04-07T19:29:36Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2", + "review_comments": [] } ] }, @@ -48402,6 +48758,11 @@ "created_at": "2026-04-07T19:31:33Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/calcom/cal.com/pull/22532", + "review_comments": [] } ] }, @@ -49618,6 +49979,24 @@ "created_at": "2026-04-07T19:31:31Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/calcom/cal.com/pull/8330", + "review_comments": [ + { + "path": "packages/lib/slots.ts", + "line": 2, + "body": "**CodeSheriff: Hallucinated API: dayjs().tz()** [HIGH]\n\nThe tz() method requires the dayjs/plugin/timezone plugin to be imported and extended, but no timezone plugin is visible in the dependencies or imports\n", + "created_at": "2026-04-10T05:10:27.448Z" + }, + { + "path": "packages/lib/slots.ts", + "line": 3, + "body": "**CodeSheriff: Hallucinated API: dayjs().tz()** [HIGH]\n\nThe tz() method requires the dayjs/plugin/timezone plugin to be imported and extended, but no timezone plugin is visible in the dependencies or imports\n", + "created_at": "2026-04-10T05:10:27.448Z" + } + ] } ] }, @@ -50648,6 +51027,11 @@ "created_at": "2026-04-07T19:29:08Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/calcom/cal.com/pull/14943", + "review_comments": [] } ] }, @@ -51616,6 +52000,11 @@ "created_at": "2026-04-07T19:32:49Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/calcom/cal.com/pull/22345", + "review_comments": [] } ] }, @@ -53480,6 +53869,24 @@ "created_at": "2026-04-07T19:33:06Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/calcom/cal.com/pull/11059", + "review_comments": [ + { + "path": "packages/app-store/_utils/oauth/parseRefreshTokenResponse.ts", + "line": 8, + "body": "**CodeSheriff: Hallucinated API: [z.string().toString()]: z.number()** [HIGH]\n\nInvalid Zod schema syntax - computed property keys with toString() are not valid in Zod object schemas\n", + "created_at": "2026-04-10T05:12:40.355Z" + }, + { + "path": "packages/app-store/_utils/oauth/parseRefreshTokenResponse.ts", + "line": 10, + "body": "**CodeSheriff: Hallucinated API: [z.string().optional().toString()]: z.unknown().optional()** [HIGH]\n\nInvalid Zod schema syntax - computed property keys with toString() are not valid in Zod object schemas\n", + "created_at": "2026-04-10T05:12:40.355Z" + } + ] } ] }, @@ -54887,6 +55294,11 @@ "created_at": "2026-04-07T19:35:41Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/calcom/cal.com/pull/7232", + "review_comments": [] } ] }, @@ -56457,6 +56869,11 @@ "created_at": "2026-04-07T19:31:18Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/calcom/cal.com/pull/14740", + "review_comments": [] } ] }, @@ -57824,6 +58241,18 @@ "created_at": "2026-04-07T19:31:37Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/calcom/cal.com/pull/10600", + "review_comments": [ + { + "path": "apps/web/components/settings/EnableTwoFactorModal.tsx", + "line": 1, + "body": "**CodeSheriff: Hallucinated API: PasswordField** [MEDIUM]\n\nPasswordField is not a valid export from @calcom/ui\n", + "created_at": "2026-04-10T05:04:41.209Z" + } + ] } ] }, @@ -59525,6 +59954,11 @@ "created_at": "2026-04-07T19:29:36Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/calcom/cal.com/pull/10967", + "review_comments": [] } ] }, @@ -60751,6 +61185,18 @@ "created_at": "2026-04-07T19:30:55Z" } ] + }, + { + "tool": "codesheriff", + "pr_url": "https://github.com/calcom/cal.com/pull/8087", + "review_comments": [ + { + "path": "packages/core/videoClient.ts", + "line": 5, + "body": "**CodeSheriff: Hallucinated API: appStore[appName as keyof typeof appStore]** [HIGH]\n\nVariable 'appName' is not defined anywhere in the code\n", + "created_at": "2026-04-10T05:03:12.404Z" + } + ] } ] }