Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions src/ModularPipelines/Console/ModuleOutputBuffer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -813,10 +813,13 @@ public void WriteTo(ILogger logger)
public string? FormatException()
=> _obfuscatedException is null
? null
: secretObfuscator.Obfuscate(_obfuscatedException.ToString(), null);
: Obfuscate(_obfuscatedException.ToString());

private string Format(object? state, Exception? logException)
=> secretObfuscator.Obfuscate(_rawFormattedMessage.Value, null) ?? string.Empty;
=> Obfuscate(_rawFormattedMessage.Value);

private string Obfuscate(string value)
=> secretObfuscator.Obfuscate(value, null);

private string FormatTyped(TState state, Exception? logException)
=> Format(state!, logException);
Expand Down
11 changes: 10 additions & 1 deletion src/ModularPipelines/Engine/ISecretObfuscator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,20 @@ namespace ModularPipelines.Engine;
/// </summary>
public interface ISecretObfuscator
{
/// <summary>
/// Gets whether any secrets are currently registered for global masking.
/// </summary>
/// <remarks>
/// This is a performance hint only. Callers must not use it to bypass safety or masking
/// behavior that custom implementations may provide.
/// </remarks>
bool HasSecrets => true;

/// <summary>
/// Obfuscates sensitive information in the provided input.
/// </summary>
/// <param name="input">The input string that may contain sensitive information.</param>
/// <param name="optionsObject">An options object that may contain sensitive properties.</param>
/// <returns>The input with sensitive information obfuscated.</returns>
string Obfuscate(string? input, object? optionsObject);
}
}
10 changes: 10 additions & 0 deletions src/ModularPipelines/Engine/SecretObfuscator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ internal class SecretObfuscator : ISecretObfuscator, IInitializer

public int Order => int.MaxValue;

public bool HasSecrets => GetRegistrationState().HasSecrets;

public SecretObfuscator(
ISecretProvider secretProvider,
IOptions<SecretMaskingOptions> maskingOptions)
Expand Down Expand Up @@ -78,6 +80,12 @@ public string Obfuscate(string? input, object? optionsObject)
caseInsensitive ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal);
}

internal SecretRegistrationState GetRegistrationState()
{
var cache = GetRegisteredSecretCache(_maskingOptions.Value.CaseInsensitive);
return new SecretRegistrationState(cache.Version, cache.SearchValues is not null);
}

internal SecretCache GetSecretCache(
object? optionsObject,
SecretMaskingOptions options,
Expand Down Expand Up @@ -257,4 +265,6 @@ internal sealed record SecretCache(
string[] Secrets,
IReadOnlySet<string> ExactSecrets,
SearchValues<string>? SearchValues);

internal readonly record struct SecretRegistrationState(long Version, bool HasSecrets);
}
47 changes: 43 additions & 4 deletions src/ModularPipelines/Logging/FormattedLogValuesObfuscator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,35 @@ public FormattedLogValuesObfuscator(ISecretObfuscator secretObfuscator)
}

public object TryObfuscateValues(object state)
{
// HasSecrets is only a hint; custom obfuscators may still apply policy-based masking.
if (_secretObfuscator is not SecretObfuscator builtInObfuscator)
{
return TryObfuscateValues(state, skipOrdinaryValues: false);
}

var registrationState = builtInObfuscator.GetRegistrationState();
var skipOrdinaryValues = !registrationState.HasSecrets;
var obfuscatedState = TryObfuscateValues(state, skipOrdinaryValues);

if (!skipOrdinaryValues)
{
return obfuscatedState;
}

// A secret may be registered while the zero-secret fast path traverses the state.
var currentRegistrationState = builtInObfuscator.GetRegistrationState();
return currentRegistrationState.Version != registrationState.Version
&& currentRegistrationState.HasSecrets
? TryObfuscateValues(state, skipOrdinaryValues: false)
: obfuscatedState;
}

private object TryObfuscateValues(object state, bool skipOrdinaryValues)
{
if (state is not IReadOnlyList<KeyValuePair<string, object?>> values)
{
return ObfuscateValue(state);
return skipOrdinaryValues ? state : ObfuscateValue(state);
}

KeyValuePair<string, object?>[]? obfuscatedValues = null;
Expand All @@ -46,7 +71,20 @@ public object TryObfuscateValues(object state)
continue;
}

var obfuscatedValue = ObfuscateValue(property.Value);
object obfuscatedValue;
if (property.Value is PreObfuscatedLogValue preObfuscatedValue)
{
obfuscatedValue = preObfuscatedValue.Value;
}
else if (!skipOrdinaryValues)
{
obfuscatedValue = ObfuscateValue(property.Value);
}
else
{
continue;
}

if (ReferenceEquals(obfuscatedValue, property.Value))
{
continue;
Expand All @@ -69,15 +107,16 @@ private object ObfuscateValue(object value)
string originalValue;
try
{
originalValue = value.ToString() ?? string.Empty;
originalValue = value as string ?? value.ToString() ?? string.Empty;
}
catch (Exception)
{
return value;
}

var obfuscatedValue = _secretObfuscator.Obfuscate(originalValue, null);
return obfuscatedValue.Equals(originalValue, StringComparison.Ordinal)
return ReferenceEquals(obfuscatedValue, originalValue)
|| obfuscatedValue.Equals(originalValue, StringComparison.Ordinal)
? value
: obfuscatedValue;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@ public async Task BufferedLogEvent_FormatsOnceAndObfuscatesEveryTime()
{
var formatterCalls = 0;
var secretObfuscator = new Mock<ISecretObfuscator>();
secretObfuscator.SetupGet(x => x.HasSecrets).Returns(true);
secretObfuscator
.Setup(x => x.Obfuscate(It.IsAny<string?>(), null))
.Returns((string? value, object? _) => value?.Replace("secret", "***") ?? string.Empty);
Expand Down Expand Up @@ -236,6 +237,7 @@ public async Task BufferedLogEvent_ReobfuscatesMessageWithCurrentSecrets()
const string secret = "late-registered-secret";
var redactSecret = false;
var secretObfuscator = new Mock<ISecretObfuscator>();
secretObfuscator.SetupGet(x => x.HasSecrets).Returns(() => redactSecret);
secretObfuscator
.Setup(x => x.Obfuscate(It.IsAny<string?>(), null))
.Returns((string? value, object? _) => redactSecret
Expand Down Expand Up @@ -268,6 +270,7 @@ public async Task BufferedLogEvent_ReobfuscatesExceptionWithCurrentSecrets()
const string secret = "late-registered-secret";
var redactSecret = false;
var secretObfuscator = new Mock<ISecretObfuscator>();
secretObfuscator.SetupGet(x => x.HasSecrets).Returns(() => redactSecret);
secretObfuscator
.Setup(x => x.Obfuscate(It.IsAny<string?>(), null))
.Returns((string? value, object? _) => redactSecret
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,84 @@
using Microsoft.Extensions.Logging;
using ModularPipelines.Engine;
using ModularPipelines.Logging;
using ModularPipelines.Options;
using Moq;

namespace ModularPipelines.UnitTests.Logging;

public class FormattedLogValuesObfuscatorTests
{
[Test]
public async Task TryObfuscateValues_DoesNotInspectStateWhenNoSecretsAreRegistered()
{
var state = new CountingToStringState();
var secretObfuscator = CreateBuiltInObfuscator();

var obfuscatedState = new FormattedLogValuesObfuscator(secretObfuscator)
.TryObfuscateValues(state);

await Assert.That(obfuscatedState).IsSameReferenceAs(state);
await Assert.That(state.ToStringCalls).IsEqualTo(0);
}

[Test]
public async Task TryObfuscateValues_PreservesCustomMaskingWhenHintIsFalse()
{
const string secret = "policy-secret";
var secretObfuscator = new Mock<ISecretObfuscator>();
secretObfuscator.SetupGet(x => x.HasSecrets).Returns(false);
secretObfuscator
.Setup(x => x.Obfuscate(It.IsAny<string?>(), null))
.Returns((string? value, object? _) =>
(value ?? string.Empty).Replace(secret, "********", StringComparison.Ordinal));
var state = new[]
{
new KeyValuePair<string, object?>("PolicyValue", secret),
};

var obfuscatedState = new FormattedLogValuesObfuscator(secretObfuscator.Object)
.TryObfuscateValues(state);
var value = ((IReadOnlyList<KeyValuePair<string, object?>>) obfuscatedState)[0].Value;

await Assert.That(value).IsEqualTo("********");
}

[Test]
public async Task TryObfuscateValues_RetriesWhenSecretIsRegisteredDuringFastPath()
{
const string secret = "dynamic-secret";
var version = 0L;
IReadOnlyList<string> secrets = [];
var secretProvider = new Mock<ISecretProvider>();
secretProvider.SetupGet(x => x.Version).Returns(() => version);
secretProvider.Setup(x => x.GetSnapshot())
.Returns(() => new SecretSnapshot(version, secrets));
var secretObfuscator = CreateBuiltInObfuscator(secretProvider.Object);
var values = new[] { new KeyValuePair<string, object?>("Value", secret) };
var registered = false;
var state = new Mock<IReadOnlyList<KeyValuePair<string, object?>>>();
state.SetupGet(x => x.Count).Returns(() =>
{
if (!registered)
{
registered = true;
secrets = [secret];
version += 2;
}

return values.Length;
});
state.Setup(x => x[0]).Returns(values[0]);
state.Setup(x => x.GetEnumerator())
.Returns(() => ((IEnumerable<KeyValuePair<string, object?>>) values).GetEnumerator());

var obfuscatedState = new FormattedLogValuesObfuscator(secretObfuscator)
.TryObfuscateValues(state.Object);
var value = ((IReadOnlyList<KeyValuePair<string, object?>>) obfuscatedState)[0].Value;

await Assert.That(value).IsEqualTo("**********");
}

[Test]
public async Task TryObfuscateValues_MasksSecretsInOriginalFormat()
{
Expand All @@ -17,6 +89,7 @@ public async Task TryObfuscateValues_MasksSecretsInOriginalFormat()

var state = logger.Invocations.Single(x => x.Method.Name == nameof(ILogger.Log)).Arguments[2];
var secretObfuscator = new Mock<ISecretObfuscator>();
secretObfuscator.SetupGet(x => x.HasSecrets).Returns(true);
secretObfuscator
.Setup(x => x.Obfuscate(It.IsAny<string?>(), null))
.Returns((string? value, object? _) => (value ?? string.Empty).Replace(secret, "********", StringComparison.Ordinal));
Expand All @@ -43,6 +116,7 @@ public async Task TryObfuscateValues_PreservesUnmaskedStructuredValueTypes()

var state = logger.Invocations.Single(x => x.Method.Name == nameof(ILogger.Log)).Arguments[2];
var secretObfuscator = new Mock<ISecretObfuscator>();
secretObfuscator.SetupGet(x => x.HasSecrets).Returns(true);
secretObfuscator
.Setup(x => x.Obfuscate(It.IsAny<string?>(), null))
.Returns((string? value, object? _) => value == "secret" ? "********" : value ?? string.Empty);
Expand All @@ -66,6 +140,7 @@ public async Task TryObfuscateValues_MasksValueTypeSecrets()

var state = logger.Invocations.Single(x => x.Method.Name == nameof(ILogger.Log)).Arguments[2];
var secretObfuscator = new Mock<ISecretObfuscator>();
secretObfuscator.SetupGet(x => x.HasSecrets).Returns(true);
secretObfuscator
.Setup(x => x.Obfuscate(It.IsAny<string?>(), null))
.Returns((string? value, object? _) => value == secret.ToString() ? "********" : value ?? string.Empty);
Expand All @@ -82,6 +157,7 @@ public async Task TryObfuscateValues_MasksCustomStructuredLogStates()
{
var state = new ModuleCompletionLogState("secret", TimeSpan.FromSeconds(1), "(none)", 0, 0);
var secretObfuscator = new Mock<ISecretObfuscator>();
secretObfuscator.SetupGet(x => x.HasSecrets).Returns(true);
secretObfuscator
.Setup(x => x.Obfuscate(It.IsAny<string?>(), null))
.Returns((string? value, object? _) => value == "secret" ? "********" : value ?? string.Empty);
Expand All @@ -98,6 +174,7 @@ public async Task TryObfuscateValues_MasksUnstructuredState()
{
const string secret = "plain-state-secret";
var secretObfuscator = new Mock<ISecretObfuscator>();
secretObfuscator.SetupGet(x => x.HasSecrets).Returns(true);
secretObfuscator
.Setup(x => x.Obfuscate(It.IsAny<string?>(), null))
.Returns((string? value, object? _) =>
Expand All @@ -114,6 +191,7 @@ public async Task TryObfuscateValues_PreservesStateWhenToStringThrows()
{
var state = new ThrowingToStringState();
var secretObfuscator = new Mock<ISecretObfuscator>();
secretObfuscator.SetupGet(x => x.HasSecrets).Returns(true);

var obfuscatedState = new FormattedLogValuesObfuscator(secretObfuscator.Object)
.TryObfuscateValues(state);
Expand All @@ -128,6 +206,29 @@ public async Task TryObfuscateValues_PreservesStateWhenToStringThrows()
public async Task TryObfuscateValues_DoesNotRescanPreObfuscatedValues()
{
var secretObfuscator = new Mock<ISecretObfuscator>();
secretObfuscator.SetupGet(x => x.HasSecrets).Returns(true);
var state = new[]
{
new KeyValuePair<string, object?>(
"CommandOutput",
new PreObfuscatedLogValue("already-masked")),
};

var obfuscatedState = new FormattedLogValuesObfuscator(secretObfuscator.Object)
.TryObfuscateValues(state);
var value = ((IReadOnlyList<KeyValuePair<string, object?>>) obfuscatedState)[0].Value;

await Assert.That(value).IsEqualTo("already-masked");
secretObfuscator.Verify(
x => x.Obfuscate(It.IsAny<string?>(), It.IsAny<object?>()),
Times.Never);
}

[Test]
public async Task TryObfuscateValues_UnwrapsPreObfuscatedValuesWithoutSecrets()
{
var secretObfuscator = new Mock<ISecretObfuscator>();
secretObfuscator.SetupGet(x => x.HasSecrets).Returns(false);
var state = new[]
{
new KeyValuePair<string, object?>(
Expand All @@ -149,4 +250,26 @@ private sealed class ThrowingToStringState
{
public override string ToString() => throw new InvalidOperationException("Cannot format state.");
}

private sealed class CountingToStringState
{
public int ToStringCalls { get; private set; }

public override string ToString()
{
ToStringCalls++;
return "state";
}
}

private static SecretObfuscator CreateBuiltInObfuscator(ISecretProvider? secretProvider = null)
{
secretProvider ??= Mock.Of<ISecretProvider>(provider =>
provider.Version == 0 &&
provider.GetSnapshot() == new SecretSnapshot(0, Array.Empty<string>()));

return new SecretObfuscator(
secretProvider,
Microsoft.Extensions.Options.Options.Create(new SecretMaskingOptions()));
}
}
Loading
Loading