Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ Prerequisites:
* [ActivitySimple](src/ActivitySimple) - Simple workflow that runs simple activities.
* [ActivityWorker](src/ActivityWorker) - Use .NET activities from a workflow in another language.
* [AspNet](src/AspNet) - Demonstration of a generic host worker and an ASP.NET workflow starter.
* [Batch](src/Batch) - Three different best practices for processing a batch of records: an
iterator pattern, a heartbeating activity, and a sliding window of parallel child workflows.
* [Bedrock](src/Bedrock) - Orchestrate a chatbot with Amazon Bedrock.
* [ClientMtls](src/ClientMtls) - How to use client certificate authentication, e.g. for Temporal Cloud.
* [ContextPropagation](src/ContextPropagation) - Context propagation via interceptors.
Expand Down
52 changes: 52 additions & 0 deletions TemporalioSamples.sln
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TemporalioSamples.SampleAppHost", "src\AspireIntegrations\TemporalioSamples.SampleAppHost\TemporalioSamples.SampleAppHost.csproj", "{CA136E75-FC34-44E1-B8B2-6E33D8AF520E}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Temporal.Extensions.Aspire.Hosting", "src\AspireIntegrations\Temporal.Extensions.Aspire.Hosting\Temporal.Extensions.Aspire.Hosting.csproj", "{89D196AD-A6CE-42FB-BF46-C80BF579FE20}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "StandaloneActivity", "StandaloneActivity", "{EAB0C45A-7620-D2D2-2901-5E7FCBFFDA77}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TemporalioSamples.StandaloneActivity", "src\StandaloneActivity\TemporalioSamples.StandaloneActivity.csproj", "{240517A1-13B5-4A67-8519-BFCF2C4591B9}"
Expand All @@ -128,6 +129,17 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "NexusStandaloneOperations",
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TemporalioSamples.NexusStandaloneOperations", "src\NexusStandaloneOperations\TemporalioSamples.NexusStandaloneOperations.csproj", "{6B6622AE-2970-4AAD-B2E4-A7EE1E2C40EA}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Batch", "Batch", "{A95C608E-3127-41B7-BC69-7495768F30C5}"
ProjectSection(SolutionItems) = preProject
src\Batch\README.md = src\Batch\README.md
EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TemporalioSamples.Batch.Iterator", "src\Batch\Iterator\TemporalioSamples.Batch.Iterator.csproj", "{5E600381-C245-44F2-996D-4736BB5A6AC5}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TemporalioSamples.Batch.HeartbeatingActivity", "src\Batch\HeartbeatingActivity\TemporalioSamples.Batch.HeartbeatingActivity.csproj", "{236FCC5A-9E97-4EC4-9E33-E8FBA98E4DA2}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TemporalioSamples.Batch.SlidingWindow", "src\Batch\SlidingWindow\TemporalioSamples.Batch.SlidingWindow.csproj", "{A2D3FA9C-EA09-4AEE-B57C-17FB40025EB1}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -750,6 +762,42 @@ Global
{6B6622AE-2970-4AAD-B2E4-A7EE1E2C40EA}.Release|x64.Build.0 = Release|Any CPU
{6B6622AE-2970-4AAD-B2E4-A7EE1E2C40EA}.Release|x86.ActiveCfg = Release|Any CPU
{6B6622AE-2970-4AAD-B2E4-A7EE1E2C40EA}.Release|x86.Build.0 = Release|Any CPU
{5E600381-C245-44F2-996D-4736BB5A6AC5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5E600381-C245-44F2-996D-4736BB5A6AC5}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5E600381-C245-44F2-996D-4736BB5A6AC5}.Debug|x64.ActiveCfg = Debug|Any CPU
{5E600381-C245-44F2-996D-4736BB5A6AC5}.Debug|x64.Build.0 = Debug|Any CPU
{5E600381-C245-44F2-996D-4736BB5A6AC5}.Debug|x86.ActiveCfg = Debug|Any CPU
{5E600381-C245-44F2-996D-4736BB5A6AC5}.Debug|x86.Build.0 = Debug|Any CPU
{5E600381-C245-44F2-996D-4736BB5A6AC5}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5E600381-C245-44F2-996D-4736BB5A6AC5}.Release|Any CPU.Build.0 = Release|Any CPU
{5E600381-C245-44F2-996D-4736BB5A6AC5}.Release|x64.ActiveCfg = Release|Any CPU
{5E600381-C245-44F2-996D-4736BB5A6AC5}.Release|x64.Build.0 = Release|Any CPU
{5E600381-C245-44F2-996D-4736BB5A6AC5}.Release|x86.ActiveCfg = Release|Any CPU
{5E600381-C245-44F2-996D-4736BB5A6AC5}.Release|x86.Build.0 = Release|Any CPU
{236FCC5A-9E97-4EC4-9E33-E8FBA98E4DA2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{236FCC5A-9E97-4EC4-9E33-E8FBA98E4DA2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{236FCC5A-9E97-4EC4-9E33-E8FBA98E4DA2}.Debug|x64.ActiveCfg = Debug|Any CPU
{236FCC5A-9E97-4EC4-9E33-E8FBA98E4DA2}.Debug|x64.Build.0 = Debug|Any CPU
{236FCC5A-9E97-4EC4-9E33-E8FBA98E4DA2}.Debug|x86.ActiveCfg = Debug|Any CPU
{236FCC5A-9E97-4EC4-9E33-E8FBA98E4DA2}.Debug|x86.Build.0 = Debug|Any CPU
{236FCC5A-9E97-4EC4-9E33-E8FBA98E4DA2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{236FCC5A-9E97-4EC4-9E33-E8FBA98E4DA2}.Release|Any CPU.Build.0 = Release|Any CPU
{236FCC5A-9E97-4EC4-9E33-E8FBA98E4DA2}.Release|x64.ActiveCfg = Release|Any CPU
{236FCC5A-9E97-4EC4-9E33-E8FBA98E4DA2}.Release|x64.Build.0 = Release|Any CPU
{236FCC5A-9E97-4EC4-9E33-E8FBA98E4DA2}.Release|x86.ActiveCfg = Release|Any CPU
{236FCC5A-9E97-4EC4-9E33-E8FBA98E4DA2}.Release|x86.Build.0 = Release|Any CPU
{A2D3FA9C-EA09-4AEE-B57C-17FB40025EB1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A2D3FA9C-EA09-4AEE-B57C-17FB40025EB1}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A2D3FA9C-EA09-4AEE-B57C-17FB40025EB1}.Debug|x64.ActiveCfg = Debug|Any CPU
{A2D3FA9C-EA09-4AEE-B57C-17FB40025EB1}.Debug|x64.Build.0 = Debug|Any CPU
{A2D3FA9C-EA09-4AEE-B57C-17FB40025EB1}.Debug|x86.ActiveCfg = Debug|Any CPU
{A2D3FA9C-EA09-4AEE-B57C-17FB40025EB1}.Debug|x86.Build.0 = Debug|Any CPU
{A2D3FA9C-EA09-4AEE-B57C-17FB40025EB1}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A2D3FA9C-EA09-4AEE-B57C-17FB40025EB1}.Release|Any CPU.Build.0 = Release|Any CPU
{A2D3FA9C-EA09-4AEE-B57C-17FB40025EB1}.Release|x64.ActiveCfg = Release|Any CPU
{A2D3FA9C-EA09-4AEE-B57C-17FB40025EB1}.Release|x64.Build.0 = Release|Any CPU
{A2D3FA9C-EA09-4AEE-B57C-17FB40025EB1}.Release|x86.ActiveCfg = Release|Any CPU
{A2D3FA9C-EA09-4AEE-B57C-17FB40025EB1}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -814,5 +862,9 @@ Global
{5D493692-53AB-4FAA-BA4D-33B1E54E9A48} = {1A647B41-53D0-4638-AE5A-6630BAAE45FC}
{17436B0C-8853-030B-9E2F-BAEB65BE59FB} = {1A647B41-53D0-4638-AE5A-6630BAAE45FC}
{6B6622AE-2970-4AAD-B2E4-A7EE1E2C40EA} = {17436B0C-8853-030B-9E2F-BAEB65BE59FB}
{A95C608E-3127-41B7-BC69-7495768F30C5} = {1A647B41-53D0-4638-AE5A-6630BAAE45FC}
{5E600381-C245-44F2-996D-4736BB5A6AC5} = {A95C608E-3127-41B7-BC69-7495768F30C5}
{236FCC5A-9E97-4EC4-9E33-E8FBA98E4DA2} = {A95C608E-3127-41B7-BC69-7495768F30C5}
{A2D3FA9C-EA09-4AEE-B57C-17FB40025EB1} = {A95C608E-3127-41B7-BC69-7495768F30C5}
EndGlobalSection
EndGlobal
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
using Temporalio.Workflows;

namespace TemporalioSamples.Batch.HeartbeatingActivity;

/// <summary>
/// A sample implementation of processing a batch by a single activity.
///
/// <para>An activity can run for as long as needed. It reports that it is still alive through a
/// heartbeat. If the worker is restarted, the activity is retried after the heartbeat timeout.
/// </para>
/// </summary>
[Workflow]
public class HeartbeatingActivityBatchWorkflow
{
/// <summary>
/// Processes the batch of records.
/// </summary>
/// <returns>Total number of processed records.</returns>
[WorkflowRun]
public async Task<int> RunAsync()
{
// No special logic needed here, as the activity is retried automatically by the service.
//
// The start-to-close timeout is set to a high value to support large batch sizes.
// A heartbeat timeout is required to quickly restart the activity in case of failures,
// and to record heartbeat details at the service.
return await Workflow.ExecuteActivityAsync(
() => RecordProcessorActivities.ProcessRecordsAsync(),
new()
{
StartToCloseTimeout = TimeSpan.FromHours(1),
HeartbeatTimeout = TimeSpan.FromSeconds(10),
});
}
}
65 changes: 65 additions & 0 deletions src/Batch/HeartbeatingActivity/Program.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
using Microsoft.Extensions.Logging;
using Temporalio.Client;
using Temporalio.Common.EnvConfig;
using Temporalio.Worker;
using TemporalioSamples.Batch.HeartbeatingActivity;

const string TaskQueue = "HeartbeatingActivityBatch";

var connectOptions = ClientEnvConfig.LoadClientConnectOptions();
connectOptions.TargetHost ??= "localhost:7233";
connectOptions.LoggerFactory = LoggerFactory.Create(builder =>
builder.AddSimpleConsole(options => options.TimestampFormat = "[HH:mm:ss] ").SetMinimumLevel(LogLevel.Information));
var client = await TemporalClient.ConnectAsync(connectOptions);

async Task RunWorkerAsync()
{
// Cancellation token cancelled on ctrl+c
using var tokenSource = new CancellationTokenSource();
Console.CancelKeyPress += (_, eventArgs) =>
{
tokenSource.Cancel();
eventArgs.Cancel = true;
};

// Run worker until cancelled. Restart it while the batch is executing to see how the
// activity timeout and retry work.
Console.WriteLine("Running worker");
using var worker = new TemporalWorker(
client,
new TemporalWorkerOptions(taskQueue: TaskQueue)
.AddAllActivities(typeof(RecordProcessorActivities), null)
.AddWorkflow<HeartbeatingActivityBatchWorkflow>());
try
{
await worker.ExecuteAsync(tokenSource.Token);
}
catch (OperationCanceledException)
{
Console.WriteLine("Worker cancelled");
}
}

async Task ExecuteWorkflowAsync()
{
var workflowId = "heartbeating-activity-batch-" + Guid.NewGuid();
Console.WriteLine($"Starting batch workflow with id '{workflowId}'.");

var handle = await client.StartWorkflowAsync(
(HeartbeatingActivityBatchWorkflow wf) => wf.RunAsync(),
new(workflowId, TaskQueue));

Console.WriteLine($"Started batch workflow. WorkflowId={handle.Id}, RunId={handle.ResultRunId}");
}

switch (args.ElementAtOrDefault(0))
{
case "worker":
await RunWorkerAsync();
break;
case "workflow":
await ExecuteWorkflowAsync();
break;
default:
throw new ArgumentException("Must pass 'worker' or 'workflow' as the first argument");
}
23 changes: 23 additions & 0 deletions src/Batch/HeartbeatingActivity/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# Heartbeating Activity Batch

A sample implementation of processing a batch by an Activity.

An Activity can run as long as needed. It reports that it is still alive through heartbeat.

If the worker is restarted, the Activity is retried after the heartbeat timeout.

Temporal allows storing data in heartbeat _details_. These details are available to the next
Activity attempt. The progress of the record processing is stored in the details to avoid
reprocessing records from the beginning on failures.

To run, first see [README.md](../../../README.md) for prerequisites. Then, run the following from
this directory in a separate terminal to start the worker. Restart the worker while the batch is
executing to see how the activity timeout and retry work.

dotnet run worker

Then in another terminal, run the workflow from this directory:

dotnet run workflow

This will show logs in the worker window of the workflow running.
78 changes: 78 additions & 0 deletions src/Batch/HeartbeatingActivity/RecordProcessorActivities.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
using Microsoft.Extensions.Logging;
using Temporalio.Activities;

namespace TemporalioSamples.Batch.HeartbeatingActivity;

/// <summary>
/// Activity that processes a whole batch of records.
///
/// <para>It relies on a fake record loader to iterate over the set of records and process them
/// one by one. The heartbeat is used to remember the offset. On activity retry, the data from
/// the last recorded heartbeat is used to minimize the number of records that are reprocessed.
/// Note that not every heartbeat call is sent to the service; the frequency depends on the
/// heartbeat timeout the activity was scheduled with. If no heartbeat timeout is set, no
/// heartbeat is ever sent to the service.</para>
///
/// <para>The biggest advantage of this approach is efficiency: it uses very few Temporal
/// resources. The biggest limitation is that it cannot deal with individual record processing
/// failures. The only options are either failing the whole batch or skipping the record. While
/// it is possible to build additional logic to record failed records somewhere, the experience
/// is not seamless.</para>
/// </summary>
public static class RecordProcessorActivities
{
// The sample always has 1000 records. The real application would iterate over an existing
// dataset or file.
private const int RecordCount = 1000;

/// <summary>
/// Processes all records in the dataset.
/// </summary>
/// <returns>The number of records processed.</returns>
[Activity]
public static async Task<int> ProcessRecordsAsync()
{
var context = ActivityExecutionContext.Current;

// On activity retry, load the last reported offset from the heartbeat details.
var offset = context.Info.HeartbeatDetails.Count > 0
? await context.Info.HeartbeatDetailAtAsync<int>(0)
: 0;
context.Logger.LogInformation("Activity ProcessRecordsAsync started with offset={Offset}", offset);

// This sample implementation processes records one by one. If needed, it can be changed
// to use a pool of tasks to process multiple records in parallel.
while (true)
{
var record = GetRecord(offset);
if (record is null)
{
return offset;
}

await ProcessRecordAsync(record, context.CancellationToken);

// Report that the activity is still alive. The assumption is that each record takes
// less time to process than the heartbeat timeout. Leverage heartbeat details to
// record the offset.
context.Heartbeat(offset);
offset++;
}
}

/// <summary>
/// Returns the record at the given offset, or <c>null</c> if the offset exceeds the dataset
/// size.
/// </summary>
private static SingleRecord? GetRecord(int offset) =>
offset < RecordCount ? new SingleRecord(offset) : null;

/// <summary>
/// Fake record processing logic.
/// </summary>
private static async Task ProcessRecordAsync(SingleRecord record, CancellationToken cancellationToken)
{
await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken);
ActivityExecutionContext.Current.Logger.LogInformation("Processed {Record}", record);
}
}
7 changes: 7 additions & 0 deletions src/Batch/HeartbeatingActivity/SingleRecord.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
namespace TemporalioSamples.Batch.HeartbeatingActivity;

/// <summary>
/// Record to process. A real application would add use-case-specific data.
/// </summary>
/// <param name="Id">Id of the record.</param>
public record SingleRecord(int Id);
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
</PropertyGroup>

</Project>
59 changes: 59 additions & 0 deletions src/Batch/Iterator/IteratorBatchWorkflow.workflow.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
using Temporalio.Workflows;

namespace TemporalioSamples.Batch.Iterator;

/// <summary>
/// Implements the iterator workflow pattern.
///
/// <para>A single workflow run processes a single page of records in parallel. Each record is
/// processed using its own <see cref="RecordProcessorWorkflow"/> child workflow.</para>
///
/// <para>After all child workflows complete, a new run of the parent workflow is created using
/// continue-as-new. The new run processes the next page of records. This way a practically
/// unlimited set of records can be processed.</para>
/// </summary>
[Workflow]
public class IteratorBatchWorkflow
{
/// <summary>
/// Processes the batch of records.
/// </summary>
/// <param name="pageSize">Number of records to process in a single workflow run.</param>
/// <param name="offset">Offset of the first record to process. 0 to start the batch
/// processing.</param>
/// <returns>Total number of processed records.</returns>
[WorkflowRun]
public async Task<int> RunAsync(int pageSize, int offset)
{
// Loads a page of records.
var records = await Workflow.ExecuteActivityAsync(
() => RecordLoaderActivities.GetRecords(pageSize, offset),
new() { StartToCloseTimeout = TimeSpan.FromSeconds(5) });

// Starts a child workflow per record asynchronously.
var results = records.Select(record =>
{
// Uses a human-friendly child id.
var childId = $"{Workflow.Info.WorkflowId}/{record.Id}";
return Workflow.ExecuteChildWorkflowAsync(
(RecordProcessorWorkflow wf) => wf.RunAsync(record),
new() { Id = childId });
}).ToList();

// Waits for all children to complete.
await Workflow.WhenAllAsync(results);

// Skips error handling for the sample's brevity, so failed RecordProcessorWorkflows are
// ignored.

// No more records in the dataset. Completes the workflow.
if (records.Count == 0)
{
return offset;
}

// Continues-as-new with the increased offset.
throw Workflow.CreateContinueAsNewException(
(IteratorBatchWorkflow wf) => wf.RunAsync(pageSize, offset + records.Count));
}
}
Loading