diff --git a/batch.go b/batch.go index 2460505..7ed6152 100644 --- a/batch.go +++ b/batch.go @@ -4,23 +4,22 @@ import ( "time" ) -// Batch takes a stream of items and returns a stream of batches based on a maximum size and a timeout. +// Batch groups consecutive values of the stream into batches. With +// timeout = -1, it accumulates values until the batch reaches size, +// then emits it. When the input is exhausted, any pending values are +// emitted as a final batch. // -// A batch is emitted when one of the following conditions is met: -// - The batch reaches the maximum size -// - The time since the first item was added to the batch exceeds the timeout -// - An error is encountered in the input stream -// - The input stream is closed +// Input errors create batch boundaries: any pending batch is emitted +// first, followed by the error as a separate item. This function never +// emits empty batches. // -// Errors are never included in batches. Each error is forwarded to the output as a separate item, -// preserving the relative order of values and errors. +// A positive timeout adds another trigger: each batch has that much +// time to fill, starting from its first value. When the timeout expires, +// the pending batch is emitted even if it is not full. This trades batch +// size for latency: sparse input produces smaller batches, but no value +// is ever held longer than timeout, assuming there's no backpressure. // -// This function never emits empty batches. To disable the timeout and emit batches only based on the size, -// set the timeout to -1. Setting the timeout to zero is not supported and will result in a panic -// -// This is a non-blocking ordered function that processes items sequentially. -// -// See the package documentation for more information on non-blocking ordered functions and error handling. +// A zero timeout panics. Use a small positive timeout instead. func Batch[A any](in <-chan Try[A], size int, timeout time.Duration) <-chan Try[[]A] { validateMinSize(size, 1) if timeout == 0 { @@ -113,10 +112,8 @@ func Batch[A any](in <-chan Try[A], size int, timeout time.Duration) <-chan Try[ return out } -// Unbatch is the inverse of [Batch]. It takes a stream of batches and returns a stream of individual items. -// -// This is a non-blocking ordered function that processes items sequentially. -// See the package documentation for more information on non-blocking ordered functions and error handling. +// Unbatch flattens a stream of slices into a stream of their values. +// This function is the inverse of [Batch]. func Unbatch[A any](in <-chan Try[[]A]) <-chan Try[A] { if in == nil { return nil diff --git a/benchmark_test.go b/benchmark_test.go index fe56edc..20e52d0 100644 --- a/benchmark_test.go +++ b/benchmark_test.go @@ -10,9 +10,9 @@ import ( ) // The stock B/op and allocs/op columns are the per-item metrics reported by -// benchmarkThroughput, integer divided, so anything under one allocation per -// item reads as zero. We use custom floating point metrics and suppress the -// stock ones - even when -benchmem is passed explicitly. +// benchmarkThroughput, calculated using integer division, so anything under +// one allocation per item reads as zero. We use custom floating-point metrics +// and suppress the stock ones - even when -benchmem is passed explicitly. // This can only be done by mutating the flag. func TestMain(m *testing.M) { flag.Parse() @@ -54,7 +54,7 @@ func benchmarkThroughput(b *testing.B, definePipeline func(in <-chan Try[int])) } // benchmarkThroughputForLevels runs benchmarkThroughput once per concurrency -// level, as a subtest named name/n. +// level as a subtest named name/n. func benchmarkThroughputForLevels(b *testing.B, name string, levels []int, definePipeline func(in <-chan Try[int], n int)) { for _, n := range levels { b.Run(fmt.Sprintf("%s/%d", name, n), func(b *testing.B) { @@ -65,8 +65,8 @@ func benchmarkThroughputForLevels(b *testing.B, name string, levels []int, defin } } -// This benchmark acts as baseline. A single drainer is -// as simple as pipeline can get. +// This benchmark acts as a baseline. A single drainer is +// as simple as a pipeline can get. func BenchmarkDrain(b *testing.B) { benchmarkThroughput(b, func(in <-chan Try[int]) { Drain(in) diff --git a/consume.go b/consume.go index 6ed8172..9fa37f7 100644 --- a/consume.go +++ b/consume.go @@ -5,15 +5,18 @@ import ( "sync/atomic" ) -// ForEach applies a function f to each item in an input stream and returns the first error encountered. +// ForEach calls f on the stream's values, like a concurrent for-range +// loop. It immediately returns the first observed error. Otherwise, it +// returns nil after the input is fully consumed and every call to f has +// returned. // -// This is a blocking unordered function that processes items concurrently using n goroutines. +// The argument n bounds the number of concurrent calls to f. When n = 1, +// ForEach processes items sequentially in stream order, the same as in a +// regular for-range loop: f can safely read and modify shared state without +// synchronization, and all its effects are visible to the caller after +// ForEach returns. // -// When n = 1, ForEach processes items sequentially in stream order, similar to a regular -// for-range loop: f can safely read and modify shared state without synchronization, -// and all its effects are visible to the caller after ForEach returns. -// -// See the package documentation for more information on blocking unordered functions and error handling. +// See the [rill] package documentation for the full contract shared by all sinks. func ForEach[A any](in <-chan Try[A], n int, f func(A) error, options ...SinkOption) error { validateN(n) validateNilFunc(f == nil) @@ -52,10 +55,10 @@ func ForEach[A any](in <-chan Try[A], n int, f func(A) error, options ...SinkOpt return Err(out, options...) } -// Err returns the first error encountered in the input stream or nil if there were no errors. +// Err immediately returns the first error of the stream. Otherwise, it +// returns nil after the input is fully consumed. // -// This is a blocking ordered function that processes items sequentially. -// See the package documentation for more information on blocking ordered functions and error handling. +// See the [rill] package documentation for the full contract shared by all sinks. func Err[A any](in <-chan Try[A], options ...SinkOption) error { defer Discard(in, options...) @@ -68,12 +71,11 @@ func Err[A any](in <-chan Try[A], options ...SinkOption) error { return nil } -// First returns the first value or error encountered in the input stream. -// If the stream is empty or its first item is an error, found is false and -// value is the zero value of A. +// First returns the first item of the stream: (value, true, nil) if +// the item is a value, (zero, false, err) if it is an error, or +// (zero, false, nil) if the stream is empty. // -// This is a blocking ordered function that processes items sequentially. -// See the package documentation for more information on blocking ordered functions and error handling. +// See the [rill] package documentation for the full contract shared by all sinks. func First[A any](in <-chan Try[A], options ...SinkOption) (value A, found bool, err error) { defer Discard(in, options...) @@ -91,13 +93,15 @@ func First[A any](in <-chan Try[A], options ...SinkOption) (value A, found bool, // sharing cannot contaminate across calls. var errFound = errors.New("found") -// Any reports whether the input stream contains an item that satisfies the condition f. -// This function returns true as soon as it finds such an item. Otherwise, it returns false. +// Any reports whether the stream contains a value that matches the +// condition f. It immediately returns (true, nil) or (false, err) on +// the first observed match or error, respectively. Otherwise, it +// returns (false, nil) after the input is fully consumed and every call +// to f has returned. // -// Any is a blocking unordered function that processes items concurrently using n goroutines. -// When n = 1, items are processed sequentially in stream order. +// The argument n bounds the number of concurrent calls to f. // -// See the package documentation for more information on blocking unordered functions and error handling. +// See the [rill] package documentation for the full contract shared by all sinks. func Any[A any](in <-chan Try[A], n int, f func(A) (bool, error), options ...SinkOption) (bool, error) { validateN(n) validateNilFunc(f == nil) @@ -119,14 +123,15 @@ func Any[A any](in <-chan Try[A], n int, f func(A) (bool, error), options ...Sin return false, err } -// All reports whether all items in the input stream satisfy the condition f. -// This function returns false as soon as it finds an item that does not satisfy the condition or encounters an error. -// Otherwise, it returns true. +// All reports whether every value in the stream matches the condition +// f. It immediately returns (false, nil) or (false, err) on the first +// observed mismatch or error, respectively. Otherwise, it returns +// (true, nil) after the input is fully consumed and every call to f has +// returned. // -// All is a blocking unordered function that processes items concurrently using n goroutines. -// When n = 1, items are processed sequentially in stream order. +// The argument n bounds the number of concurrent calls to f. // -// See the package documentation for more information on blocking unordered functions and error handling. +// See the [rill] package documentation for the full contract shared by all sinks. func All[A any](in <-chan Try[A], n int, f func(A) (bool, error), options ...SinkOption) (bool, error) { validateN(n) validateNilFunc(f == nil) diff --git a/doc.go b/doc.go index fc4e768..6befffe 100644 --- a/doc.go +++ b/doc.go @@ -1,80 +1,158 @@ -// Package rill provides composable channel-based concurrency primitives for Go that simplify parallel processing, -// batching, and stream handling. It offers building blocks for constructing concurrent pipelines from -// reusable parts while maintaining precise control over concurrency levels. The package reduces boilerplate, -// abstracts away goroutine orchestration, features centralized error handling, and has zero external dependencies. +// Package rill provides composable concurrency primitives: functions over +// plain channels that transform, filter, batch, reduce, and consume data +// streams while propagating errors and optionally preserving order. // -// # Streams and Try Containers +// Rill is not a framework: its functions can be used on their own or +// composed into multi-stage pipelines. Either way, they are compatible +// with existing channel-based code. // -// In this package, a stream is a channel of [Try] containers. -// Each Try represents either a value or an error. -// When an "empty stream" is referred to, it means a channel of Try containers that has been closed and was never written to. +// # Pipelines and streams // -// Most functions in this package are concurrent, and the level of concurrency can be controlled by the argument n. -// Some functions share common behaviors and characteristics, which are described below. +// The pipeline model in this package is similar to the one described in the +// Go blog's "Pipelines and cancellation" (https://go.dev/blog/pipelines), +// but it unifies error handling by letting errors travel downstream along with +// values. In rill's terms, the post's definition of a pipeline becomes: // -// # Non-blocking functions +// A pipeline is a series of stages connected by streams - channels whose items +// are [Try] structs, each holding either a value or an error. Under the hood, +// each stage runs one or more goroutines that: // -// Functions such as [Map], [Filter], and [Batch] take a stream as an input and return a new stream as an output. -// They do not block and return the output stream immediately. All the processing is done in the background by the goroutine pools they spawn. -// These functions forward all errors from the input stream to the output stream. -// Any errors returned by the user-provided functions are also sent to the output stream. -// When such a function reaches the end of the input stream, it closes the output stream, stops processing and cleans up resources. +// - receive values and errors from upstream via input streams +// - process the received values, usually producing new values or errors +// - send the results downstream via output streams +// - forward upstream errors to the output streams ([Catch] is the only +// exception) // -// Such functions are designed to be composed together to build complex processing pipelines: +// Usually, most stages in a pipeline have one input stream and one output +// stream. The exceptions are the first stage, which has no input stream, and +// the last stage, which has no output stream. These stages are called the +// source and the sink, respectively. The [Merge] and [Tee] functions have more +// inputs/outputs and can be used to build DAG pipelines. // -// stage2 := rill.Map(input, ...) -// stage3 := rill.Batch(stage2, ...) -// stage4 := rill.Map(stage3, ...) -// results := rill.Unbatch(stage4, ...) -// // consume the results and handle errors with some blocking function +// ids := rill.FromSlice(userIDs, nil) // source +// filtered := rill.Filter(ids, 5, ...) // stage, concurrency = 5 +// batches := rill.Batch(filtered, ...) // stage +// transformed := rill.Map(batches, 3, ...) // stage, concurrency = 3 +// err := rill.ForEach(transformed, 2, ...) // sink, concurrency = 2 // -// # Blocking functions +// Intermediate stages return their output streams immediately, while their +// goroutines continue working in the background. These stages always fully +// consume and process their inputs before closing their outputs. A closed +// output becomes an "all upstream work is done" signal that travels +// downstream along with values and errors. // -// Functions such as [ForEach], [Reduce] and [MapReduce] are used at the last stage of the pipeline -// to consume the stream and return the final result or error. +// Sinks are different: they block until their outcome is known, then return +// even if work remains in the pipeline. What "outcome known" means depends on +// the sink. For example: // -// Usually, these functions block until one of the following conditions is met: -// - The end of the stream is reached. In this case, the function returns the final result. -// - An error is encountered either in the input stream or in some user-provided function. In this case, the function returns the error. +// - [ForEach] immediately returns the first error it observes; otherwise, it +// fully consumes the input +// - [Any] can additionally short-circuit on the first match it finds +// - [First] consumes one item and returns // -// In case of an early termination (before reaching the end of the input stream), such functions return immediately -// but spawn a background goroutine that discards the remaining items from the input channel. This is done to prevent goroutine -// leaks by ensuring that all goroutines feeding the stream are allowed to complete. -// The input stream should not be used anymore after calling such functions. +// On an early return, a sink drains and discards any remaining input in the +// background, so upstream stages don't block forever and leak their goroutines. // -// It's also possible to consume the pipeline results manually, for example using a for-range loop. -// In this case, add a deferred call to [Discard] before the loop to ensure that goroutines are not leaked. +// # Context and cancellation // -// defer rill.Discard(results) +// It's up to the caller whether to cancel the extra work that happens +// after an early return. Expensive work and large/infinite sources are usually +// context-aware, so all that's needed is to cancel the context they captured: // -// for res := range results { -// if res.Error != nil { -// return res.Error -// } -// // process res.Value -// } +// ctx, cancel := context.WithCancel(ctx) +// defer cancel() // -// # Unordered functions +// // source and other pipeline stages go here // -// Functions such as [Map], [Filter], and [FlatMap] write items to the output stream as soon as they become available. -// Due to the concurrent nature of these functions, the order of items in the output stream may not match the order of items in the input stream. -// These functions prioritize performance and concurrency over maintaining the original order. +// err := rill.ForEach(transformed, 5, func(x int) error { +// return process(ctx, x) +// }) // -// # Ordered functions +// // outcome known; cancel manually or rely on the deferred cancel +// cancel() // -// Functions such as [OrderedMap] or [OrderedFilter] preserve the order of items from the input stream. -// These functions are still concurrent, but use special synchronization techniques to ensure that -// items are written to the output stream in the same order as they were read from the input stream. -// This additional synchronization has some overhead, but it is negligible for i/o bound workloads. +// # Structured concurrency // -// Some other functions, such as [ToSlice], [Batch] or [First] are not concurrent and are ordered by nature. +// When the caller wants not only to request cancellation but also to wait for +// the pipeline to settle (no work remains and every user callback has +// returned), rill provides the [Scope] API, which is like errgroup for +// pipelines. // -// # Error handling +// scope, ctx := rill.NewScope(ctx) +// defer scope.Cancel() // -// Error handling can be non-trivial in concurrent applications. Rill simplifies this by providing a structured error handling approach. -// As described above, all errors are automatically propagated down the pipeline to the final stage, where they can be caught. -// This allows the pipeline to terminate after the first error is encountered and return it to the caller. +// // source and other pipeline stages go here // -// In cases where more complex error handling logic is required, the [Catch] function can be used. -// It can catch and handle errors at any point in the pipeline, providing the flexibility to handle not only the first error, but any of them. +// err := rill.ForEach(transformed, 5, func(x int) error { +// return process(ctx, x) +// }, scope) +// +// // outcome known +// +// scope.Wait() // cancel ctx and wait for settlement +// +// // it's now safe to release resources and observe side effects +// +// Under the hood, [Scope.Wait] waits for the sink's own work to finish +// and for the "all upstream work is done" signal carried by the sink's +// input streams. +// +// In computation-only pipelines that never fail or short-circuit, everything +// settles by the time the sink returns, so [Scope] is not needed. +// +// # Ordered stages +// +// By default, stages write results to their output streams as soon as they are +// ready, in completion order. In concurrent stages, that order depends on how +// the Go runtime schedules the stages' goroutines and on the time it takes to +// produce each result. +// +// For cases where the input order must be preserved, rill provides ordered +// functions, such as [OrderedMap] or [OrderedFilter]. They stay concurrent, but +// each worker holds its result until all earlier results are sent, so the +// output order matches the input order at the cost of some latency. This +// ordering guarantee holds for both values and errors. +// +// # Backpressure +// +// Backpressure means that sending to an unbuffered channel blocks until the +// receiver on the other end is ready to receive. Rill naturally inherits this +// property: a slow stage in the pipeline blocks the previous stage, and it in +// turn blocks the stage before that, and so on, until the slow stage +// catches up. +// +// When this is not desirable, use [Buffer] to add slack between stages. +// +// # Nil handling +// +// Rill relies on input streams eventually closing for pipelines to finish. +// Nil channels never emit values or close, so passing nil as an input +// can leak goroutines or leave a sink blocked forever. +// +// # Panics +// +// Rill validates the arguments to its functions and panics on misuse, such as +// zero or negative concurrency. Rill does not automatically recover from panics +// in user callbacks: a panicking callback can crash the process, as it would in +// any hand-written concurrent code. +// +// # Extending rill +// +// Almost any custom function that takes or returns streams is compatible +// with rill. For example, it's easy to write a context-aware source that +// streams rows from a database table, or a sink that collects all observed +// errors into a slice. +// +// The easiest way to write a custom stage is to compose it from existing +// functions rill provides. For manually written stages, a few simple rules +// keep background draining and settlement working. Ordinary Go channel +// code usually satisfies most of them: +// +// - sources must eventually close their output stream; a source that can +// run forever must watch a context and be cancellable +// - intermediate stages must close their output stream, but only after the +// input is fully consumed and processed +// - non-concurrent sinks must start with a deferred +// rill.Discard(in, options...), followed by a for-range loop that returns +// as soon as the sink's outcome is known package rill diff --git a/example_test.go b/example_test.go index 40a716b..28178d3 100644 --- a/example_test.go +++ b/example_test.go @@ -19,8 +19,8 @@ import ( // --- Package examples --- -// This example demonstrates a Rill pipeline that fetches users from an API, -// updates their status to active and saves them back. +// This example demonstrates a rill pipeline that fetches users from an API, +// updates their status to active, and saves them back. // Both operations are performed concurrently. // [ForEach] returns on the first error, and context cancellation via defer stops all remaining fetches. func Example() { @@ -58,8 +58,8 @@ func Example() { fmt.Println("Error:", err) } -// This example demonstrates a Rill pipeline that fetches users from an API, -// and updates their status to active and saves them back. +// This example demonstrates a rill pipeline that fetches users from an API, +// updates their status to active, and saves them back. // Users are fetched concurrently and in batches to reduce the number of API calls. func Example_batching() { ctx, cancel := context.WithCancel(context.Background()) @@ -107,12 +107,14 @@ func Example_batching() { // This example demonstrates how batching can be used to group similar concurrent database updates into a single query. // The UpdateUserTimestamp function is used to update the last_active_at column in the users table. Updates are not -// executed immediately, but are rather queued and then sent to the database in batches of up to 5. +// executed immediately but are instead queued and then sent to the database in batches of up to 5. // -// When updates are sparse, it can take some time to collect a full batch. In this case the [Batch] function +// When updates are sparse, it can take some time to collect a full batch. In this case, the [Batch] function // emits partial batches, ensuring that updates are delayed by at most 100ms. // -// For simplicity, this example does not have retries, error handling and synchronization +// For simplicity, this example does not include retries, error handling, or synchronization. +// A more complete version of this pattern, with context support, error handling, and +// synchronization, is described at https://destel.dev/blog/real-time-batching-in-go. func Example_batchingRealTime() { // Start the background worker that processes the updates go updateUserTimestampWorker() @@ -137,13 +139,13 @@ func Example_batchingRealTime() { // This is the queue of user IDs to update. var userIDsToUpdate = make(chan int) -// UpdateUserTimestamp is the public API for updating the last_active_at column in the users table +// UpdateUserTimestamp is the public API for updating the last_active_at column in the users table. func UpdateUserTimestamp(userID int) { userIDsToUpdate <- userID } // This is a background worker that sends queued updates to the database in batches. -// For simplicity, there are no retries, error handling and synchronization +// For simplicity, this worker does not include retries, error handling, or synchronization. func updateUserTimestampWorker() { // convert the channel of user IDs into a stream ids := rill.FromChan(userIDsToUpdate, nil) @@ -164,12 +166,12 @@ func updateUserTimestampWorker() { // hosted online. // // Downloading all files at once would consume too much memory, while processing -// them one-by-one would take too long. And traditional concurrency patterns do not preserve the order of files, +// them one by one would take too long. Traditional concurrency patterns do not preserve the order of files // and would make it challenging to find the first match. // -// The combination of [OrderedFilter] and [First] functions solves the problem, +// The combination of the [OrderedFilter] and [First] functions solves the problem // while downloading and holding in memory at most 5 files at the same time. -// [First] returns on the first match, this triggers the context cancellation via defer, +// [First] returns on the first match; this triggers context cancellation via defer, // stopping URL generation and file downloads. func Example_orderingAndContext() { ctx, cancel := context.WithCancel(context.Background()) @@ -216,9 +218,12 @@ func Example_orderingAndContext() { } } -// This example demonstrates using [FlatMap] to fetch users from multiple departments concurrently. -// Additionally, it demonstrates how to write a reusable streaming wrapper over paginated API calls - the StreamUsers function -func Example_flatMap() { +// This example demonstrates the parallel streaming pattern: [FlatMap] turns each +// department into its own stream of users and merges these streams into one, +// fetching from several departments concurrently. +// Additionally, it demonstrates how to write a reusable streaming wrapper over paginated API calls - +// the StreamUsers function. +func Example_parallelStreaming() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -270,8 +275,8 @@ func StreamUsers(ctx context.Context, query *mockapi.UserQuery) <-chan rill.Try[ } // This example demonstrates how to gracefully stop a pipeline on the first error. -// The CheckAllUsersExist uses several concurrent workers and returns an error as soon as it encounters a non-existent user. -// Such early return triggers the context cancellation, which in turn stops all remaining users fetches. +// The CheckAllUsersExist function uses several concurrent workers and returns an error as soon as it encounters a non-existent user. +// Such an early return triggers context cancellation, which in turn stops all remaining user fetches. func Example_context() { ctx := context.Background() @@ -280,7 +285,7 @@ func Example_context() { fmt.Printf("Check result: %v\n", err) } -// CheckAllUsersExist uses several concurrent workers to check if all users with given IDs exist. +// CheckAllUsersExist uses several concurrent workers to check if all users with the given IDs exist. func CheckAllUsersExist(ctx context.Context, concurrency int, ids []int) error { ctx, cancel := context.WithCancel(ctx) defer cancel() // cancel the remaining requests after the first error @@ -334,7 +339,7 @@ func ExampleAny() { fmt.Println("Error: ", err) } -// Also check out the package level examples to see Batch in action +// See the package-level examples for more realistic uses of Batch. func ExampleBatch() { // Generate a stream of numbers 0 to 49, where a new number is emitted every 50ms numbers := rill.Generate(func(send func(int), sendError func(error)) { @@ -374,7 +379,6 @@ func ExampleCatch() { printStream(ids) } -// The same example as for the [Catch], but using ordered versions of functions. func ExampleOrderedCatch() { // Convert a slice of strings into a stream strs := rill.FromSlice([]string{"1", "2", "3", "4", "5", "not a number 6", "7", "8", "9", "10"}, nil) @@ -436,7 +440,6 @@ func ExampleFilter() { printStream(primes) } -// The same example as for the [Filter], but using ordered versions of functions. func ExampleOrderedFilter() { // Convert a slice of numbers into a stream numbers := rill.FromSlice([]int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil) @@ -467,7 +470,6 @@ func ExampleFilterMap() { printStream(squares) } -// The same example as for the [FilterMap], but using ordered versions of functions. func ExampleOrderedFilterMap() { // Convert a slice of numbers into a stream numbers := rill.FromSlice([]int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil) @@ -521,7 +523,6 @@ func ExampleFlatMap() { printStream(result) } -// The same example as for the [FlatMap], but using ordered versions of functions. func ExampleOrderedFlatMap() { // Convert a slice of numbers into a stream numbers := rill.FromSlice([]int{1, 2, 3, 4, 5}, nil) @@ -557,31 +558,6 @@ func ExampleForEach() { fmt.Println("Error:", err) } -// There is no ordered version of the ForEach function. To achieve ordered processing, use concurrency set to 1. -// If you need a concurrent and ordered ForEach, then do all processing with the [OrderedMap], -// and then use ForEach with concurrency set to 1 at the final stage. -func ExampleForEach_ordered() { - // Convert a slice of numbers into a stream - numbers := rill.FromSlice([]int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil) - - // Square each number - // Concurrency = 3; Ordered - squares := rill.OrderedMap(numbers, 3, func(x int) (int, error) { - return square(x), nil - }) - - // Print results. - // Concurrency = 1; Ordered - err := rill.ForEach(squares, 1, func(y int) error { - fmt.Println(y) - return nil - }) - - // Handle errors - fmt.Println("Error:", err) -} - -// Generate a stream of URLs from https://example.com/file-0.txt to https://example.com/file-9.txt func ExampleGenerate() { urls := rill.Generate(func(send func(string), sendError func(error)) { for i := range 10 { @@ -592,12 +568,11 @@ func ExampleGenerate() { printStream(urls) } -// Generate an infinite stream of natural numbers (1, 2, 3, ...). -// New numbers are sent to the stream every 500ms until the context is canceled func ExampleGenerate_context() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() + // Keep generating numbers until the context is canceled after 5s. numbers := rill.Generate(func(send func(int), sendError func(error)) { for i := 1; ctx.Err() == nil; i++ { send(i) @@ -621,7 +596,6 @@ func ExampleMap() { printStream(squares) } -// The same example as for the [Map], but using ordered versions of functions. func ExampleOrderedMap() { // Convert a slice of numbers into a stream numbers := rill.FromSlice([]int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil) @@ -813,10 +787,6 @@ func ExampleToSeq2() { } } -// This example demonstrates how to wait until the pipeline has no callbacks left to run. -// [ForEach] returns as soon as the error is known, while the source and the remaining -// workers are still going. [Scope.Wait] cancels the scope's Context and blocks -// until all of them have stopped. func ExampleNewScope() { scope, ctx := rill.NewScope(context.Background()) defer scope.Cancel() // extra cancel to make sure the context doesn't leak @@ -868,8 +838,8 @@ func ExampleNewScope() { // --- Helpers --- -// helper function that checks if a number is prime -// and simulates some additional work using sleep +// isPrime checks if a number is prime +// and simulates some additional work by sleeping. func isPrime(n int) bool { simulateWork(500 * time.Millisecond) @@ -884,14 +854,13 @@ func isPrime(n int) bool { return true } -// helper function that squares the number -// and simulates some additional work using sleep +// square returns the square of x and simulates some additional work by sleeping. func square(x int) int { simulateWork(500 * time.Millisecond) return x * x } -// printStream prints all items from a stream (one per line) and an error if any. +// printStream prints all items from a stream (one per line) and an error, if any. func printStream[A any](stream <-chan rill.Try[A]) { fmt.Println("Result:") err := rill.ForEach(stream, 1, func(x A) error { diff --git a/helpers_test.go b/helpers_test.go index 5817a66..f8d685c 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -5,7 +5,7 @@ import ( ) // Item[A] is a comparable variation of Try[A] that's used for testing. -// Stores errors as messages +// It stores errors as strings. type Item[A any] struct { Value A Error string @@ -48,7 +48,7 @@ func toItemSlice[A any](in <-chan Try[A]) []Item[A] { return s } -// Converts a stream in a slice of values and a slice of error messages. +// Converts a stream into a slice of values and a slice of error messages. func toSliceAndErrors[A any](in <-chan Try[A]) ([]A, []string) { var values []A var errors []string diff --git a/internal/core/durable_mutex.go b/internal/core/durable_mutex.go index e4d4c5d..177821d 100644 --- a/internal/core/durable_mutex.go +++ b/internal/core/durable_mutex.go @@ -8,16 +8,16 @@ import ( // DurableMutex is a mutual exclusion lock whose contending callers block // durably when used inside a [testing/synctest] bubble. // -// Most critical sections do plain non-blocking work. For them a stock +// Most critical sections do plain non-blocking work. For them, a stock // [sync.Mutex] is the right choice, with or without synctest. // DurableMutex exists for one specific pattern: critical sections that do -// durably blocking operations while holding the lock -for example, +// durably blocking operations while holding the lock - for example, // atomically receiving values from a channel and appending them to a slice. // -// Under synctest the stock mutex breaks this pattern: while the holder is +// Under synctest, the stock mutex breaks this pattern: while the holder is // durably blocked on a channel, other goroutines are non-durably blocked on the Lock() calls, // so the bubble can neither advance time nor report a deadlock, and the test freezes. -// With DurableMutex the test runs normally, and a genuine deadlock is caught and reported by the framework. +// With DurableMutex, the test runs normally, and a genuine deadlock is caught and reported by the framework. // // DurableMutex does not attempt to prevent starvation. Under contention, // acquisition order depends on runtime scheduling, and the same goroutine @@ -27,7 +27,7 @@ import ( // A DurableMutex used within a synctest bubble is local to that bubble: // every Lock and Unlock call on it must occur in the same bubble. // -// The zero value is ready to use, unlocking an unlocked mutex +// The zero value is ready to use. Unlocking an unlocked mutex // panics, and a DurableMutex must not be copied after first use. type DurableMutex struct { state atomic.Int32 diff --git a/internal/core/loops.go b/internal/core/loops.go index ca63e44..3cd3e8b 100644 --- a/internal/core/loops.go +++ b/internal/core/loops.go @@ -4,8 +4,8 @@ import ( "sync" ) -// Loop allows to process items from the input channel concurrently using n goroutines. -// If done channel is not nil, it will be closed after all items are processed. +// Loop processes items from the input channel concurrently using n goroutines. +// If the done channel is not nil, it will be closed after all items are processed. func Loop[A, B any](in <-chan A, done chan<- B, n int, f func(A)) { if n == 1 { go func() { @@ -38,13 +38,17 @@ func Loop[A, B any](in <-chan A, done chan<- B, n int, f func(A)) { } } -// OrderedLoop is similar to Loop, but it allows to write results to some channel in the same order as items were read from the input. -// If done channel is not nil, it will be closed after all items are processed. -// Special "canWrite" channel is passed to user's function f. Typical f function looks like this: -// - Do some processing (this part is executed concurrently). -// - Read from canWrite channel exactly once. This step is required. Otherwise, behavior is undefined. -// - Write result of the processing somewhere. This step is optional. -// This way processing is done concurrently, but results are written in order. +// OrderedLoop is similar to Loop, but it allows the caller to write results to a +// channel in the same order as the items were read from the input. +// If the done channel is not nil, it will be closed after all items are processed. +// A special "canWrite" channel is passed to the user's function f. +// A typical implementation of f performs the following steps: +// +// - Do some processing (this part is executed concurrently). +// - Read from the canWrite channel exactly once. This step is required. Otherwise, behavior is undefined. +// - Write the result of the processing somewhere. This step is optional. +// +// This way, processing is done concurrently, but results are written in order. func OrderedLoop[A, B any](in <-chan A, done chan<- B, n int, f func(a A, canWrite <-chan struct{})) { if n == 1 { canWrite := make(chan struct{}, 1) diff --git a/internal/core/pool.go b/internal/core/pool.go index 9e2308d..49831aa 100644 --- a/internal/core/pool.go +++ b/internal/core/pool.go @@ -2,10 +2,10 @@ package core import "sync" -// Pool is a pool of reusable values. It grows on demand and never shrinks, -// it's caller's responsibility to ensure that the pool stays bounded. +// Pool is a pool of reusable values. It grows on demand and never shrinks; +// it's the caller's responsibility to ensure that the pool stays bounded. // -// A Pool is safe for concurrent use, unless Unsynchronized is set. Its fields +// A Pool is safe for concurrent use unless Unsynchronized is set. Its fields // must not be changed once the pool is in use. type Pool[T any] struct { // New creates a value when the pool is empty. It is required. @@ -23,7 +23,7 @@ type Pool[T any] struct { items []T } -// Get returns a value from the pool, or a new one if the pool is empty. +// Get returns a value from the pool or a new one if the pool is empty. func (p *Pool[T]) Get() T { if !p.Unsynchronized { p.mu.Lock() diff --git a/internal/list/list.go b/internal/list/list.go index 6796adc..7d4cbeb 100644 --- a/internal/list/list.go +++ b/internal/list/list.go @@ -11,7 +11,7 @@ type Node[T any] struct { list *List[T] } -// Next returns the next node, or nil if n is the last one or detached. +// Next returns the next node or nil if n is the last one or is detached. func (n *Node[T]) Next() *Node[T] { if l := n.list; l != nil && n.next != &l.root { return n.next @@ -19,7 +19,7 @@ func (n *Node[T]) Next() *Node[T] { return nil } -// Prev returns the previous node, or nil if n is the first one or detached. +// Prev returns the previous node or nil if n is the first one or is detached. func (n *Node[T]) Prev() *Node[T] { if l := n.list; l != nil && n.prev != &l.root { return n.prev @@ -54,7 +54,7 @@ func New[T any]() *List[T] { return l } -// Front returns the first node, or nil if the list is empty. +// Front returns the first node or nil if the list is empty. func (l *List[T]) Front() *Node[T] { if n := l.root.next; n != &l.root { return n @@ -62,7 +62,7 @@ func (l *List[T]) Front() *Node[T] { return nil } -// Back returns the last node, or nil if the list is empty. +// Back returns the last node or nil if the list is empty. func (l *List[T]) Back() *Node[T] { if n := l.root.prev; n != &l.root { return n @@ -77,21 +77,21 @@ func (l *List[T]) PushBack(n *Node[T]) { l.insertAfter(n, l.root.prev) } -// PushFront inserts n at the front of the list. Same contract for n as -// [List.PushBack]. +// PushFront inserts n at the front of the list. It has the same contract +// for n as [List.PushBack]. func (l *List[T]) PushFront(n *Node[T]) { l.insertAfter(n, &l.root) } -// InsertAfter inserts n after mark, which must be a node of l. Same contract -// for n as [List.PushBack]. +// InsertAfter inserts n after mark, which must be a node of l. It has the same +// contract for n as [List.PushBack]. func (l *List[T]) InsertAfter(n, mark *Node[T]) { if mark.list == l { l.insertAfter(n, mark) } } -// InsertBefore inserts n before mark. Same contract as [List.InsertAfter]. +// InsertBefore inserts n before mark. It has the same contract as [List.InsertAfter]. func (l *List[T]) InsertBefore(n, mark *Node[T]) { if mark.list == l { l.insertAfter(n, mark.prev) diff --git a/internal/th/assertions.go b/internal/th/assertions.go index 428660c..eef7a61 100644 --- a/internal/th/assertions.go +++ b/internal/th/assertions.go @@ -175,10 +175,10 @@ func ExpectLeak(t *testing.T, f func(t *testing.T)) { } } -// outcomes: +// Outcomes: // 0 - nothing blocked -// 1 - some goroutines blocked, but not main -// 2 - main blocked and possibly some goroutines also blocked +// 1 - some goroutines blocked, but not the main goroutine +// 2 - the main goroutine blocked, possibly along with other goroutines func checkBlock(t *testing.T, f func(t *testing.T)) (outcome int) { mainBlocked := true diff --git a/internal/th/helpers.go b/internal/th/helpers.go index ee4a076..e9f29ce 100644 --- a/internal/th/helpers.go +++ b/internal/th/helpers.go @@ -46,14 +46,14 @@ func DontClose[A any](in <-chan A) <-chan A { } // ExpectNoRace is a semantic name for a bare unsynchronized read. -// Tests sometimes need to do an unsynchronized access to a variable, to +// Tests sometimes need to perform an unsynchronized read of a variable to // have the race detector confirm that all writes in other goroutines // happen before this read. It's enough to do a no-op read: // // _ = myVariable // -// This works, but requires an explaining comment at every site. -// ExpectNoRace is also a no-op, but makes the call site clearer: +// This works but requires an explanatory comment at every site. +// ExpectNoRace is also a no-op but makes the call site clearer: // // th.ExpectNoRace(myVariable) // @@ -64,8 +64,8 @@ func ExpectNoRace[T any](value T) { } // DelayEach forwards items, sleeping for the given duration before each one. -// Under synctest this makes it impossible to consume the channel in zero fake -// time, hence one goroutine (main) can observe the intermediate state of another +// Under synctest, this makes it impossible to consume the channel in zero fake +// time, so one goroutine (main) can observe the intermediate state of another // goroutine (drain) consuming the stream. This function is usually // paired with [ExpectOpenChan]. // @@ -126,7 +126,7 @@ func TestLevels(t *testing.T, levels []int, f func(t *testing.T, n int)) { } // RunSynctest runs a subtest in a synctest bubble. -// It panics if any unless all goroutines started from f exit cleanly. +// It panics unless all goroutines started from f exit cleanly. func RunSynctest(t *testing.T, name string, f func(t *testing.T)) { t.Run(name, func(t *testing.T) { synctest.Test(t, f) diff --git a/iter.go b/iter.go index 60c2644..0fcb21f 100644 --- a/iter.go +++ b/iter.go @@ -5,11 +5,14 @@ import ( "sync" ) -// FromSeq converts an iterator into a stream. -// If err is not nil, the function ignores the passed seq and returns a stream with a single error. +// FromSeq converts an iterator into a stream. If err is not nil, +// FromSeq returns a stream with only that error and ignores seq. +// Otherwise, the values of seq are forwarded to the output, and the +// output is closed once seq ends. // -// Such function signature allows concise wrapping of functions that return an -// iterator and an error: +// This signature allows concise wrapping of functions that return an +// iterator and an error. FromSeq assumes a non-nil error means +// someFunc() could not construct the iterator. // // stream := rill.FromSeq(someFunc()) func FromSeq[A any](seq iter.Seq[A], err error) <-chan Try[A] { @@ -33,7 +36,9 @@ func FromSeq[A any](seq iter.Seq[A], err error) <-chan Try[A] { } // FromSeq2 converts an iterator of value-error pairs into a stream. -// For pairs with a non-nil error, FromSeq2 emits an error item and ignores the value. +// Each pair becomes one item. For pairs with a non-nil error, FromSeq2 +// emits an error item and ignores the value. The output is closed once +// seq ends. func FromSeq2[A any](seq iter.Seq2[A, error]) <-chan Try[A] { validateNilFunc(seq == nil) @@ -47,13 +52,15 @@ func FromSeq2[A any](seq iter.Seq2[A, error]) <-chan Try[A] { return out } -// ToSeq2 converts an input stream into an iterator of value-error pairs. -// Errors are yielded as ordinary pairs and do not stop the iteration; -// handle them inside the loop. +// ToSeq2 converts the stream into an iterator of value-error pairs, +// typically consumed with a for-range loop. Pairs are yielded until the +// stream is exhausted or the loop exits with break or return. Error +// items do not stop the iteration: they are yielded as ordinary pairs. // -// ToSeq2 returns a single-use iterator, which must be called for the pipeline -// to settle. If iteration stops early using break or return, ToSeq2 drains the -// remaining input in the background. +// The returned iterator is single-use and must be ranged over for the +// pipeline to settle. If the loop exits early with break or return, +// ToSeq2 drains the input in the background before reporting settlement +// via a [Scope]. func ToSeq2[A any](in <-chan Try[A], options ...SinkOption) iter.Seq2[A, error] { // Unlike other sinks, ToSeq2 opens the options at the call site: its work // happens while the iterator is ranged, which can be arbitrarily far from diff --git a/merge.go b/merge.go index bb3df0c..84cbd39 100644 --- a/merge.go +++ b/merge.go @@ -4,45 +4,47 @@ import ( "github.com/destel/rill/internal/core" ) -// Merge performs a fan-in operation on the list of input channels, returning a single output channel. -// The resulting channel will contain all items from all inputs, -// and will be closed when all inputs are exhausted. -// In particular, Merge with no arguments returns an already closed channel. +// Merge performs a fan-in: it combines multiple input channels into one output +// channel. It reads all inputs concurrently, so a slow input does not delay +// forwarding items from the others. Items are interleaved as they arrive, +// preserving the relative order of items from each input. // -// This is a non-blocking function that processes items from each input sequentially. -// -// See the package documentation for more information on non-blocking functions and error handling. +// The output is closed only when all inputs are exhausted. A nil input is +// never exhausted, so the output never closes. Merge with no arguments +// returns an empty closed channel. func Merge[A any](ins ...<-chan A) <-chan A { return core.Merge(ins...) } -// Split2 divides the input stream into two output streams based on the predicate function f: -// The splitting behavior is determined by the boolean return value of f. When f returns true, the item is sent to the outTrue stream, -// otherwise it is sent to the outFalse stream. In case of any error, the item is sent to both output streams. -// Both output streams must be consumed independently to avoid deadlocks. +// Split2 divides the stream into two streams: values that match the +// condition f go to outTrue, and the rest go to outFalse. Errors are +// sent to both outputs. // -// This is a non-blocking unordered function that processes items concurrently using n goroutines. -// An ordered version of this function, [OrderedSplit2], is also available. +// The streams must be consumed concurrently to avoid a deadlock. // -// See the package documentation for more information on non-blocking unordered functions and error handling. +// The argument n bounds the number of concurrent calls to f. Items are +// written to the outputs as they become ready, so their order can +// differ from the input order when n > 1. Use [OrderedSplit2] to +// preserve the order. // // Deprecated: Split2 will be removed in v1.0. Since the introduction of [Tee] -// in v0.8, splitting no longer needs a dedicated operation — it can be composed +// in v0.8, splitting no longer needs a dedicated operation - it can be composed // from existing ones. Unlike Split2, the composition is also not limited to two // branches. // -// Quite often the predicate is a simple pure check (field comparison, type -// switch, etc). In such cases splitting is just [Tee] plus a [Filter] on each -// branch: +// Quite often, the predicate is a simple, pure check (a field comparison, a +// type switch, etc.). In such cases, splitting is just [Tee] plus a [Filter] on +// each branch: // // adults, minors := rill.Tee(users) // adults = rill.Filter(adults, 1, func(u User) (bool, error) { return u.Age >= 18, nil }) // minors = rill.Filter(minors, 1, func(u User) (bool, error) { return u.Age < 18, nil }) // -// If the predicate is expensive, stateful, or can fail, it must be evaluated -// once per item, before [Tee]: inline this function's implementation, which -// tags each item with the decision and routes on the tag. The same pattern -// extends to n-way splitting by tagging with an index or key instead of a bool. +// If the predicate is expensive or stateful, or if it can fail, it must be +// evaluated once per item, before [Tee]: inline this function's implementation, +// which tags each item with the decision and routes on the tag. The same +// pattern extends to n-way splitting by tagging with an index or key instead of +// a bool. func Split2[A any](in <-chan Try[A], n int, f func(A) (bool, error)) (outTrue <-chan Try[A], outFalse <-chan Try[A]) { validateN(n) validateNilFunc(f == nil) @@ -64,25 +66,27 @@ func Split2[A any](in <-chan Try[A], n int, f func(A) (bool, error)) (outTrue <- return } -// OrderedSplit2 is the ordered version of [Split2]. +// OrderedSplit2 is the ordered version of [Split2]: the outputs +// preserve the input order for values and errors alike. // // Deprecated: OrderedSplit2 will be removed in v1.0. Since the introduction of -// [Tee] in v0.8, splitting no longer needs a dedicated operation — it can be +// [Tee] in v0.8, splitting no longer needs a dedicated operation - it can be // composed from existing ones. Unlike OrderedSplit2, the composition is also // not limited to two branches. // -// Quite often the predicate is a simple pure check (field comparison, type -// switch, etc). In such cases splitting is just [Tee] plus an [OrderedFilter] -// on each branch: +// Quite often, the predicate is a simple, pure check (a field comparison, a +// type switch, etc.). In such cases, splitting is just [Tee] plus an +// [OrderedFilter] on each branch: // // adults, minors := rill.Tee(users) // adults = rill.OrderedFilter(adults, 1, func(u User) (bool, error) { return u.Age >= 18, nil }) // minors = rill.OrderedFilter(minors, 1, func(u User) (bool, error) { return u.Age < 18, nil }) // -// If the predicate is expensive, stateful, or can fail, it must be evaluated -// once per item, before [Tee]: inline this function's implementation, which -// tags each item with the decision and routes on the tag. The same pattern -// extends to n-way splitting by tagging with an index or key instead of a bool. +// If the predicate is expensive or stateful, or if it can fail, it must be +// evaluated once per item, before [Tee]: inline this function's implementation, +// which tags each item with the decision and routes on the tag. The same +// pattern extends to n-way splitting by tagging with an index or key instead of +// a bool. func OrderedSplit2[A any](in <-chan Try[A], n int, f func(A) (bool, error)) (outTrue <-chan Try[A], outFalse <-chan Try[A]) { validateN(n) validateNilFunc(f == nil) @@ -106,9 +110,8 @@ func OrderedSplit2[A any](in <-chan Try[A], n int, f func(A) (bool, error)) (out // Tee duplicates the input: it returns two channels that both carry every // item from the input, forwarded as it arrives. Both outputs are closed -// once the input is exhausted. -// -// The outputs must be consumed concurrently to avoid a deadlock. +// once the input is exhausted. They must be consumed concurrently to avoid +// a deadlock. // // If deep copying of values is needed, use [Map] on one or both // outputs: diff --git a/merge_test.go b/merge_test.go index 944cedd..00ae1bf 100644 --- a/merge_test.go +++ b/merge_test.go @@ -10,7 +10,7 @@ import ( "github.com/destel/rill/internal/th" ) -// Full behavior of Merge is tested in the internal/core package. +// The full behavior of Merge is tested in the internal/core package. // This test only pins the wrapper wiring. func TestMerge(t *testing.T) { synctest.Test(t, func(t *testing.T) { diff --git a/mockapi/files.go b/mockapi/files.go index 17a60c4..548f583 100644 --- a/mockapi/files.go +++ b/mockapi/files.go @@ -6,7 +6,7 @@ import ( "time" ) -// DownloadFile simulates a file download. It returns the whole content as []byte. +// DownloadFile simulates a file download. It returns the entire contents as a byte slice. func DownloadFile(ctx context.Context, url string) ([]byte, error) { if err := simulateWork(ctx, 1000*time.Millisecond); err != nil { return nil, err diff --git a/mockapi/users.go b/mockapi/users.go index 0d26b98..59465ae 100644 --- a/mockapi/users.go +++ b/mockapi/users.go @@ -1,6 +1,6 @@ // Package mockapi provides a very basic mock API for examples and demos. // It's intentionally kept public to enable running and experimenting with examples in the Go Playground. -// The implementation is naive and uses full scan for all operations. +// The implementation is naive and uses a full scan for all operations. package mockapi import ( @@ -79,7 +79,7 @@ func GetUser(ctx context.Context, id int) (*User, error) { return &user, nil } -// GetUsers returns a list of users by IDs. +// GetUsers returns a list of users by their IDs. // If a user is not found, nil is returned in the corresponding position. func GetUsers(ctx context.Context, ids []int) ([]*User, error) { if err := simulateWork(ctx, 1000*time.Millisecond); err != nil { diff --git a/options.go b/options.go index b89bfa6..866490a 100644 --- a/options.go +++ b/options.go @@ -10,8 +10,8 @@ func (o sinkOptions) settle() { } } -// A SinkOption is an optional argument accepted by every sink, such as a -// [Scope]. The interface cannot be implemented outside this package. +// A SinkOption is an optional argument accepted by every sink. +// [Scope] implements this interface. type SinkOption interface { apply(options *sinkOptions) } diff --git a/reduce.go b/reduce.go index e13175b..4dd00b5 100644 --- a/reduce.go +++ b/reduce.go @@ -8,20 +8,24 @@ import ( "github.com/destel/rill/internal/list" ) -// Reduce combines all items from the input stream into a single value using a binary function f. +// Reduce combines all values of the stream into a single value using +// the binary function f. // -// Treating f as a binary operator "*", Reduce computes in[0] * in[1] * ... * in[N-1]: -// items are combined in stream order, but the parenthesization is unspecified -// and may vary from run to run. This requires f to be associative - -// (a * b) * c == a * (b * c) - so that every parenthesization yields the same -// result. Commutativity is not required. +// Treating f as a binary operator "*", Reduce computes +// in[0] * in[1] * ... * in[N-1]: values are combined in stream order, +// but the parenthesization is unspecified and may vary from run to run. +// This requires f to be associative - (a * b) * c == a * (b * c) - so +// that every parenthesization yields the same result. Commutativity is +// not required. // -// The hasResult return flag is set to true if the stream contained at least one value and no error was encountered, -// otherwise it is set to false. +// Reduce immediately returns (zero, false, err) on the first observed +// error. Otherwise, it returns (zero, false, nil) if the stream is +// empty, or (result, true, nil) after the input is fully consumed and +// every call to f has returned. // -// Reduce is a blocking function that processes items concurrently using n goroutines. +// The argument n bounds the number of concurrent calls to f. // -// See the package documentation for more information on blocking functions and error handling. +// See the [rill] package documentation for the full contract shared by all sinks. func Reduce[A any](in <-chan Try[A], n int, f func(A, A) (A, error), options ...SinkOption) (result A, hasResult bool, err error) { validateN(n) validateNilFunc(f == nil) @@ -246,21 +250,22 @@ func Reduce[A any](in <-chan Try[A], n int, f func(A, A) (A, error), options ... return First(out, options...) } -// MapReduce transforms the input stream into a Go map using mapper and reducer functions. -// The transformation is performed in two concurrent phases. +// MapReduce builds a map from the stream: mapper turns each value into +// a key-value pair, and reducer combines the values that share a key. // -// - The mapper function transforms each input item into a key-value pair. -// - The reducer function reduces values of the same key into a single value. -// This phase has the same semantics as the [Reduce] function: for each key, -// values are combined in stream order, but the parenthesization is unspecified, -// so the reducer must be associative. +// For each key, the values are combined as in [Reduce]: in stream +// order, with unspecified parenthesization, so reducer must be +// associative. Commutativity is not required. // -// An empty input stream produces an empty map. +// MapReduce immediately returns (nil, err) on the first observed +// error. Otherwise, it returns the map after the input is fully +// consumed and every call to mapper and reducer has returned. An empty +// stream results in an empty map. // -// MapReduce is a blocking function that processes items concurrently using nm and nr goroutines -// for the mapper and reducer functions respectively. +// The arguments nm and nr bound the number of concurrent calls to +// mapper and reducer, respectively. // -// See the package documentation for more information on blocking functions and error handling. +// See the [rill] package documentation for the full contract shared by all sinks. func MapReduce[A any, K comparable, V any](in <-chan Try[A], nm int, mapper func(A) (K, V, error), nr int, reducer func(V, V) (V, error), options ...SinkOption) (map[K]V, error) { validateN(nm) validateNilFunc(mapper == nil) diff --git a/scope.go b/scope.go index 59f8c3e..70a6ced 100644 --- a/scope.go +++ b/scope.go @@ -8,8 +8,7 @@ import ( // A Scope tracks the lifecycle of a pipeline and lets the caller wait until // all of its work is done - including any work that happens after a sink's // early return. -// -// A scope passed to a sink as a [SinkOption] tracks not only that sink, but +// A scope passed to a sink as a [SinkOption] tracks not only that sink but // also the whole pipeline behind it. // // For branching pipelines (see [Tee]), multiple sinks can be attached to the diff --git a/transform.go b/transform.go index 68186cc..7c79f85 100644 --- a/transform.go +++ b/transform.go @@ -4,13 +4,13 @@ import ( "github.com/destel/rill/internal/core" ) -// Map takes a stream of items of type A and transforms them into items of type B using a function f. -// Returns a new stream of transformed items. +// Map takes a stream of values of type A and returns a stream of values of +// type B, using f to transform each. When f returns an error, it's written +// to the output instead of a value. // -// This is a non-blocking unordered function that processes items concurrently using n goroutines. -// An ordered version of this function, [OrderedMap], is also available. -// -// See the package documentation for more information on non-blocking unordered functions and error handling. +// The argument n bounds the number of concurrent calls to f. +// Results and errors are written to the output in completion order. +// Use [OrderedMap] to preserve the input order. func Map[A, B any](in <-chan Try[A], n int, f func(A) (B, error)) <-chan Try[B] { validateN(n) validateNilFunc(f == nil) @@ -29,7 +29,8 @@ func Map[A, B any](in <-chan Try[A], n int, f func(A) (B, error)) <-chan Try[B] }) } -// OrderedMap is the ordered version of [Map]. +// OrderedMap is the ordered version of [Map]: +// it writes results and errors in input order rather than completion order. func OrderedMap[A, B any](in <-chan Try[A], n int, f func(A) (B, error)) <-chan Try[B] { validateN(n) validateNilFunc(f == nil) @@ -48,13 +49,14 @@ func OrderedMap[A, B any](in <-chan Try[A], n int, f func(A) (B, error)) <-chan }) } -// Filter takes a stream of items of type A and filters them using a predicate function f. -// Returns a new stream of items that passed the filter. -// -// This is a non-blocking unordered function that processes items concurrently using n goroutines. -// An ordered version of this function, [OrderedFilter], is also available. +// Filter takes a stream of values and returns a new stream, keeping +// only the values that match the condition f. When f returns an error, +// it's written to the output instead of the value. +// Errors are never filtered out. // -// See the package documentation for more information on non-blocking unordered functions and error handling. +// The argument n bounds the number of concurrent calls to f. +// Results and errors are written to the output in completion order. +// Use [OrderedFilter] to preserve the input order. func Filter[A any](in <-chan Try[A], n int, f func(A) (bool, error)) <-chan Try[A] { validateN(n) validateNilFunc(f == nil) @@ -73,7 +75,8 @@ func Filter[A any](in <-chan Try[A], n int, f func(A) (bool, error)) <-chan Try[ }) } -// OrderedFilter is the ordered version of [Filter]. +// OrderedFilter is the ordered version of [Filter]: +// it writes results and errors in input order rather than completion order. func OrderedFilter[A any](in <-chan Try[A], n int, f func(A) (bool, error)) <-chan Try[A] { validateN(n) validateNilFunc(f == nil) @@ -92,14 +95,14 @@ func OrderedFilter[A any](in <-chan Try[A], n int, f func(A) (bool, error)) <-ch }) } -// FilterMap takes a stream of items of type A, applies a function f that can filter and transform them into items of type B. -// Returns a new stream of transformed items that passed the filter. This operation is equivalent to a -// [Filter] followed by a [Map]. +// FilterMap takes a stream of values of type A and returns a stream of +// values of type B, using f to transform each value and decide whether +// to keep the result. When f returns an error, it's written to the +// output instead of a value. Errors are never filtered out. // -// This is a non-blocking unordered function that processes items concurrently using n goroutines. -// An ordered version of this function, [OrderedFilterMap], is also available. -// -// See the package documentation for more information on non-blocking unordered functions and error handling. +// The argument n bounds the number of concurrent calls to f. +// Results and errors are written to the output in completion order. +// Use [OrderedFilterMap] to preserve the input order. func FilterMap[A, B any](in <-chan Try[A], n int, f func(A) (B, bool, error)) <-chan Try[B] { validateN(n) validateNilFunc(f == nil) @@ -118,7 +121,8 @@ func FilterMap[A, B any](in <-chan Try[A], n int, f func(A) (B, bool, error)) <- }) } -// OrderedFilterMap is the ordered version of [FilterMap]. +// OrderedFilterMap is the ordered version of [FilterMap]: +// it writes results and errors in input order rather than completion order. func OrderedFilterMap[A, B any](in <-chan Try[A], n int, f func(A) (B, bool, error)) <-chan Try[B] { validateN(n) validateNilFunc(f == nil) @@ -137,13 +141,16 @@ func OrderedFilterMap[A, B any](in <-chan Try[A], n int, f func(A) (B, bool, err }) } -// FlatMap takes a stream of items of type A and transforms each item into a new sub-stream of items of type B using a function f. -// Those sub-streams are then flattened into a single output stream, which is returned. -// -// This is a non-blocking unordered function that processes items concurrently using n goroutines. -// An ordered version of this function, [OrderedFlatMap], is also available. +// FlatMap takes a stream of values of type A and returns a stream of +// values of type B, using f to expand each value into its own sub-stream. +// The sub-streams are flattened into the output: every item is forwarded, +// values and errors alike. // -// See the package documentation for more information on non-blocking unordered functions and error handling. +// The argument n bounds the number of sub-streams consumed concurrently: +// each worker consumes one sub-stream to the end before starting the next. +// When n > 1, items from different sub-streams can interleave in the +// output. Use [OrderedFlatMap] to concatenate the sub-streams in the +// input order. func FlatMap[A, B any](in <-chan Try[A], n int, f func(A) <-chan Try[B]) <-chan Try[B] { validateN(n) validateNilFunc(f == nil) @@ -169,7 +176,41 @@ func FlatMap[A, B any](in <-chan Try[A], n int, f func(A) <-chan Try[B]) <-chan return out } -// OrderedFlatMap is the ordered version of [FlatMap]. +// OrderedFlatMap is the ordered version of [FlatMap]: the output consists +// of the sub-streams concatenated in the input order. +// +// The argument n bounds the number of concurrent calls to f. The +// sub-streams are prepared concurrently but - unlike in [FlatMap] - +// are consumed one at a time and in order: nothing reads from a sub-stream +// before its turn. In practice, to keep the stage concurrent, a +// sub-stream must do all or part of its expensive work ahead of its +// turn. +// +// Consider a stream of URLs: each file should be downloaded, and its +// lines should be streamed to the output, all in order. Downloading is the +// expensive work here. +// +// Example 1: f downloads the whole file into memory and then streams the +// lines from there. Up to 5 downloads run concurrently. +// +// rill.OrderedFlatMap(urls, 5, func(u string) <-chan rill.Try[string] { +// lines, err := getFileLines(u) +// return rill.FromSlice(lines, err) +// }) +// +// Example 2: as the file is being downloaded, f streams the lines +// through a [Buffer] that lets the sub-stream run ahead of its turn. +// Again, up to 5 downloads run concurrently, but a download that runs +// ahead of its turn pauses after its first 100 lines until the turn comes. +// +// rill.OrderedFlatMap(urls, 5, func(u string) <-chan rill.Try[string] { +// lines := streamFileLines(u) +// return rill.Buffer(lines, 100) +// }) +// +// The two examples do the same thing: they buffer the lines, with or +// without a bound. Without any buffering, the downloads would run one +// at a time, and the stage would become sequential. func OrderedFlatMap[A, B any](in <-chan Try[A], n int, f func(A) <-chan Try[B]) <-chan Try[B] { validateN(n) validateNilFunc(f == nil) @@ -197,17 +238,14 @@ func OrderedFlatMap[A, B any](in <-chan Try[A], n int, f func(A) <-chan Try[B]) return out } -// Catch allows handling errors in the middle of a stream processing pipeline. -// Every error encountered in the input stream is passed to the function f for handling. -// -// The outcome depends on the return value of f: -// - If f returns nil, the error is considered handled and filtered out from the output stream. -// - If f returns a non-nil error, the original error is replaced with the result of f. -// -// This is a non-blocking unordered function that handles errors concurrently using n goroutines. -// An ordered version of this function, [OrderedCatch], is also available. +// Catch takes a stream and returns a new stream with the errors +// optionally handled by f. Each error is passed to f, which returns nil +// to drop it from the stream, the same error to keep it, or a different +// one to replace it. Values never reach f and are passed through as-is. // -// See the package documentation for more information on non-blocking unordered functions and error handling. +// The argument n bounds the number of concurrent calls to f. +// Items are written to the output in completion order. +// Use [OrderedCatch] to preserve the input order. func Catch[A any](in <-chan Try[A], n int, f func(error) error) <-chan Try[A] { validateN(n) validateNilFunc(f == nil) @@ -226,7 +264,8 @@ func Catch[A any](in <-chan Try[A], n int, f func(error) error) <-chan Try[A] { }) } -// OrderedCatch is the ordered version of [Catch]. +// OrderedCatch is the ordered version of [Catch]: +// it writes items in input order rather than completion order. func OrderedCatch[A any](in <-chan Try[A], n int, f func(error) error) <-chan Try[A] { validateN(n) validateNilFunc(f == nil) diff --git a/util.go b/util.go index 1b65b6c..0a0a568 100644 --- a/util.go +++ b/util.go @@ -6,12 +6,15 @@ import ( "github.com/destel/rill/internal/core" ) -// Drain consumes and discards all items from an input channel, blocking until the channel is exhausted. +// Drain consumes and discards all items of the channel, blocking until +// it is exhausted. func Drain[A any](in <-chan A) { core.Drain(in) } -// Discard returns immediately, consumes the input channel in the background, and discards all its items. +// Discard returns immediately, then drains and discards all items of +// the channel in the background. A [Scope] passed as an option can be +// used to find out when draining completes. func Discard[A any](in <-chan A, options ...SinkOption) { opts := collectSinkOptions(options) @@ -36,24 +39,21 @@ func Discard[A any](in <-chan A, options ...SinkOption) { }() } -// DrainNB returns immediately, consumes the input channel in the background, and discards all its items. +// DrainNB is a non-blocking version of [Drain]. // -// Deprecated: use [Discard] instead. DrainNB will be removed in v1.0. +// Deprecated: use [Discard] instead, which is identical. DrainNB will +// be removed in v1.0. func DrainNB[A any](in <-chan A) { Discard(in) } -// Buffer takes a channel of items and returns a buffered channel of exact same items in the same order. -// This can be useful for preventing write operations on the input channel from blocking, especially if subsequent stages -// in the processing pipeline are slow. -// Up to size items can be buffered before back pressure is applied to the upstream producer. -// -// Typical usage of Buffer might look like this: +// Buffer returns a channel of the specified capacity and forwards all input +// items to it. The output is closed once the input is exhausted. // // users := getUsers(ctx, companyID) // users = rill.Buffer(users, 100) -// // Now work with the users channel as usual. -// // Up to 100 users can be buffered if subsequent stages of the pipeline are slow. +// // Up to 100 users can be buffered if subsequent stages of the +// // pipeline are slow. func Buffer[A any](in <-chan A, size int) <-chan A { validateMinSize(size, 0) return core.Buffer(in, size) diff --git a/wrap.go b/wrap.go index 03878f1..e5e0714 100644 --- a/wrap.go +++ b/wrap.go @@ -1,14 +1,14 @@ package rill -// Try represents either a value of type A or an error. -// When Error is non-nil, callers must ignore Value. +// Try holds either a value of type A or an error. When Error is +// non-nil, Value is meaningless. type Try[A any] struct { Value A Error error } -// Stream is a type alias for a channel of [Try] containers. -// This alias is optional, but it can make the code more readable. +// Stream is a type alias for a receive-only channel of [Try] structs. +// Using it is optional but improves readability. // // Before: // @@ -23,11 +23,11 @@ type Try[A any] struct { // } type Stream[T any] = <-chan Try[T] -// Wrap converts a value-error pair into a [Try]. -// If err is non-nil, Wrap returns an error item and ignores value. -// It's a convenience function to avoid creating a [Try] container manually and benefit from type inference. +// Wrap converts a value-error pair into a [Try]. If err is not nil, +// Wrap returns an error item and ignores value. // -// Such function signature also allows concise wrapping of functions that return a value and an error: +// This signature allows concise wrapping of functions that return a +// value and an error: // // item := rill.Wrap(strconv.Atoi("42")) func Wrap[A any](value A, err error) Try[A] { @@ -40,10 +40,12 @@ func Wrap[A any](value A, err error) Try[A] { // FromSlice converts a slice into a stream. // If err is not nil, it is added to the end of the stream. // -// The slice is read in the background until the returned stream -// is fully consumed. Modifying it before is a data race. +// Modifying the slice before the stream is fully consumed is a data +// race. // -// Such function signature allows concise wrapping of functions that return a slice and an error: +// This signature allows concise wrapping of functions that return a +// slice and an error. FromSlice assumes that a non-empty slice along +// with an error is a partial result and preserves both. // // stream := rill.FromSlice(someFunc()) func FromSlice[A any](slice []A, err error) <-chan Try[A] { @@ -75,12 +77,12 @@ func FromSlice[A any](slice []A, err error) <-chan Try[A] { return out } -// ToSlice converts an input stream into a slice. -// If the stream contains errors, ToSlice returns the values that precede -// the first error, along with that error. +// ToSlice collects the stream's values into a slice. When ToSlice +// encounters an error, it immediately returns that error along with the +// partial slice. Otherwise, it consumes the stream to the end and +// returns a slice of all values. // -// This is a blocking ordered function that processes items sequentially. -// See the package documentation for more information on blocking ordered functions and error handling. +// See the [rill] package documentation for the full contract shared by all sinks. func ToSlice[A any](in <-chan Try[A], options ...SinkOption) ([]A, error) { defer Discard(in, options...) @@ -94,10 +96,16 @@ func ToSlice[A any](in <-chan Try[A], options ...SinkOption) ([]A, error) { return res, nil } -// FromChan converts a regular channel into a stream. -// If err is not nil, the function ignores the passed values and returns a stream with a single error. +// FromChan converts a regular channel into a stream. If err is not nil, +// FromChan returns a stream with only that error and ignores values. +// Otherwise, values are forwarded to the output as they arrive, and the +// output is closed once the input is exhausted. // -// Such function signature allows concise wrapping of functions that return a channel and an error: +// A nil input is never exhausted, so the output never closes. +// +// This signature allows concise wrapping of functions that return a +// channel and an error. FromChan assumes a non-nil error means +// someFunc() could not construct the channel. // // stream := rill.FromChan(someFunc()) func FromChan[A any](values <-chan A, err error) <-chan Try[A] { @@ -122,13 +130,16 @@ func FromChan[A any](values <-chan A, err error) <-chan Try[A] { return out } -// FromChans creates a stream from independent value and error channels. -// Items from both inputs are added to the output stream as they arrive, and nil -// errors are skipped. -// The output stream is closed only when both input channels are exhausted. -// In particular, if at least one input is nil, the output stream never closes. +// FromChans converts separate value and error channels into a single +// stream. Values and errors are forwarded to the output as they arrive, +// and nil errors are skipped. The output is closed only when both +// inputs are exhausted. +// +// A nil input is never exhausted, so the output never closes. // -// Such function signature allows concise wrapping of functions that return two channels: +// This signature allows concise wrapping of functions that return two +// channels. FromChans assumes someFunc() returns two channels that +// eventually close. // // stream := rill.FromChans(someFunc()) func FromChans[A any](values <-chan A, errs <-chan error) <-chan Try[A] { @@ -168,9 +179,10 @@ func FromChans[A any](values <-chan A, errs <-chan error) <-chan Try[A] { return out } -// ToChans splits an input stream into two channels: one for values and one for errors. -// Both output channels are closed when the input stream is exhausted. -// They must be consumed concurrently to avoid deadlocks. +// ToChans splits the stream into two channels, one for values and one for +// errors, and forwards each item to the appropriate one. Both channels are +// closed once the input is exhausted. They must be consumed concurrently +// to avoid a deadlock. func ToChans[A any](in <-chan Try[A]) (<-chan A, <-chan error) { if in == nil { return nil, nil @@ -195,9 +207,9 @@ func ToChans[A any](in <-chan Try[A]) (<-chan A, <-chan error) { return out, errs } -// Generate is a shorthand for creating streams. -// It provides a more ergonomic way of sending both values and errors to a stream, manages goroutine and channel lifecycle. -// Nil errors passed to sendError are skipped. +// Generate is shorthand for creating streams: it manages the +// goroutine and channel lifecycle. Inside f, send writes a value to the +// stream, and sendError writes an error unless it is nil. // // stream := rill.Generate(func(send func(int), sendError func(error)) { // for i := 0; i < 100; i++ { @@ -206,7 +218,7 @@ func ToChans[A any](in <-chan Try[A]) (<-chan A, <-chan error) { // sendError(someError) // }) // -// Here's how the same code would look without Generate: +// The same stream without Generate: // // stream := make(chan rill.Try[int]) // go func() {