Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ internal sealed class MongoDbArrowArrayStream : IArrowArrayStream
private readonly Schema schema;
private readonly MongoDbColumnPlan[] columns;
private readonly int batchSize;
private readonly IAsyncCursor<BsonDocument> cursor;
private readonly IMongoCollection<BsonDocument> collection;
private readonly MongoDbQuery query;
private IAsyncCursor<BsonDocument>? cursor;
private IEnumerator<BsonDocument>? currentBatch;
private bool finished;

Expand All @@ -41,25 +43,8 @@ public MongoDbArrowArrayStream(
this.schema = schema;
this.columns = columns;
this.batchSize = batchSize;

IFindFluent<BsonDocument, BsonDocument> find = collection.Find(query.Filter);
if (query.Projection is not null)
{
find = find.Project<BsonDocument>(query.Projection);
}

if (query.Limit.HasValue)
{
if (query.Limit.Value > int.MaxValue)
{
throw new ArgumentOutOfRangeException(nameof(query), query.Limit, "MongoDB limit cannot exceed Int32.MaxValue.");
}

find = find.Limit((int)query.Limit.Value);
}

find.Options.BatchSize = batchSize;
cursor = find.ToCursor();
this.collection = collection;
this.query = query;
}

public Schema Schema => schema;
Expand Down Expand Up @@ -104,12 +89,14 @@ public MongoDbArrowArrayStream(
public void Dispose()
{
currentBatch?.Dispose();
cursor.Dispose();
cursor?.Dispose();
currentBatch = null;
cursor = null;
}

private bool TryReadNextDocument(out BsonDocument document)
{
IAsyncCursor<BsonDocument> activeCursor = EnsureCursor();
while (true)
{
if (currentBatch is not null && currentBatch.MoveNext())
Expand All @@ -120,13 +107,41 @@ private bool TryReadNextDocument(out BsonDocument document)

currentBatch?.Dispose();
currentBatch = null;
if (!cursor.MoveNext())
if (!activeCursor.MoveNext())
{
document = [];
return false;
}

currentBatch = cursor.Current.GetEnumerator();
currentBatch = activeCursor.Current.GetEnumerator();
}
}

private IAsyncCursor<BsonDocument> EnsureCursor()
{
if (cursor is not null)
{
return cursor;
}

IFindFluent<BsonDocument, BsonDocument> find = collection.Find(query.Filter);
if (query.Projection is not null)
{
find = find.Project<BsonDocument>(query.Projection);
}

if (query.Limit.HasValue)
{
if (query.Limit.Value > int.MaxValue)
{
throw new ArgumentOutOfRangeException(nameof(query), query.Limit, "MongoDB limit cannot exceed Int32.MaxValue.");
}

find = find.Limit((int)query.Limit.Value);
}

find.Options.BatchSize = batchSize;
cursor = find.ToCursor();
return cursor;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,6 @@ public sealed class MongoDbDatabaseOptions
public int BatchSize { get; init; } = 1024;

public int SchemaInferenceLimit { get; init; } = 100;

public int? DefaultLimit { get; init; } = 500;
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ public sealed class MongoDbStreamingTableProvider : StreamingTableProvider
{
private readonly IMongoCollection<BsonDocument> collection;
private readonly int batchSize;
private readonly int? defaultLimit;
private readonly MongoDbColumnPlan[] columns;
private readonly MongoDbQueryBuilder queryBuilder = new();

Expand All @@ -52,7 +53,13 @@ public MongoDbStreamingTableProvider(MongoDbTableOptions options)
throw new ArgumentOutOfRangeException(nameof(options), options.SchemaInferenceLimit, "Schema inference limit must be greater than zero.");
}

if (options.DefaultLimit.HasValue && options.DefaultLimit.Value <= 0)
{
throw new ArgumentOutOfRangeException(nameof(options), options.DefaultLimit, "Default limit must be greater than zero when specified.");
}

batchSize = options.BatchSize;
defaultLimit = options.DefaultLimit;
IMongoClient client = options.Client ?? CreateClient(options.ConnectionString, nameof(options));
collection = client.GetDatabase(options.DatabaseName).GetCollection<BsonDocument>(options.CollectionName);
columns = InferColumns(collection, options.SchemaInferenceLimit);
Expand All @@ -77,6 +84,11 @@ private MongoDbArrowArrayStream CreateStream(StreamingTableScanRequest request)
{
MongoDbColumnPlan[] projectedColumns = ProjectColumns(request);
MongoDbQuery query = queryBuilder.Build(request, projectedColumns);
if (!query.Limit.HasValue && defaultLimit.HasValue)
{
query = query with { Limit = (ulong)defaultLimit.Value };
}

Schema projectedSchema = BuildSchema(projectedColumns);
return new MongoDbArrowArrayStream(collection, query, projectedSchema, projectedColumns, batchSize);
}
Expand Down Expand Up @@ -124,7 +136,11 @@ private static Schema BuildSchema(IEnumerable<MongoDbColumnPlan> columns)
builder.Field(field => field
.Name(column.Name)
.DataType(column.DataType)
.Nullable(column.Nullable));
// MongoDB collections are schemaless: a field that is present
// and non-null in the inference sample can still be missing or
// null in later documents. The appenders already emit nulls for
// those cases, so expose all MongoDB fields as nullable.
.Nullable(true));
}

return builder.Build();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,6 @@ public sealed class MongoDbTableOptions
public int BatchSize { get; init; } = 1024;

public int SchemaInferenceLimit { get; init; } = 100;
}

public int? DefaultLimit { get; init; } = 500;
}
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,11 @@ public static void RegisterMongoDb(this SessionContext context, MongoDbDatabaseO
throw new ArgumentOutOfRangeException(nameof(options), options.SchemaInferenceLimit, "Schema inference limit must be greater than zero.");
}

if (options.DefaultLimit.HasValue && options.DefaultLimit.Value <= 0)
{
throw new ArgumentOutOfRangeException(nameof(options), options.DefaultLimit, "Default limit must be greater than zero when specified.");
}

string databaseName = ResolveDatabaseName(options.ConnectionString, options.DatabaseName);
IMongoClient client = options.Client ?? CreateClient(options.ConnectionString, nameof(options));
IMongoDatabase database = client.GetDatabase(databaseName);
Expand All @@ -57,6 +62,7 @@ public static void RegisterMongoDb(this SessionContext context, MongoDbDatabaseO
CollectionName = collectionName,
BatchSize = options.BatchSize,
SchemaInferenceLimit = options.SchemaInferenceLimit,
DefaultLimit = options.DefaultLimit,
}));
}
}
Expand All @@ -65,6 +71,10 @@ public static void RegisterMongoDb(this SessionContext context, string name, Mon
{
ArgumentNullException.ThrowIfNull(context);
ArgumentNullException.ThrowIfNull(options);
if (options.DefaultLimit.HasValue && options.DefaultLimit.Value <= 0)
{
throw new ArgumentOutOfRangeException(nameof(options), options.DefaultLimit, "Default limit must be greater than zero when specified.");
}

context.RegisterStreamingTable(name, new MongoDbStreamingTableProvider(options));
}
Expand Down
Loading