diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbArrowArrayStream.cs index a4938d2..bad3c13 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbArrowArrayStream.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbArrowArrayStream.cs @@ -27,7 +27,9 @@ internal sealed class MongoDbArrowArrayStream : IArrowArrayStream private readonly Schema schema; private readonly MongoDbColumnPlan[] columns; private readonly int batchSize; - private readonly IAsyncCursor cursor; + private readonly IMongoCollection collection; + private readonly MongoDbQuery query; + private IAsyncCursor? cursor; private IEnumerator? currentBatch; private bool finished; @@ -41,25 +43,8 @@ public MongoDbArrowArrayStream( this.schema = schema; this.columns = columns; this.batchSize = batchSize; - - IFindFluent find = collection.Find(query.Filter); - if (query.Projection is not null) - { - find = find.Project(query.Projection); - } - - if (query.Limit.HasValue) - { - if (query.Limit.Value > int.MaxValue) - { - throw new ArgumentOutOfRangeException(nameof(query), query.Limit, "MongoDB limit cannot exceed Int32.MaxValue."); - } - - find = find.Limit((int)query.Limit.Value); - } - - find.Options.BatchSize = batchSize; - cursor = find.ToCursor(); + this.collection = collection; + this.query = query; } public Schema Schema => schema; @@ -104,12 +89,14 @@ public MongoDbArrowArrayStream( public void Dispose() { currentBatch?.Dispose(); - cursor.Dispose(); + cursor?.Dispose(); currentBatch = null; + cursor = null; } private bool TryReadNextDocument(out BsonDocument document) { + IAsyncCursor activeCursor = EnsureCursor(); while (true) { if (currentBatch is not null && currentBatch.MoveNext()) @@ -120,13 +107,41 @@ private bool TryReadNextDocument(out BsonDocument document) currentBatch?.Dispose(); currentBatch = null; - if (!cursor.MoveNext()) + if (!activeCursor.MoveNext()) { document = []; return false; } - currentBatch = cursor.Current.GetEnumerator(); + currentBatch = activeCursor.Current.GetEnumerator(); } } + + private IAsyncCursor EnsureCursor() + { + if (cursor is not null) + { + return cursor; + } + + IFindFluent find = collection.Find(query.Filter); + if (query.Projection is not null) + { + find = find.Project(query.Projection); + } + + if (query.Limit.HasValue) + { + if (query.Limit.Value > int.MaxValue) + { + throw new ArgumentOutOfRangeException(nameof(query), query.Limit, "MongoDB limit cannot exceed Int32.MaxValue."); + } + + find = find.Limit((int)query.Limit.Value); + } + + find.Options.BatchSize = batchSize; + cursor = find.ToCursor(); + return cursor; + } } diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs index a96f860..c428725 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs @@ -32,4 +32,6 @@ public sealed class MongoDbDatabaseOptions public int BatchSize { get; init; } = 1024; public int SchemaInferenceLimit { get; init; } = 100; + + public int? DefaultLimit { get; init; } = 500; } diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs index 052774e..4497606 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs @@ -26,6 +26,7 @@ public sealed class MongoDbStreamingTableProvider : StreamingTableProvider { private readonly IMongoCollection collection; private readonly int batchSize; + private readonly int? defaultLimit; private readonly MongoDbColumnPlan[] columns; private readonly MongoDbQueryBuilder queryBuilder = new(); @@ -52,7 +53,13 @@ public MongoDbStreamingTableProvider(MongoDbTableOptions options) throw new ArgumentOutOfRangeException(nameof(options), options.SchemaInferenceLimit, "Schema inference limit must be greater than zero."); } + if (options.DefaultLimit.HasValue && options.DefaultLimit.Value <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.DefaultLimit, "Default limit must be greater than zero when specified."); + } + batchSize = options.BatchSize; + defaultLimit = options.DefaultLimit; IMongoClient client = options.Client ?? CreateClient(options.ConnectionString, nameof(options)); collection = client.GetDatabase(options.DatabaseName).GetCollection(options.CollectionName); columns = InferColumns(collection, options.SchemaInferenceLimit); @@ -77,6 +84,11 @@ private MongoDbArrowArrayStream CreateStream(StreamingTableScanRequest request) { MongoDbColumnPlan[] projectedColumns = ProjectColumns(request); MongoDbQuery query = queryBuilder.Build(request, projectedColumns); + if (!query.Limit.HasValue && defaultLimit.HasValue) + { + query = query with { Limit = (ulong)defaultLimit.Value }; + } + Schema projectedSchema = BuildSchema(projectedColumns); return new MongoDbArrowArrayStream(collection, query, projectedSchema, projectedColumns, batchSize); } @@ -124,7 +136,11 @@ private static Schema BuildSchema(IEnumerable columns) builder.Field(field => field .Name(column.Name) .DataType(column.DataType) - .Nullable(column.Nullable)); + // MongoDB collections are schemaless: a field that is present + // and non-null in the inference sample can still be missing or + // null in later documents. The appenders already emit nulls for + // those cases, so expose all MongoDB fields as nullable. + .Nullable(true)); } return builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbTableOptions.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbTableOptions.cs index a3f6d73..9506d77 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbTableOptions.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbTableOptions.cs @@ -32,4 +32,6 @@ public sealed class MongoDbTableOptions public int BatchSize { get; init; } = 1024; public int SchemaInferenceLimit { get; init; } = 100; -} + + public int? DefaultLimit { get; init; } = 500; +} \ No newline at end of file diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs index a9bb5a1..11e6a7c 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs @@ -44,6 +44,11 @@ public static void RegisterMongoDb(this SessionContext context, MongoDbDatabaseO throw new ArgumentOutOfRangeException(nameof(options), options.SchemaInferenceLimit, "Schema inference limit must be greater than zero."); } + if (options.DefaultLimit.HasValue && options.DefaultLimit.Value <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.DefaultLimit, "Default limit must be greater than zero when specified."); + } + string databaseName = ResolveDatabaseName(options.ConnectionString, options.DatabaseName); IMongoClient client = options.Client ?? CreateClient(options.ConnectionString, nameof(options)); IMongoDatabase database = client.GetDatabase(databaseName); @@ -57,6 +62,7 @@ public static void RegisterMongoDb(this SessionContext context, MongoDbDatabaseO CollectionName = collectionName, BatchSize = options.BatchSize, SchemaInferenceLimit = options.SchemaInferenceLimit, + DefaultLimit = options.DefaultLimit, })); } } @@ -65,6 +71,10 @@ public static void RegisterMongoDb(this SessionContext context, string name, Mon { ArgumentNullException.ThrowIfNull(context); ArgumentNullException.ThrowIfNull(options); + if (options.DefaultLimit.HasValue && options.DefaultLimit.Value <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.DefaultLimit, "Default limit must be greater than zero when specified."); + } context.RegisterStreamingTable(name, new MongoDbStreamingTableProvider(options)); }