Large File Uplaod issue

[ecodetech]

Just ask something
Is there any limit on file uploading size?
How many chunk of bytes are read at a time while uploading a bigger file? 512/ 1024 bytes or more while uploading?

Describe the bug
I am trying to upload a text file from SPIFFS. I am able to do it for smaller files, it works.
When tried for larger files, the upload does not happen. File is of 659818 bytes and contains alphanumeric texts data.

To Reproduce

#include <WiFi.h>
#include <WiFiClientSecure.h>
#include <SPIFFS.h>

#define SSL_CLIENT WiFiClientSecure
#define ENABLE_SERVICE_AUTH
#define ENABLE_STORAGE
#define ENABLE_FS

#include <FirebaseClient.h>
//#include "ExampleFunctions.h" // Provides the functions used in the examples.

#define WIFI_SSID "WIFI_AP"
#define WIFI_PASSWORD "WIFI_PASSWORD"

#define FIREBASE_PROJECT_ID "PROJECT_ID"
#define FIREBASE_CLIENT_EMAIL "CLIENT_EMAIL"
const char PRIVATE_KEY[] PROGMEM = "-----BEGIN PRIVATE KEY-----XXXXXXXXXXXX-----END PRIVATE KEY-----\n";

// Define the Firebase storage bucket ID e.g bucket-name.appspot.com */
#define STORAGE_BUCKET_ID "BUCKET-NAME.appspot.com"

ServiceAuth sa_auth(FIREBASE_CLIENT_EMAIL, FIREBASE_PROJECT_ID, PRIVATE_KEY, 3000 /* expire period in seconds (<= 3600) */);

FirebaseApp app;

SSL_CLIENT ssl_client;

using AsyncClient = AsyncClientClass;
AsyncClient aClient(ssl_client);

Storage storage;

bool taskComplete = false;

AsyncResult storageResult;

File myFile;


void processData(AsyncResult &aResult);
void file_operation_callback(File &file, const char *filename, file_operating_mode mode );

FileConfig text_file("/backup.txt", file_operation_callback); // Can be set later with media_file.setFile("/media.mp4", file_operation_callback);

void setup() {
  // put your setup code here, to run once:
  Serial.begin(115200);
  SPIFFS.begin();
  delay(5000);


  WiFi.begin(WIFI_SSID, WIFI_PASSWORD);

  Serial.print("Connecting to Wi-Fi");
  while (WiFi.status() != WL_CONNECTED)
  {
    Serial.print(".");
    delay(300);
  }
  Serial.println();
  Serial.print("Connected with IP: ");
  Serial.println(WiFi.localIP());
  Serial.println();

  Firebase.printf("Firebase Client v%s\n", FIREBASE_CLIENT_VERSION);
  ssl_client.setInsecure();
  app.setTime(get_ntp_time());

  Serial.println("Initializing app...");
  initializeApp(aClient, app, getAuth(sa_auth), auth_debug_print, "🔐 authTask");

  // Or intialize the app and wait.
  // initializeApp(aClient, app, getAuth(user_auth), 120 * 1000, auth_debug_print);

  app.getApp<Storage>(storage);

}

void loop() {
  // To maintain the authentication process.
  app.loop();

  if (app.ready() && !taskComplete)
  {
    taskComplete = true;

    Serial.println("Uploading file...");

    bool status = storage.upload(aClient, FirebaseStorage::Parent(STORAGE_BUCKET_ID, "backup.txt"), getFile(text_file), "plain/text");
 
    if (status)
      Serial.println("🔼 Upload task(await), complete!✅️");
    else
      Firebase.printf("Error, msg: %s, code: %d\n", aClient.lastError().message().c_str(), aClient.lastError().code());
  }

  // For async call with AsyncResult.
  processData(storageResult);
}

void file_operation_callback(File &file, const char *filename, file_operating_mode mode )
{
  // FILE_OPEN_MODE_READ, FILE_OPEN_MODE_WRITE and FILE_OPEN_MODE_APPEND are defined in this library
  // MY_FS is defined in this example
  switch (mode)
  {
    case file_mode_open_read:
      Serial.println("Open read mode");
      myFile = SPIFFS.open(filename, FILE_OPEN_MODE_READ);
      
      Serial.print("\tSIZE: ");
      Serial.println(myFile.size());
      if (!myFile || file.isDirectory()) {
        Serial.println("- failed to open file for reading");
      }

      break;
    case file_mode_open_write:
      Serial.println("Open write mode");
      myFile = SPIFFS.open(filename, FILE_OPEN_MODE_WRITE);
      break;
    case file_mode_open_append:
      Serial.println("Open append mode");
      myFile = SPIFFS.open(filename, FILE_OPEN_MODE_APPEND);
      break;
    case file_mode_remove:
      Serial.println("Open remove mode");
      //SPIFFS.remove(filename);
      break;
    default:
      break;
  }
  // Set the internal FS object with global File object.
  file = myFile;
}

void processData(AsyncResult &aResult)
{
    // Exits when no result is available when calling from the loop.
    if (!aResult.isResult())
        return;

    if (aResult.isEvent())
    {
        Firebase.printf("Event task: %s, msg: %s, code: %d\n", aResult.uid().c_str(), aResult.appEvent().message().c_str(), aResult.appEvent().code());
    }

    if (aResult.isDebug())
    {
        Firebase.printf("Debug task: %s, msg: %s\n", aResult.uid().c_str(), aResult.debug().c_str());
    }

    if (aResult.isError())
    {
        Firebase.printf("Error task: %s, msg: %s, code: %d\n", aResult.uid().c_str(), aResult.error().message().c_str(), aResult.error().code());
    }

    if (aResult.downloadProgress())
    {
        Firebase.printf("Downloaded, task: %s, %d%s (%d of %d)\n", aResult.uid().c_str(), aResult.downloadInfo().progress, "%", aResult.downloadInfo().downloaded, aResult.downloadInfo().total);
        if (aResult.downloadInfo().total == aResult.downloadInfo().downloaded)
        {
            Firebase.printf("Download task: %s, complete!✅️\n", aResult.uid().c_str());
        }
    }

    if (aResult.uploadProgress())
    {
        Firebase.printf("Uploaded, task: %s, %d%s (%d of %d)\n", aResult.uid().c_str(), aResult.uploadInfo().progress, "%", aResult.uploadInfo().uploaded, aResult.uploadInfo().total);
        if (aResult.uploadInfo().total == aResult.uploadInfo().uploaded)
        {
            Firebase.printf("Upload task: %s, complete!✅️\n", aResult.uid().c_str());
            Serial.print("Download URL: ");
            Serial.println(aResult.uploadInfo().downloadUrl);
        }
    }
}

Expected behaviour
It is expected the file will upload successfully, it should be seen in Firebase Storage.

Serial Terminal Log

16:55:50.459 ->
16:55:50.459 -> Firebase Client v2.2.4
16:55:50.459 -> Getting time from NTP server... success
16:55:51.354 -> Initializing app...
16:55:51.354 -> Event task: 🔐 authTask, msg: initializing, code: 1
16:55:51.354 -> Event task: 🔐 authTask, msg: token signing, code: 6
16:55:51.850 -> Event task: 🔐 authTask, msg: authenticating, code: 7
16:55:51.850 -> Debug task: 🔐 authTask, msg: Connecting to server...
16:55:51.850 -> Event task: 🔐 authTask, msg: auth request sent, code: 8
16:55:53.574 -> Event task: 🔐 authTask, msg: auth response received, code: 9
16:55:53.574 -> Debug task: 🔐 authTask, msg: Terminating the server connection...
16:55:53.574 -> Event task: 🔐 authTask, msg: ready, code: 10
16:55:53.574 -> Uploading file...
16:55:53.607 -> Open read mode
16:55:53.640 -> SIZE: 659818
16:55:54.766 -> Open read mode
16:55:54.766 -> SIZE: 659818
16:55:54.766 -> 🔼 Upload task(await), complete!✅️

IDE and its version:

• Arduino 1.8.18

ESP32 Arduino Core SDK version

• Version 2.0.5

Additional tests
I also created another custom file wrapper for reading data from another external SPI flash as a File. When that file is passed to this example as a file, the data is read as 1 byte at a time in the log I printed. I was expecting it to read data in bigger chunks like 512 bytes.
This caused the TCP error -2 and file upload failed.

Please help on this. Let me know if I made any mistake while using this example.

backup.txt

[mobizt]

The default raw chunk size for TCP data sending for header and payload is 2048 bytes.

Your ESP32 Arduino core is outdated and may not be compatible.
You should update the ESP32 Arduino Core SDK to latest version.