ADBM-2913: Implement repo-push command

src/Makefile.in

@@ -83,6 +83,7 @@ SRCS = \
 	command/repo/common.c \
 	command/repo/get.c \
 	command/repo/ls.c \
+	command/repo/push.c \
 	command/repo/put.c \
 	command/repo/rm.c \
 	command/restore/blockChecksum.c \

src/build/config/config.yaml

@@ -107,6 +107,16 @@ command:
     log-level-default: DEBUG
     parameter-allowed: true
 
+  repo-push:
+    command-role:
+      async: {}
+      local: {}
+      remote: {}
+    internal: true
+    log-file: false
+    log-level-default: DEBUG
+    parameter-allowed: true
+
   repo-put:
     command-role:
       remote: {}
@@ -328,6 +338,7 @@ option:
       backup: {}
       check: {}
       expire: {}
+      repo-push: {}
       restore: {}
       stanza-create: {}
       stanza-upgrade: {}
@@ -392,6 +403,8 @@ option:
       manifest:
         default: latest
         required: true
+      repo-push:
+        required: true
       restore:
         default: latest
         required: true
@@ -425,6 +438,8 @@ option:
         required: false
       repo-ls:
         required: false
+      repo-push:
+        required: true
       repo-put:
         required: false
       repo-rm:
@@ -572,6 +587,7 @@ option:
       manifest: {}
       repo-get: {}
       repo-ls: {}
+      repo-push: {}
       repo-put: {}
       repo-rm: {}
       restore: {}
@@ -614,6 +630,7 @@ option:
       manifest: {}
       repo-get: {}
       repo-ls: {}
+      repo-push: {}
       repo-put: {}
       repo-rm: {}
       restore: {}
@@ -785,6 +802,7 @@ option:
     command:
       archive-push: {}
       backup: {}
+      repo-push: {}
     command-role:
       main: {}
 
@@ -812,6 +830,7 @@ option:
       manifest: {}
       repo-get: {}
       repo-ls: {}
+      repo-push: {}
       repo-put: {}
       restore: {}
       stanza-create: {}
@@ -1731,6 +1750,9 @@ option:
         command-role:
           main: {}
           remote: {}
+      repo-push:
+        command-role:
+          main: {}
       repo-put:
         command-role:
           main: {}
@@ -1851,6 +1873,10 @@ option:
         command-role:
           main: {}
           remote: {}
+      repo-push:
+        command-role:
+          main: {}
+          remote: {}
       repo-put:
         command-role:
           main: {}

src/build/help/help.xml

@@ -2414,6 +2414,27 @@
                 </option-list>
             </command>
 
+            <command id="repo-push" name="Repository Push File">
+                <summary>Upload a metadata file to the repository.</summary>
+
+                <text>
+                    <p>Upload a metadata file to the repository. The metadata file is stored in the same storage as the restore point and inherits the compression settings defined in the pgBackRest configuration.</p>
+                </text>
+
+                <option-list>
+                    <option id="set" name="Set">
+                        <summary>Backup set.</summary>
+
+                        <text>
+                            <p>Identifier of the backup set (restore point). The metadata file will be placed in this set's directory.</p>
+                        </text>
+
+                        <example>20150131-153358F_20150131-153401I</example>
+                    </option>
+                </option-list>
+
+            </command>
+
             <command id="repo-put" name="Repository Put File">
                 <summary>Put a file in the repository.</summary>
 

src/command/repo/push.c

@@ -0,0 +1,208 @@
+/***********************************************************************************************************************************
+Repository Put Command
+***********************************************************************************************************************************/
+#include "build.auto.h"
+
+#include <unistd.h>
+
+#include "command/repo/common.h"
+#include "common/compress/helper.h"
+#include "common/debug.h"
+#include "common/io/fdRead.h"
+#include "common/io/filter/size.h"
+#include "common/io/io.h"
+#include "common/log.h"
+#include "common/memContext.h"
+#include "common/type/string.h"
+#include "config/config.h"
+#include "info/manifest.h"
+#include "storage/helper.h"
+
+static String *
+composeDestinationPath(const String *backupLabel, const String *fileName)
+{
+    FUNCTION_LOG_BEGIN(logLevelDebug);
+        FUNCTION_LOG_PARAM(STRING, backupLabel);
+        FUNCTION_LOG_PARAM(STRING, fileName);
+    FUNCTION_LOG_END();
+
+    String *const result = strNewFmt(STORAGE_REPO_BACKUP "/%s/%s", strZ(backupLabel), strZ(fileName));
+
+    FUNCTION_LOG_RETURN(STRING, result);
+}
+
+/***********************************************************************************************************************************
+Write source IO to destination file
+***********************************************************************************************************************************/
+static void
+storagePushProcess(const String *file, CompressType compressType, int compressLevel)
+{
+    FUNCTION_LOG_BEGIN(logLevelDebug);
+        FUNCTION_LOG_PARAM(STRING, file);
+        FUNCTION_LOG_PARAM(ENUM, compressType);
+        FUNCTION_LOG_PARAM(INT, compressLevel);
+    FUNCTION_LOG_END();
+
+    // Ensure that the file exists and readable
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        // Normalize source file path
+        // Get current working dir
+        char currentWorkDir[1024];
+        THROW_ON_SYS_ERROR(getcwd(currentWorkDir, sizeof(currentWorkDir)) == NULL, FormatError, "unable to get cwd");
+
+        String *sourcePath = strPathAbsolute(file, strNewZ(currentWorkDir));
+
+        // Repository Path Formation
+
+        const String *backupLabel = cfgOptionStr(cfgOptSet);
+
+        String *destFilename = strCat(strNew(), strFileName(file));
+        compressExtCat(destFilename, compressType);
+
+        String *destPath = composeDestinationPath(backupLabel, destFilename);
+
+        // Is path valid for repo?
+        destPath = repoPathIsValid(destPath);
+
+        bool repoChecksum = false;
+        const Storage *storage = storageRepoWrite();
+        const StorageWrite *const destination = storageNewWriteP(storage, destPath);
+
+        IoRead *const sourceIo = storageReadIo(storageNewReadP(storageLocal(), sourcePath));
+        IoWrite *const destinationIo = storageWriteIo(destination);
+
+        IoFilterGroup *const readFilterGroup = ioReadFilterGroup(sourceIo);
+        IoFilterGroup *const writeFilterGroup = ioWriteFilterGroup(destinationIo);
+
+        const String *manifestFileName = strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE, strZ(backupLabel));
+        Manifest *manifest = manifestLoadFile(
+            storage, manifestFileName,
+            cipherTypeNone, NULL);
+
+        // Add SHA1 filter
+        ioFilterGroupAdd(writeFilterGroup, cryptoHashNew(hashTypeSha1));
+
+        // Add compression
+        if (compressType != compressTypeNone)
+        {
+            ioFilterGroupAdd(
+                writeFilterGroup,
+                compressFilterP(compressType, compressLevel));
+
+            repoChecksum = true;
+        }
+
+        // Capture checksum of file stored in the repo if filters that modify the output have been applied
+        if (repoChecksum)
+            ioFilterGroupAdd(writeFilterGroup, cryptoHashNew(hashTypeSha1));
+
+        // Add size filter last to calculate repo size
+        ioFilterGroupAdd(writeFilterGroup, ioSizeNew());
+
+        // Add size filter last to calculate source file size
+        ioFilterGroupAdd(readFilterGroup, ioSizeNew());
+
+        // Open source and destination
+        ioReadOpen(sourceIo);
+        ioWriteOpen(destinationIo);
+
+        // Copy data from source to destination
+        ioCopyP(sourceIo, destinationIo);
+
+        // Close the source and destination
+        ioReadClose(sourceIo);
+        ioWriteClose(destinationIo);
+
+        // Use base path to set ownership and mode
+        const ManifestPath *const basePath = manifestPathFind(manifest, MANIFEST_TARGET_PGDATA_STR);
+
+        // Add to manifest
+        uint64_t rsize = pckReadU64P(ioFilterGroupResultP(readFilterGroup, SIZE_FILTER_TYPE));
+        uint64_t wsize = pckReadU64P(ioFilterGroupResultP(writeFilterGroup, SIZE_FILTER_TYPE));
+        ManifestFile customFile =
+        {
+            .name = destFilename,
+            .mode = basePath->mode & (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH),
+            .size = wsize,
+            .sizeOriginal = rsize,
+            .sizeRepo = wsize,
+            .timestamp = time(NULL),
+            .checksumSha1 = bufPtr(pckReadBinP(ioFilterGroupResultP(writeFilterGroup, CRYPTO_HASH_FILTER_TYPE, .idx = 0))),
+        };
+
+        if (repoChecksum)
+        {
+            PackRead *packRead = ioFilterGroupResultP(writeFilterGroup, CRYPTO_HASH_FILTER_TYPE, .idx = 1);
+            ASSERT(packRead != NULL);
+            customFile.checksumRepoSha1 = bufPtr(pckReadBinP(packRead));
+        }
+
+        bool found = false;
+
+        for (unsigned int fileIdx = 0; fileIdx < manifestCustomFileTotal(manifest); fileIdx++)
+        {
+            ManifestFile manifestFile = manifestCustomFile(manifest, fileIdx);
+
+            if (strCmp(manifestFile.name, destFilename) == 0){
+                manifestFile.mode = customFile.mode;
+                manifestFile.size = customFile.size;
+                manifestFile.sizeOriginal = customFile.sizeOriginal;
+                manifestFile.sizeRepo = customFile.sizeRepo;
+                manifestFile.timestamp = customFile.timestamp;
+                manifestFile.checksumSha1 = customFile.checksumSha1;
+
+                manifestCustomFileUpdate(manifest, &manifestFile);
+                found = true;
+                break;
+            }
+        }
+
+        if (!found)
+            manifestCustomFileAdd(manifest, &customFile);
+
+        // Save manifest
+        IoWrite *const manifestWrite = storageWriteIo(
+            storageNewWriteP(
+                storageRepoWrite(),
+                manifestFileName
+                ));
+
+        manifestSave(manifest, manifestWrite);
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    FUNCTION_LOG_RETURN_VOID();
+}
+
+/**********************************************************************************************************************************/
+FN_EXTERN void
+cmdStoragePush(void)
+{
+    FUNCTION_LOG_VOID(logLevelDebug);
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        const StringList *params = cfgCommandParam();
+
+        if (strLstSize(params) != 1)
+            THROW(ParamInvalidError, "exactly one parameter is required");
+
+        String *filename = strLstGet(params, 0);
+
+        // Check that the filename does not end with a slash
+        if (strEndsWith(filename, FSLASH_STR))
+            THROW(ParamInvalidError, "file parameter should not end with a slash");
+
+        LOG_INFO_FMT(
+            "push file %s to the archive.",
+            strZ(filename));
+
+        storagePushProcess(filename,
+                           compressTypeEnum(cfgOptionStrId(cfgOptCompressType)),
+                           cfgOptionInt(cfgOptCompressLevel));
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    FUNCTION_LOG_RETURN_VOID();
+}

src/command/repo/push.h

@@ -0,0 +1,13 @@
+/***********************************************************************************************************************************
+Repository Push Command
+***********************************************************************************************************************************/
+#ifndef COMMAND_STORAGE_PUSH_H
+#define COMMAND_STORAGE_PUSH_H
+
+/***********************************************************************************************************************************
+Functions
+***********************************************************************************************************************************/
+// Push a metadata file into the repository
+FN_EXTERN void cmdStoragePush(void);
+
+#endif

src/common/type/string.c

@@ -796,6 +796,27 @@ strPath(const String *const this)
             end - this->pub.buffer <= 1 ? (size_t)(end - this->pub.buffer) : (size_t)(end - this->pub.buffer - 1)));
 }
 
+/**********************************************************************************************************************************/
+FN_EXTERN String *
+strFileName(const String *const this)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(STRING, this);
+    FUNCTION_TEST_END();
+
+    ASSERT(this != NULL);
+
+    const char *end = this->pub.buffer + strSize(this);
+    const char *ptr = end;
+
+    while (ptr > this->pub.buffer && *(ptr - 1) != '/')
+        ptr--;
+
+    FUNCTION_TEST_RETURN(
+        STRING,
+        strNewZN(ptr, (size_t)(end - ptr)));
+}
+
 /**********************************************************************************************************************************/
 FN_EXTERN String *
 strPathAbsolute(const String *const this, const String *const base)

src/common/type/string.h

@@ -197,6 +197,9 @@ FN_EXTERN String *strLower(String *this);
 // Return the path part of a string (i.e. everything before the last / or "" if there is no /)
 FN_EXTERN String *strPath(const String *this);
 
+// Return the file name part of a string (i.e. everything after the last / or "" if there is no /)
+FN_EXTERN String *strFileName(const String *const this);
+
 // Combine with a base path to get an absolute path
 FN_EXTERN String *strPathAbsolute(const String *this, const String *base);