From f0f95c9da66ad88739b8f5c605bc78bcc0270232 Mon Sep 17 00:00:00 2001 From: thelopesbr Date: Mon, 18 Sep 2023 21:32:55 -0300 Subject: [PATCH] configurando o fluxo --- config/multer.js | 25 +++++++++++++++++++++ src/controllers/import.js | 23 +++----------------- src/middlewares/uploadArchive.js | 35 ++++++++++++++++++++++++++++++ src/routes/import.js | 8 ++++--- src/services/import.js | 29 +++++++++++++------------ src/services/multer.js | 37 -------------------------------- 6 files changed, 83 insertions(+), 74 deletions(-) create mode 100644 config/multer.js create mode 100644 src/middlewares/uploadArchive.js delete mode 100644 src/services/multer.js diff --git a/config/multer.js b/config/multer.js new file mode 100644 index 0000000..e0f6753 --- /dev/null +++ b/config/multer.js @@ -0,0 +1,25 @@ +const multer = require('multer'); +const { nanoid } = require('nanoid'); +const fs = require('fs-extra'); +const path = require('path'); + + +exports.config = { + fileFilter: (req, file, cb) => { + const fileExtension = path.extname(file.originalname); + if (fileExtension === '.tgz') { + cb(null, true); + } else { + cb(new Error('Invalid file type.')); + } + }, + storage: multer.diskStorage({ + destination: (req, file, cb) => { + const path = `/tmp/import/${nanoid(8)}` + fs.mkdirSync(path, {recursive: true}) + cb(null, path) + }, + filename: (req, file, cb) => { + cb(null, file.originalname) + }}) +}; \ No newline at end of file diff --git a/src/controllers/import.js b/src/controllers/import.js index 5eee947..d47bcd1 100644 --- a/src/controllers/import.js +++ b/src/controllers/import.js @@ -1,34 +1,17 @@ const service = require('../services/import'); -const serviceMulter = require('../services/multer'); + /** * @param {import('express').Request} req * @param {import('express').Response} res */ const run = async (req, res) => { - await service.run(); + const {bucket, name, path} = req.fileInMinio; + await service.run(bucket, name, path); res.sendStatus(200); }; -const uploadArchive = () => { - return serviceMulter.importUploader.single('file'); -}; - -/** - * @param {import('express').Request} req - * @param {import('express').Response} res - */ -const checkUploadArchive = async (req, res) => { - console.log('Arquivo recebido',req.file); - - if (!req.file) - return res.status(400).json({ success: false, message: 'you need to provide an import archive.'}); - await serviceMulter.putUploadToMinio(req) - return res.status(200).json({ success: true, message: 'Success.'}); -}; module.exports = { run, - uploadArchive, - checkUploadArchive, }; diff --git a/src/middlewares/uploadArchive.js b/src/middlewares/uploadArchive.js new file mode 100644 index 0000000..8615200 --- /dev/null +++ b/src/middlewares/uploadArchive.js @@ -0,0 +1,35 @@ +const fs = require('fs-extra'); +const objstore = require('../services/objstore'); +const config = require('../config') + + +const putStreamUploadToMinio = async (req) => { + await objstore.putStream(config.minio.importsBucket, req.file.originalname, req.file.buffer); +}; + +// Por que PUT e não POST? +const putUploadToMinio = async (req) => { + await objstore.put(config.minio.importsBucket, req.file.filename, req.file.path); + await fs.unlink(req.file.path); + await fs.rmdir('/tmp/import', {recursive: true}); + + return { + bucket: config.minio.importsBucket, + name: req.file.filename, + path: req.file.path, + } +}; + + // está checando a requisição apenas, não o diretorio do servidor (É necessário chegar o servidor?) +const checkUploadArchive = async (req, res, next) => { + + if (!req.file) + return res.status(400).json({ success: false, message: 'you need to provide an import archive.'}); + const fileInMinio = await putUploadToMinio(req) + req.fileInMinio = fileInMinio; + return next(); +}; + +module.exports = { + checkUploadArchive, +}; \ No newline at end of file diff --git a/src/routes/import.js b/src/routes/import.js index fa9b8a8..fa80aac 100644 --- a/src/routes/import.js +++ b/src/routes/import.js @@ -1,14 +1,16 @@ const express = require('express'); const controller = require('../controllers/import'); -const service = require('../services/multer'); - const router = express(); +const multer = require('multer'); +const multerConfig = require('../../config/multer').config; +const uploadArchive = require('../middlewares/uploadArchive'); + // POST /import/run -> Run import router.post('/run', controller.run); // POST /import -> upload import archive -router.post('/', controller.uploadArchive('file'), controller.checkUploadArchive); +router.post('/', multer(multerConfig).single('file'), uploadArchive.checkUploadArchive,controller.run); // GET /export/{instance} -> List exports by instance // router.get('/:instance', controller.listByInstance); diff --git a/src/services/import.js b/src/services/import.js index 7dae2d4..ba2419e 100644 --- a/src/services/import.js +++ b/src/services/import.js @@ -26,13 +26,9 @@ const createControlRecord = async(iteration) => { await db.query(sql) } -const resolveImportName = (instance, iteration) => `${instance}.${iteration}.tgz`; - -const run = async () => { - console.log('importing') +const run = async (bucket, importName, path) => { const nextImport = await getNextImport() - const importName = 'IES.1.tgz'; - // const importName = resolveImportName(config.instance, nextImport); + const tmpPath = `/tmp/avapolos_syncer/${nanoid(8)}`; const tmpImportPath = `${tmpPath}/import.tgz`; const tmpImportDataPath = `${tmpPath}/import`; @@ -40,24 +36,29 @@ const run = async () => { const moodledataFiledirPath = `${await container.getVolumeMountpointByContainer('moodle', "/app/moodledata")}/filedir`; // const nextImport = 50 - console.log(`next iteration: ${nextImport}`) + //console.log(`next iteration: ${nextImport}`) - if (! await objstore.has(config.minio.exportsBucket, importName)) throw new Error('import archive was not found'); + if (! await objstore.has(bucket, importName)) throw new Error('import archive was not found'); - await objstore.get(config.minio.exportsBucket, importName, tmpImportPath); - await compressing.tgz.uncompress(tmpImportPath, tmpImportDataPath); + await objstore.get(bucket, importName, tmpImportPath); + await compressing.tgz.uncompress(tmpImportPath, tmpImportDataPath); + console.log(`${tmpImportDataPath}/database`) console.log(syncDataPath) - + + // Apaga o diretorio de dados do container de sincronização await fs.rmdir(syncDataPath, { recursive: true }); - await fs.copy(`${tmpImportDataPath}/database`, syncDataPath, { recursive: true }); + // NÃO copia os dados para o diretorio, apresenta erro nessa etapa (Não sei o motivo) + fs.copy(`${tmpImportDataPath}/database`, syncDataPath, { recursive: true }); + await container.stop(config.replication.main); await container.start(config.replication.sync); await container.start(config.replication.main); - waitForHealthy(async () => { + + await waitForHealthy(async () => { await db.query("SELECT bdr.wait_slot_confirm_lsn(NULL, NULL)"); console.log('stopping sync db') container.stop(config.replication.sync); @@ -65,9 +66,9 @@ const run = async () => { }) await createControlRecord(nextImport); + await container.runCommand('moodle', ["php", "/app/public/admin/cli/purge_caches.php"]) await container.restart('moodle'); - return } module.exports = { diff --git a/src/services/multer.js b/src/services/multer.js deleted file mode 100644 index 13ca7d3..0000000 --- a/src/services/multer.js +++ /dev/null @@ -1,37 +0,0 @@ -const multer = require('multer'); -const { nanoid } = require('nanoid'); -const fs = require('fs-extra'); -const objstore = require('./objstore') -const config = require('../config') - - -const importStreamUploader = multer({storage: multer.memoryStorage()}); - -const importUploader = multer({storage: multer.diskStorage({ - destination: (req, file, cb) => { - const path = `/tmp/import/${nanoid(8)}` - fs.mkdirSync(path, {recursive: true}) - cb(null, path) - }, - filename: (req, file, cb) => { - cb(null, file.originalname) - }}) -}); - -const putStreamUploadToMinio = async (req) => { - await objstore.putStream(config.minio.importsBucket, req.file.originalname, req.file.buffer); -}; - -const putUploadToMinio = async (req) => { - await objstore.put(config.minio.importsBucket, req.file.filename, req.file.path); - console.log(req.file.destination); - await fs.unlink(req.file.path); - await fs.rmdir('/tmp/import', {recursive: true}); -}; - -module.exports = { - importStreamUploader, - importUploader, - putStreamUploadToMinio, - putUploadToMinio -}; \ No newline at end of file