From e97fee48441717f3b508ac855339d0fb8210be53 Mon Sep 17 00:00:00 2001 From: Pitu Date: Sun, 27 Dec 2020 04:27:56 +0900 Subject: [PATCH 1/2] Fixes chunked uploads not being saved to albums or thumbnails --- src/api/routes/uploads/chunksPOST.js | 21 +++++--- src/api/routes/uploads/uploadPOST.js | 61 +---------------------- src/api/utils/Util.js | 54 ++++++++++++++++++++ src/site/components/uploader/Uploader.vue | 4 ++ 4 files changed, 74 insertions(+), 66 deletions(-) diff --git a/src/api/routes/uploads/chunksPOST.js b/src/api/routes/uploads/chunksPOST.js index 061cfb0..ee95227 100644 --- a/src/api/routes/uploads/chunksPOST.js +++ b/src/api/routes/uploads/chunksPOST.js @@ -12,7 +12,10 @@ class uploadPOST extends Route { }); } - async run(req, res) { + async run(req, res, db) { + const user = await Util.isAuthorized(req); + if (!user && process.env.PUBLIC_MODE === 'false') return res.status(401).json({ message: 'Not authorized to use this resource' }); + const filename = Util.getUniqueFilename(randomstring.generate(32)); // console.log('Files', req.body.files); const info = { @@ -40,6 +43,7 @@ class uploadPOST extends Route { // Save some data info.name = `${filename}${ext || ''}`; info.url += `${filename}${ext || ''}`; + info.data = chunk; for (let i = 0; i < chunkDir.length; i++) { const dir = path.join(__dirname, @@ -54,15 +58,18 @@ class uploadPOST extends Route { await jetpack.removeAsync(chunkOutput); } + Util.generateThumbnails(info.name); + const insertedId = await Util.saveFileToDatabase(req, res, user, db, info, { + originalname: info.data.original, mimetype: info.data.type + }); + if (!insertedId) return res.status(500).json({ message: 'There was an error saving the file.' }); + info.deleteUrl = `${process.env.DOMAIN}/api/file/${insertedId[0]}`; + Util.saveFileToAlbum(db, req.headers.albumid, insertedId); + delete info.chunk; + return res.status(201).send({ message: 'Sucessfully merged the chunk(s).', ...info - /* - name: `${filename}${ext || ''}`, - size: exists.size, - url: `${process.env.DOMAIN}/${exists.name}`, - deleteUrl: `${process.env.DOMAIN}/api/file/${exists.id}` - */ }); } } diff --git a/src/api/routes/uploads/uploadPOST.js b/src/api/routes/uploads/uploadPOST.js index 5458d48..5d04da1 100644 --- a/src/api/routes/uploads/uploadPOST.js +++ b/src/api/routes/uploads/uploadPOST.js @@ -106,7 +106,7 @@ class uploadPOST extends Route { if (!remappedKeys || !remappedKeys.uuid) { Util.generateThumbnails(uploadedFile.name); - insertedId = await this.saveFileToDatabase(req, res, user, db, uploadedFile, file); + insertedId = await Util.saveFileToDatabase(req, res, user, db, uploadedFile, file); if (!insertedId) return res.status(500).json({ message: 'There was an error saving the file.' }); uploadedFile.deleteUrl = `${process.env.DOMAIN}/api/file/${insertedId[0]}`; @@ -114,7 +114,7 @@ class uploadPOST extends Route { If the upload had an album specified we make sure to create the relation and update the according timestamps.. */ - this.saveFileToAlbum(db, albumId, insertedId); + Util.saveFileToAlbum(db, albumId, insertedId); } uploadedFile = Util.constructFilePublicLink(uploadedFile); @@ -151,62 +151,6 @@ class uploadPOST extends Route { return exists; } - async saveFileToAlbum(db, albumId, insertedId) { - if (!albumId) return; - - const now = moment.utc().toDate(); - try { - await db.table('albumsFiles').insert({ albumId, fileId: insertedId[0] }); - await db.table('albums').where('id', albumId).update('editedAt', now); - } catch (error) { - console.error(error); - } - } - - async saveFileToDatabase(req, res, user, db, file, originalFile) { - /* - Save the upload information to the database - */ - const now = moment.utc().toDate(); - let insertedId = null; - try { - /* - This is so fucking dumb - */ - if (process.env.DB_CLIENT === 'sqlite3') { - insertedId = await db.table('files').insert({ - userId: user ? user.id : null, - name: file.name, - original: originalFile.originalname, - type: originalFile.mimetype || '', - size: file.size, - hash: file.hash, - ip: req.ip, - createdAt: now, - editedAt: now - }); - } else { - insertedId = await db.table('files').insert({ - userId: user ? user.id : null, - name: file.name, - original: originalFile.originalname, - type: originalFile.mimetype || '', - size: file.size, - hash: file.hash, - ip: req.ip, - createdAt: now, - editedAt: now - }, 'id'); - } - return insertedId; - } catch (error) { - console.error('There was an error saving the file to the database'); - console.error(error); - return null; - // return res.status(500).json({ message: 'There was an error uploading the file.' }); - } - } - _remapKeys(body) { const keys = Object.keys(body); if (keys.length) { @@ -217,7 +161,6 @@ class uploadPOST extends Route { } return body; } - return keys; } } diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 35d726e..905f217 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -226,6 +226,60 @@ class Util { } static generateThumbnails = ThumbUtil.generateThumbnails; + static async saveFileToDatabase(req, res, user, db, file, originalFile) { + /* + Save the upload information to the database + */ + const now = moment.utc().toDate(); + let insertedId = null; + try { + /* + This is so fucking dumb + */ + if (process.env.DB_CLIENT === 'sqlite3') { + insertedId = await db.table('files').insert({ + userId: user ? user.id : null, + name: file.name, + original: originalFile.originalname, + type: originalFile.mimetype || '', + size: file.size, + hash: file.hash, + ip: req.ip, + createdAt: now, + editedAt: now + }); + } else { + insertedId = await db.table('files').insert({ + userId: user ? user.id : null, + name: file.name, + original: originalFile.originalname, + type: originalFile.mimetype || '', + size: file.size, + hash: file.hash, + ip: req.ip, + createdAt: now, + editedAt: now + }, 'id'); + } + return insertedId; + } catch (error) { + console.error('There was an error saving the file to the database'); + console.error(error); + return null; + } + } + + static async saveFileToAlbum(db, albumId, insertedId) { + if (!albumId) return; + + const now = moment.utc().toDate(); + try { + await db.table('albumsFiles').insert({ albumId, fileId: insertedId[0] }); + await db.table('albums').where('id', albumId).update('editedAt', now); + } catch (error) { + console.error(error); + } + } } module.exports = Util; diff --git a/src/site/components/uploader/Uploader.vue b/src/site/components/uploader/Uploader.vue index 70acf91..f180546 100644 --- a/src/site/components/uploader/Uploader.vue +++ b/src/site/components/uploader/Uploader.vue @@ -184,6 +184,10 @@ export default { type: file.type, count: file.upload.totalChunkCount }] + }, { + headers: { + albumId: this.selectedAlbum ? this.selectedAlbum : null + } }); this.processResult(file, data); From aa7d2453171b3a596a1be6676eaf39cc93fe178f Mon Sep 17 00:00:00 2001 From: Pitu Date: Sun, 27 Dec 2020 04:48:03 +0900 Subject: [PATCH 2/2] feat: Add hash checking for chunked uploads --- src/api/routes/uploads/chunksPOST.js | 22 ++++++++++++++++++++++ src/api/routes/uploads/uploadPOST.js | 13 +------------ src/api/utils/Util.js | 11 +++++++++++ 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/api/routes/uploads/chunksPOST.js b/src/api/routes/uploads/chunksPOST.js index ee95227..9cf7338 100644 --- a/src/api/routes/uploads/chunksPOST.js +++ b/src/api/routes/uploads/chunksPOST.js @@ -58,6 +58,28 @@ class uploadPOST extends Route { await jetpack.removeAsync(chunkOutput); } + /* + If a file with the same hash and user is found, delete this + uploaded copy and return a link to the original + */ + info.hash = await Util.getFileHash(info.name); + let existingFile = await Util.checkIfFileExists(db, user, info.hash); + if (existingFile) { + existingFile = Util.constructFilePublicLink(existingFile); + res.json({ + message: 'Successfully uploaded the file.', + name: existingFile.name, + hash: existingFile.hash, + size: existingFile.size, + url: `${process.env.DOMAIN}/${existingFile.name}`, + deleteUrl: `${process.env.DOMAIN}/api/file/${existingFile.id}`, + repeated: true + }); + + return Util.deleteFile(info.name); + } + + // Otherwise generate thumbs and do the rest Util.generateThumbnails(info.name); const insertedId = await Util.saveFileToDatabase(req, res, user, db, info, { originalname: info.data.original, mimetype: info.data.type diff --git a/src/api/routes/uploads/uploadPOST.js b/src/api/routes/uploads/uploadPOST.js index 5d04da1..449999e 100644 --- a/src/api/routes/uploads/uploadPOST.js +++ b/src/api/routes/uploads/uploadPOST.js @@ -79,7 +79,7 @@ class uploadPOST extends Route { For this we need to wait until we have a filename so that we can delete the uploaded file. */ - const exists = await this.checkIfFileExists(db, user, hash); + const exists = await Util.checkIfFileExists(db, user, hash); if (exists) return this.fileExists(res, exists, filename); if (remappedKeys && remappedKeys.uuid) { @@ -140,17 +140,6 @@ class uploadPOST extends Route { return Util.deleteFile(filename); } - async checkIfFileExists(db, user, hash) { - const exists = await db.table('files') - .where(function() { // eslint-disable-line func-names - if (user) this.where('userId', user.id); - else this.whereNull('userId'); - }) - .where({ hash }) - .first(); - return exists; - } - _remapKeys(body) { const keys = Object.keys(body); if (keys.length) { diff --git a/src/api/utils/Util.js b/src/api/utils/Util.js index 905f217..e52fac2 100644 --- a/src/api/utils/Util.js +++ b/src/api/utils/Util.js @@ -108,6 +108,17 @@ class Util { return hash; } + static async checkIfFileExists(db, user, hash) { + const exists = await db.table('files') + .where(function() { // eslint-disable-line func-names + if (user) this.where('userId', user.id); + else this.whereNull('userId'); + }) + .where({ hash }) + .first(); + return exists; + } + static getFilenameFromPath(fullPath) { return fullPath.replace(/^.*[\\\/]/, ''); // eslint-disable-line no-useless-escape }