Merge pull request #250 from Zephyrrus/feature/system_status_page

Feature/system status page

package.json

@@ -34,8 +34,8 @@
 		"node": ">=12.0.0"
 	},
 	"dependencies": {
-		"@mdi/font": "^5.3.45",
-		"@nuxtjs/axios": "^5.4.1",
+		"@mdi/font": "^5.8.55",
+		"@nuxtjs/axios": "^5.12.5",
 		"adm-zip": "^0.4.13",
 		"bcrypt": "^5.0.0",
 		"blake3": "^2.1.4",
@@ -47,24 +47,26 @@
 		"compression": "^1.7.2",
 		"cookie-universal-nuxt": "^2.0.14",
 		"cors": "^2.8.5",
+		"cron": "^1.8.2",
 		"dotenv": "^6.2.0",
 		"dumper.js": "^1.3.1",
 		"express": "^4.17.1",
 		"express-rate-limit": "^3.4.0",
 		"ffmpeg-probe": "^1.0.6",
 		"file-saver": "^2.0.1",
+		"file-type": "^16.1.0",
 		"fluent-ffmpeg": "^2.1.2",
 		"fs-jetpack": "^2.2.2",
 		"helmet": "^3.15.1",
 		"imagesloaded": "^4.1.4",
 		"jsonwebtoken": "^8.5.0",
-		"knex": "^0.16.3",
+		"knex": "^0.21.15",
 		"masonry-layout": "^4.2.2",
 		"moment": "^2.24.0",
 		"morgan": "^1.10.0",
 		"multer": "^1.4.1",
 		"mysql": "^2.16.0",
-		"nuxt": "2.12.2",
+		"nuxt": "^2.12.2",
 		"nuxt-dropzone": "^0.2.8",
 		"pg": "^7.8.1",
 		"qoa": "^0.2.0",
@@ -74,6 +76,7 @@
 		"serve-static": "^1.13.2",
 		"sharp": "^0.27.0",
 		"sqlite3": "^5.0.0",
+		"systeminformation": "^4.34.5",
 		"uuid": "^3.3.2",
 		"v-clipboard": "^2.2.1",
 		"vue-axios": "^2.1.4",

src/api/database/migrations/20190221225812_initialMigration.js

@@ -80,6 +80,7 @@ exports.up = async knex => {
 		table.timestamp('createdAt');
 	});
 };
+
 exports.down = async knex => {
 	await knex.schema.dropTableIfExists('users');
 	await knex.schema.dropTableIfExists('albums');

src/api/database/migrations/20201227023216_addUniques.js

@@ -28,6 +28,7 @@ exports.up = async knex => {
 		table.unique(['fileId', 'tagId']);
 	});
 };
-exports.down = async knex => {
+
+exports.down = async () => {
 	// Nothing
 };

src/api/database/migrations/20210105222742_addStatisticsTable.js

@@ -0,0 +1,16 @@
+
+exports.up = async knex => {
+	await knex.schema.createTable('statistics', table => {
+		table.increments();
+		table.integer('batchId');
+		table.string('type');
+		table.json('data');
+		table.timestamp('createdAt');
+
+		table.unique(['batchId', 'type']);
+	});
+};
+
+exports.down = async knex => {
+	await knex.schema.dropTableIfExists('statistics');
+};

src/api/routes/admin/userDemote.js

@@ -14,7 +14,8 @@ class userDemote extends Route {
 		try {
 			await db.table('users')
 				.where({ id })
-				.update({ isAdmin: false });
+				.update({ isAdmin: false })
+				.wasMutated();
 		} catch (error) {
 			return super.error(res, error);
 		}

src/api/routes/admin/userDisable.js

@@ -14,7 +14,8 @@ class userDisable extends Route {
 		try {
 			await db.table('users')
 				.where({ id })
-				.update({ enabled: false });
+				.update({ enabled: false })
+				.wasMutated();
 		} catch (error) {
 			return super.error(res, error);
 		}

src/api/routes/admin/userEnable.js

@@ -14,7 +14,8 @@ class userEnable extends Route {
 		try {
 			await db.table('users')
 				.where({ id })
-				.update({ enabled: true });
+				.update({ enabled: true })
+				.wasMutated();
 		} catch (error) {
 			return super.error(res, error);
 		}

src/api/routes/albums/albumDELETE.js

@@ -26,7 +26,8 @@ class albumDELETE extends Route {
 			await db.table('albumsLinks').where({ albumId: id }).delete();
 
 			// Delete any album links created for this album
-			await db.table('links').where({ albumId: id }).delete();
+			await db.table('links').where({ albumId: id }).delete()
+				.wasMutated();
 
 			return res.json({ message: 'The album was deleted successfully' });
 		} catch (error) {

src/api/routes/albums/albumPOST.js

@@ -28,7 +28,11 @@ class albumPOST extends Route {
 			editedAt: now
 		};
 
-		const dbRes = await db.table('albums').insert(insertObj);
+		const dbRes = await db
+			.table('albums')
+			.insert(insertObj)
+			.returning('id')
+			.wasMutated();
 
 		insertObj.id = dbRes.pop();
 

src/api/routes/albums/albumPurgeDELETE.js

@@ -18,7 +18,8 @@ class albumDELETE extends Route {
 
 		try {
 			await Util.deleteAllFilesFromAlbum(id);
-			await db.table('albums').where({ id }).delete();
+			await db.table('albums').where({ id }).delete()
+				.wasMutated();
 			return res.json({ message: 'The album was deleted successfully' });
 		} catch (error) {
 			return super.error(res, error);

src/api/routes/albums/albumZipGET.js

@@ -74,7 +74,8 @@ class albumGET extends Route {
 			Util.createZip(filesToZip, album);
 			await db.table('albums')
 				.where('id', link.albumId)
-				.update('zippedAt', db.fn.now());
+				.update('zippedAt', db.fn.now())
+				.wasMutated();
 
 			const filePath = path.join(__dirname, '../../../../', process.env.UPLOAD_FOLDER, 'zips', `${album.userId}-${album.id}.zip`);
 			const fileName = `${process.env.SERVICE_NAME}-${identifier}.zip`;

src/api/routes/albums/link/linkPOST.js

@@ -63,7 +63,7 @@ class linkPOST extends Route {
 				expiresAt: null,
 				views: 0
 			};
-			await db.table('links').insert(insertObj);
+			await db.table('links').insert(insertObj).wasMutated();
 
 			return res.json({
 				message: 'The link was created successfully',

src/api/routes/auth/registerPOST.js

@@ -55,7 +55,7 @@ class registerPOST extends Route {
 			editedAt: now,
 			enabled: true,
 			isAdmin: false
-		});
+		}).wasMutated();
 		return res.json({ message: 'The account was created successfully' });
 	}
 }

src/api/routes/files/albumAddPOST.js

@@ -18,7 +18,7 @@ class albumAddPOST extends Route {
 
 		try {
 			await db.table('albumsFiles')
-				.insert({ fileId, albumId });
+				.insert({ fileId, albumId }).wasMutated();
 		} catch (error) {
 			return super.error(res, error);
 		}

src/api/routes/files/albumDelPOST.js

@@ -19,7 +19,8 @@ class albumDelPOST extends Route {
 		try {
 			await db.table('albumsFiles')
 				.where({ fileId, albumId })
-				.delete();
+				.delete()
+				.wasMutated();
 		} catch (error) {
 			return super.error(res, error);
 		}

src/api/routes/files/tagAddBatchPOST.js

@@ -20,7 +20,7 @@ class tagAddBatchPOST extends Route {
 			try {
 				const tag = await db.table('tags').where({ name: tagName, userId: user.id }).first();
 				if (!tag) throw new Error('Tag doesn\'t exist in the database');
-				await db.table('fileTags').insert({ fileId, tagId: tag.id });
+				await db.table('fileTags').insert({ fileId, tagId: tag.id }).wasMutated();
 
 				addedTags.push(tag);
 			} catch (e) {

src/api/routes/files/tagAddPOST.js

@@ -20,7 +20,7 @@ class tagAddPOST extends Route {
 		if (!tag) return res.status(400).json({ message: 'Tag doesn\'t exist. ' });
 
 		try {
-			await db.table('fileTags').insert({ fileId, tagId: tag.id });
+			await db.table('fileTags').insert({ fileId, tagId: tag.id }).wasMutated();
 		} catch (error) {
 			return super.error(res, error);
 		}

src/api/routes/files/tagDelPOST.js

@@ -22,7 +22,8 @@ class tagDelPost extends Route {
 		try {
 			await db.table('fileTags')
 				.where({ fileId, tagId: tag.id })
-				.delete();
+				.delete()
+				.wasMutated();
 		} catch (error) {
 			return super.error(res, error);
 		}

src/api/routes/service/statsGET.js

@@ -0,0 +1,48 @@
+const Route = require('../../structures/Route');
+const StatsGenerator = require('../../utils/StatsGenerator');
+const moment = require('moment');
+
+// Thank you Bobby for the stats code https://github.com/BobbyWibowo/lolisafe/blob/safe.fiery.me/controllers/utilsController.js
+class filesGET extends Route {
+	constructor() {
+		super('/service/statistics', 'get', { adminOnly: true });
+	}
+
+	async run(req, res, db) {
+		const cachedStats = await db('statistics')
+			.select('type', 'data', 'batchId', 'createdAt')
+			.where('batchId', '=', db('statistics').max('batchId'));
+
+		let stats = cachedStats.reduce((acc, { type, data, createdAt }) => {
+			try {
+				// pg returns json, sqlite retuns a string...
+				if (typeof data === 'string' || data instanceof String) {
+					acc[type] = JSON.parse(data);
+				} else {
+					acc[type] = data;
+				}
+
+				acc[type].meta = {
+					cached: true,
+					generatedOn: moment(createdAt).format('MMMM Do YYYY, h:mm:ss a z'), // pg returns this as a date, sqlite3 returns an unix timestamp :<
+					type: StatsGenerator.Type.HIDDEN
+				};
+			} catch (e) {
+				console.error(e);
+			}
+
+			return acc;
+		}, {});
+
+		stats = { ...stats, ...(await StatsGenerator.getMissingStats(db, Object.keys(stats))) };
+
+		const ordered = StatsGenerator.keyOrder.reduce((acc, k) => {
+			acc[k] = stats[k];
+			return acc;
+		}, {});
+
+		return res.json({ statistics: ordered });
+	}
+}
+
+module.exports = filesGET;

src/api/routes/tags/tagDELETE.js

@@ -26,7 +26,8 @@ class tagDELETE extends Route {
 			/*
 				Delete the tag
 			*/
-			await db.table('tags').where({ id }).delete();
+			await db.table('tags').where({ id }).delete()
+				.wasMutated();
 			return res.json({ message: 'The tag was deleted successfully', data: tag });
 		} catch (error) {
 			return super.error(res, error);

src/api/routes/tags/tagPOST.js

@@ -25,7 +25,10 @@ class tagPOST extends Route {
 			editedAt: now
 		};
 
-		const dbRes = await db.table('tags').insert(insertObj);
+		const dbRes = await db.table('tags')
+			.insert(insertObj)
+			.returning('id')
+			.wasMutated();
 
 		insertObj.id = dbRes.pop();
 

src/api/routes/uploads/uploadPOST.js

@@ -1,6 +1,7 @@
 const path = require('path');
 const jetpack = require('fs-jetpack');
 const multer = require('multer');
+
 const Util = require('../../utils/Util');
 const Route = require('../../structures/Route');
 const multerStorage = require('../../utils/multerStorage');
@@ -10,6 +11,22 @@ const chunkedUploadsTimeout = 1800000;
 const chunksDir = path.join(__dirname, '../../../../', process.env.UPLOAD_FOLDER, 'chunks');
 const uploadDir = path.join(__dirname, '../../../../', process.env.UPLOAD_FOLDER);
 
+
+const cleanUpChunks = async (uuid, onTimeout) => {
+	// Remove tmp file
+	await jetpack.removeAsync(path.join(chunksData[uuid].root, chunksData[uuid].filename))
+		.catch(error => {
+			if (error.code !== 'ENOENT') console.error(error);
+		});
+
+	// Remove UUID dir
+	await jetpack.removeAsync(chunksData[uuid].root);
+
+	// Delete cached chunks data
+	if (!onTimeout) chunksData[uuid].clearTimeout();
+	delete chunksData[uuid];
+};
+
 class ChunksData {
 	constructor(uuid, root) {
 		this.uuid = uuid;
@@ -134,7 +151,7 @@ const uploadFile = async (req, res) => {
 	// If the uploaded file is a chunk then just say that it was a success
 	const uuid = req.body.uuid;
 	if (chunksData[uuid] !== undefined) {
-		req.files.forEach(file => {
+		req.files.forEach(() => {
 			chunksData[uuid].chunks++;
 		});
 		res.json({ success: true });
@@ -143,13 +160,13 @@ const uploadFile = async (req, res) => {
 
 	const infoMap = req.files.map(file => ({
 		path: path.join(uploadDir, file.filename),
-		data: file
+		data: { ...file, mimetype: Util.getMimeFromType(file.fileType) || file.mimetype || '' }
 	}));
 
 	return infoMap[0];
 };
 
-const finishChunks = async (req, res) => {
+const finishChunks = async req => {
 	const check = file => typeof file.uuid !== 'string' ||
     !chunksData[file.uuid] ||
     chunksData[file.uuid].chunks < 2;
@@ -172,6 +189,8 @@ const finishChunks = async (req, res) => {
 			*/
 
 			file.extname = typeof file.original === 'string' ? Util.getExtension(file.original) : '';
+			file.fileType = chunksData[file.uuid].fileType;
+			file.mimetype = Util.getMimeFromType(chunksData[file.uuid].fileType) || file.mimetype || '';
 
 			if (Util.isExtensionBlocked(file.extname)) {
 				throw `${file.extname ? `${file.extname.substr(1).toUpperCase()} files` : 'Files with no extension'} are not permitted.`; // eslint-disable-line no-throw-literal
@@ -201,7 +220,7 @@ const finishChunks = async (req, res) => {
 				filename: name,
 				originalname: file.original || '',
 				extname: file.extname,
-				mimetype: file.type || '',
+				mimetype: file.mimetype,
 				size: file.size,
 				hash
 			};
@@ -228,21 +247,6 @@ const finishChunks = async (req, res) => {
 	}
 };
 
-const cleanUpChunks = async (uuid, onTimeout) => {
-	// Remove tmp file
-	await jetpack.removeAsync(path.join(chunksData[uuid].root, chunksData[uuid].filename))
-		.catch(error => {
-			if (error.code !== 'ENOENT') console.error(error);
-		});
-
-	// Remove UUID dir
-	await jetpack.removeAsync(chunksData[uuid].root);
-
-	// Delete cached chunks data
-	if (!onTimeout) chunksData[uuid].clearTimeout();
-	delete chunksData[uuid];
-};
-
 class uploadPOST extends Route {
 	constructor() {
 		super('/upload', 'post', {

src/api/structures/Database.js

@@ -0,0 +1,54 @@
+const nodePath = require('path');
+const Knex = require('knex');
+
+// eslint-disable-next-line func-names
+Knex.QueryBuilder.extend('wasMutated', function() {
+	this.client.config.userParams.lastMutationTime = Date.now();
+	return this;
+});
+
+const db = Knex({
+	client: process.env.DB_CLIENT,
+	connection: {
+		host: process.env.DB_HOST,
+		user: process.env.DB_USER,
+		password: process.env.DB_PASSWORD,
+		database: process.env.DB_DATABASE,
+		filename: nodePath.join(__dirname, '../../../database/database.sqlite')
+	},
+	postProcessResponse: result => {
+		/*
+			Fun fact: Depending on the database used by the user and given that I don't want
+			to force a specific database for everyone because of the nature of this project,
+			some things like different data types for booleans need to be considered like in
+			the implementation below where sqlite returns 1 and 0 instead of true and false.
+		*/
+		const booleanFields = ['enabled', 'enableDownload', 'isAdmin', 'nsfw'];
+
+		const processResponse = row => {
+			Object.keys(row).forEach(key => {
+				if (booleanFields.includes(key)) {
+					if (row[key] === 0) row[key] = false;
+					else if (row[key] === 1) row[key] = true;
+				}
+			});
+			return row;
+		};
+
+		if (Array.isArray(result)) return result.map(row => processResponse(row));
+		if (typeof result === 'object') return processResponse(result);
+		return result;
+	},
+	useNullAsDefault: process.env.DB_CLIENT === 'sqlite3',
+	log: {
+		warn: msg => {
+			if (typeof msg === 'string' && msg.startsWith('.returning()')) return;
+			console.warn(msg);
+		}
+	},
+	userParams: {
+		lastMutationTime: null
+	}
+});
+
+module.exports = db;

src/api/structures/Route.js

@@ -1,39 +1,5 @@
-const nodePath = require('path');
 const JWT = require('jsonwebtoken');
-const db = require('knex')({
-	client: process.env.DB_CLIENT,
-	connection: {
-		host: process.env.DB_HOST,
-		user: process.env.DB_USER,
-		password: process.env.DB_PASSWORD,
-		database: process.env.DB_DATABASE,
-		filename: nodePath.join(__dirname, '../../../database/database.sqlite')
-	},
-	postProcessResponse: result => {
-		/*
-			Fun fact: Depending on the database used by the user and given that I don't want
-			to force a specific database for everyone because of the nature of this project,
-			some things like different data types for booleans need to be considered like in
-			the implementation below where sqlite returns 1 and 0 instead of true and false.
-		*/
-		const booleanFields = ['enabled', 'enableDownload', 'isAdmin', 'nsfw'];
-
-		const processResponse = row => {
-			Object.keys(row).forEach(key => {
-				if (booleanFields.includes(key)) {
-					if (row[key] === 0) row[key] = false;
-					else if (row[key] === 1) row[key] = true;
-				}
-			});
-			return row;
-		};
-
-		if (Array.isArray(result)) return result.map(row => processResponse(row));
-		if (typeof result === 'object') return processResponse(result);
-		return result;
-	},
-	useNullAsDefault: process.env.DB_CLIENT === 'sqlite3'
-});
+const db = require('./Database');
 const moment = require('moment');
 const log = require('../utils/Log');
 

src/api/structures/Server.js

@@ -14,8 +14,11 @@ const jetpack = require('fs-jetpack');
 const path = require('path');
 const morgan = require('morgan');
 const rfs = require('rotating-file-stream');
+const CronJob = require('cron').CronJob;
 const log = require('../utils/Log');
 
+const Util = require('../utils/Util');
+
 // eslint-disable-next-line no-unused-vars
 const rateLimiter = new RateLimit({
 	windowMs: parseInt(process.env.RATE_LIMIT_WINDOW, 10),
@@ -55,6 +58,9 @@ class Server {
 		// Serve the uploads
 		this.server.use(express.static(path.join(__dirname, '../../../uploads')));
 		this.routesFolder = path.join(__dirname, '../routes');
+
+		// Save the cron job instances in case we want to stop them later
+		this.jobs = {};
 	}
 
 	registerAllTheRoutes() {
@@ -95,6 +101,11 @@ class Server {
 		});
 	}
 
+	createJobs() {
+		// TODO: move into the database config. (we can just show the crontab line for start, later on we can add dropdowns and stuff)
+		this.jobs.stats = new CronJob('0 0 * * * *', Util.saveStatsToDb, null, true);
+	}
+
 	start() {
 		jetpack.dir('uploads/chunks');
 		jetpack.dir('uploads/thumbs/square');
@@ -105,6 +116,8 @@ class Server {
 			log.success(`Backend ready and listening on port ${this.port}`);
 		});
 		server.setTimeout(600000);
+
+		this.createJobs();
 	}
 }
 

src/api/utils/StatsGenerator.js

@@ -0,0 +1,225 @@
+const si = require('systeminformation');
+
+class StatsGenerator {
+	// symbols would be better because they're unique, but harder to serialize them
+	static Type = Object.freeze({
+		// should contain key value: number
+		TIME: 'time',
+		// should contain key value: number
+		BYTE: 'byte',
+		// should contain key value: { used: number, total: number }
+		BYTE_USAGE: 'byteUsage',
+		// should contain key data: Array<{ key: string, value: number | string }>
+		// and optionally a count/total
+		DETAILED: 'detailed',
+		// hidden type should be skipped during iteration, can contain anything
+		// these should be treated on a case by case basis on the frontend
+		HIDDEN: 'hidden'
+	});
+
+	static statGenerators = {
+		system: StatsGenerator.getSystemInfo,
+		fileSystems: StatsGenerator.getFileSystemsInfo,
+		uploads: StatsGenerator.getUploadsInfo,
+		users: StatsGenerator.getUsersInfo,
+		albums: StatsGenerator.getAlbumStats
+	};
+
+	static keyOrder = Object.keys(StatsGenerator.statGenerators);
+
+	static async getSystemInfo() {
+		const os = await si.osInfo();
+
+		const currentLoad = await si.currentLoad();
+		const mem = await si.mem();
+		const time = si.time();
+		const nodeUptime = process.uptime();
+
+		return {
+			'Platform': `${os.platform} ${os.arch}`,
+			'Distro': `${os.distro} ${os.release}`,
+			'Kernel': os.kernel,
+			'CPU Load': `${currentLoad.currentload.toFixed(1)}%`,
+			'CPUs Load': currentLoad.cpus.map(cpu => `${cpu.load.toFixed(1)}%`).join(', '),
+			'System Memory': {
+				value: {
+					used: mem.active,
+					total: mem.total
+				},
+				type: StatsGenerator.Type.BYTE_USAGE
+			},
+			'Memory Usage': {
+				value: process.memoryUsage().rss,
+				type: StatsGenerator.Type.BYTE
+			},
+			'System Uptime': {
+				value: time.uptime,
+				type: StatsGenerator.Type.TIME
+			},
+			'Node.js': `${process.versions.node}`,
+			'Service Uptime': {
+				value: Math.floor(nodeUptime),
+				type: StatsGenerator.Type.TIME
+			}
+		};
+	}
+
+	static async getFileSystemsInfo() {
+		const stats = {};
+
+		const fsSize = await si.fsSize();
+		for (const fs of fsSize) {
+			stats[`${fs.fs} (${fs.type}) on ${fs.mount}`] = {
+				value: {
+					total: fs.size,
+					used: fs.used
+				},
+				type: StatsGenerator.Type.BYTE_USAGE
+			};
+		}
+
+		return stats;
+	}
+
+	static async getUploadsInfo(db) {
+		const stats = {
+			'Total': 0,
+			'Images': 0,
+			'Videos': 0,
+			'Others': {
+				data: {},
+				count: 0,
+				type: StatsGenerator.Type.DETAILED
+			},
+			'Size in DB': {
+				value: 0,
+				type: StatsGenerator.Type.BYTE
+			}
+		};
+
+		const getFilesCountAndSize = async () => {
+			const uploads = await db.table('files').select('size');
+
+			return {
+				'Total': uploads.length,
+				'Size in DB': {
+					value: uploads.reduce((acc, upload) => acc + parseInt(upload.size, 10), 0),
+					type: StatsGenerator.Type.BYTE
+				}
+			};
+		};
+
+		const getImagesCount = async () => {
+			const Images = await db.table('files')
+				.where('type', 'like', `image/%`)
+				.count('id as count')
+				.then(rows => rows[0].count);
+
+			return { Images };
+		};
+
+		const getVideosCount = async () => {
+			const Videos = await db.table('files')
+				.where('type', 'like', `video/%`)
+				.count('id as count')
+				.then(rows => rows[0].count);
+
+			return { Videos };
+		};
+
+		const getOthersCount = async () => {
+			// rename to key, value from type, count
+			const data = await db.table('files')
+				.select('type as key')
+				.count('id as value')
+				.whereNot('type', 'like', `image/%`)
+				.whereNot('type', 'like', `video/%`)
+				.groupBy('key')
+				.orderBy('value', 'desc');
+
+			const count = data.reduce((acc, val) => acc + val.value, 0);
+
+			return {
+				Others: {
+					data,
+					count,
+					type: StatsGenerator.Type.DETAILED
+				}
+			};
+		};
+
+		const result = await Promise.all([getFilesCountAndSize(), getImagesCount(), getVideosCount(), getOthersCount()]);
+
+		return { ...stats, ...Object.assign({}, ...result) };
+	}
+
+	static async getUsersInfo(db) {
+		const stats = {
+			Total: 0,
+			Admins: 0,
+			Disabled: 0
+		};
+
+		const users = await db.table('users');
+		stats.Total = users.length;
+
+		 for (const user of users) {
+			if (!user.enabled) {
+				stats.Disabled++;
+			}
+
+			if (user.isAdmin) {
+				stats.Admins++;
+			}
+		}
+
+		return stats;
+	}
+
+	static async getAlbumStats(db) {
+		const stats = {
+			'Total': 0,
+			'NSFW': 0,
+			'Generated archives': 0,
+			'Generated identifiers': 0,
+			'Files in albums': 0
+		};
+
+		const albums = await db.table('albums');
+		stats.Total = albums.length;
+		for (const album of albums) {
+			if (album.nsfw) stats.NSFW++;
+			if (album.zipGeneratedAt) stats['Generated archives']++; // XXX: Bobby checks each after if a zip really exists on the disk. Is it really needed?
+		}
+
+		stats['Generated identifiers'] = await db.table('albumsLinks').count('id as count').then(rows => rows[0].count);
+		stats['Files in albums'] = await db.table('albumsFiles')
+			.whereNotNull('albumId')
+			.count('id as count')
+			.then(rows => rows[0].count);
+
+		return stats;
+	}
+
+	static async getStats(db) {
+		const res = {};
+
+		for (const [name, funct] of Object.entries(StatsGenerator.statGenerators)) {
+			res[name] = await funct(db);
+		}
+
+		return res;
+	}
+
+	static async getMissingStats(db, existingStats) {
+		const res = {};
+
+		for (const [name, funct] of Object.entries(StatsGenerator.statGenerators)) {
+			if (existingStats.indexOf(name) === -1)	res[name] = await funct(db);
+		}
+
+		return res;
+	}
+}
+
+module.exports = StatsGenerator;

src/api/utils/Util.js

@@ -3,27 +3,20 @@ const jetpack = require('fs-jetpack');
 const randomstring = require('randomstring');
 const path = require('path');
 const JWT = require('jsonwebtoken');
-const db = require('knex')({
-	client: process.env.DB_CLIENT,
-	connection: {
-		host: process.env.DB_HOST,
-		user: process.env.DB_USER,
-		password: process.env.DB_PASSWORD,
-		database: process.env.DB_DATABASE,
-		filename: path.join(__dirname, '../../../database/database.sqlite')
-	},
-	useNullAsDefault: process.env.DB_CLIENT === 'sqlite'
-});
+const db = require('../structures/Database');
 const moment = require('moment');
 const Zip = require('adm-zip');
 const uuidv4 = require('uuid/v4');
 
 const log = require('./Log');
 const ThumbUtil = require('./ThumbUtil');
+const StatsGenerator = require('./StatsGenerator');
 
 const blockedExtensions = process.env.BLOCKED_EXTENSIONS.split(',');
 const preserveExtensions = ['.tar.gz', '.tar.z', '.tar.bz2', '.tar.lzma', '.tar.lzo', '.tar.xz'];
 
+let statsLastSavedTime = null;
+
 class Util {
 	static uploadPath = path.join(__dirname, '../../../', process.env.UPLOAD_FOLDER);
 
@@ -35,6 +28,10 @@ class Util {
 		return blockedExtensions.includes(extension);
 	}
 
+	static getMimeFromType(fileTypeMimeObj) {
+		return fileTypeMimeObj ? fileTypeMimeObj.mime : undefined;
+	}
+
 	static constructFilePublicLink(file) {
 		/*
 			TODO: This wont work without a reverse proxy serving both
@@ -102,7 +99,8 @@ class Util {
 				await db
 					.table('files')
 					.where('name', filename)
-					.delete();
+					.delete()
+					.wasMutated();
 			}
 		} catch (error) {
 			log.error(`There was an error removing the file < ${filename} >`);
@@ -225,6 +223,7 @@ class Util {
 
 	static async storeFileToDb(req, res, user, file, db) {
 		const dbFile = await db.table('files')
+			// eslint-disable-next-line func-names
 			.where(function() {
 				if (user === undefined) {
 					this.whereNull('userId');
@@ -259,9 +258,9 @@ class Util {
 
 		let fileId;
 		if (process.env.DB_CLIENT === 'sqlite3') {
-			fileId = await db.table('files').insert(data);
+			fileId = await db.table('files').insert(data).wasMutated();
 		} else {
-			fileId = await db.table('files').insert(data, 'id');
+			fileId = await db.table('files').insert(data, 'id').wasMutated();
 		}
 
 		return {
@@ -275,7 +274,7 @@ class Util {
 
 		const now = moment.utc().toDate();
 		try {
-			await db.table('albumsFiles').insert({ albumId, fileId: insertedId[0] });
+			await db.table('albumsFiles').insert({ albumId, fileId: insertedId[0] }).wasMutated();
 			await db.table('albums').where('id', albumId).update('editedAt', now);
 		} catch (error) {
 			console.error(error);
@@ -311,6 +310,42 @@ class Util {
 
 		return extname + multi;
 	}
+
+	// TODO: Allow choosing what to save to db and what stats we care about in general
+	// TODO: if a stat is not saved to db but selected to be shows on the dashboard, it will be generated during the request
+	static async saveStatsToDb(force) {
+		// If there were no changes since the instance started, don't generate new stats
+		// OR
+		// if we alredy saved a stats to the db, and there were no new changes to the db since then
+		// skip generating and saving new stats.
+		if (!force &&
+			(!db.userParams.lastMutationTime ||
+				(statsLastSavedTime && statsLastSavedTime > db.userParams.lastMutationTime)
+			)
+		) {
+			return;
+		}
+
+		const now = moment.utc().toDate();
+		const stats = await StatsGenerator.getStats(db);
+
+		let batchId = 1;
+
+		const res = (await db('statistics').max({ lastBatch: 'batchId' }))[0];
+		if (res && res.lastBatch) {
+			batchId = res.lastBatch + 1;
+		}
+
+		try {
+			for (const [type, data] of Object.entries(stats)) {
+				await db.table('statistics').insert({ type, data: JSON.stringify(data), createdAt: now, batchId });
+			}
+
+			statsLastSavedTime = now.getTime();
+		} catch (error) {
+			console.error(error);
+		}
+	}
 }
 
 module.exports = Util;

src/api/utils/multerStorage.js

@@ -2,13 +2,14 @@ const fs = require('fs');
 const path = require('path');
 const blake3 = require('blake3');
 const jetpack = require('fs-jetpack');
+const FileType = require('file-type');
 
 function DiskStorage(opts) {
 	this.getFilename = opts.filename;
 
 	if (typeof opts.destination === 'string') {
 		jetpack.dir(opts.destination);
-		this.getDestination = function($0, $1, cb) { cb(null, opts.destination); };
+		this.getDestination = ($0, $1, cb) => { cb(null, opts.destination); };
 	} else {
 		this.getDestination = opts.destination;
 	}
@@ -52,25 +53,44 @@ DiskStorage.prototype._handleFile = function _handleFile(req, file, cb) {
 			file.stream.on('data', d => hash.update(d));
 
 			if (file._isChunk) {
-				file.stream.on('end', () => {
-					cb(null, {
-						destination,
-						filename,
-						path: finalPath
+				if (file._chunksData.chunks === 0) {
+					FileType.stream(file.stream).then(ftStream => {
+						file._chunksData.fileType = ftStream.fileType;
+						file.stream.on('end', () => {
+							cb(null, {
+								destination,
+								filename,
+								path: finalPath,
+								fileType: file._chunksData.fileType
+							});
+						});
+						ftStream.pipe(outStream, { end: false });
 					});
-				});
-				file.stream.pipe(outStream, { end: false });
+				} else {
+					file.stream.on('end', () => {
+						cb(null, {
+							destination,
+							filename,
+							path: finalPath,
+							fileType: file._chunksData.fileType
+						});
+					});
+					file.stream.pipe(outStream, { end: false });
+				}
 			} else {
-				outStream.on('finish', () => {
-					cb(null, {
-						destination,
-						filename,
-						path: finalPath,
-						size: outStream.bytesWritten,
-						hash: hash.digest('hex')
+				FileType.stream(file.stream).then(ftStream => {
+					outStream.on('finish', () => {
+						cb(null, {
+							destination,
+							filename,
+							path: finalPath,
+							size: outStream.bytesWritten,
+							hash: hash.digest('hex'),
+							fileType: ftStream.fileType
+						});
 					});
+					ftStream.pipe(outStream);
 				});
-				file.stream.pipe(outStream);
 			}
 		});
 	});
@@ -86,6 +106,4 @@ DiskStorage.prototype._removeFile = function _removeFile(req, file, cb) {
 	fs.unlink(path, cb);
 };
 
-module.exports = function(opts) {
-	return new DiskStorage(opts);
-};
+module.exports = opts => new DiskStorage(opts);

src/site/components/album/AlbumDetails.vue

@@ -101,7 +101,13 @@
 						<div class="level-item">
 							<b-switch
 								:value="nsfw"
-								@input="toggleNsfw()" />
+								:rounded="false"
+								type="is-warning"
+								class="has-text-light"
+								left-label
+								@input="toggleNsfw()">
+								NSFW
+							</b-switch>
 						</div>
 						<div class="level-item">
 							<button

src/site/components/footer/Footer.vue

@@ -72,6 +72,9 @@ export default {
 <style lang="scss" scoped>
 	@import '~/assets/styles/_colors.scss';
 	footer {
+		pointer-events: none;
+		touch-action: none;
+
 		@media screen and (min-width: 1025px) {
 			position: fixed;
 			bottom: 0;
@@ -84,6 +87,9 @@ export default {
 
 		.container {
 			.column {
+				pointer-events: auto;
+				touch-action: auto;
+
 				text-align: center;
 				@media screen and (min-width: 1025px) {
 					margin-right: 2rem;

src/site/components/sidebar/Sidebar.vue

@@ -29,7 +29,7 @@
 				</template>
 				<b-menu-item icon="account" label="Users" tag="nuxt-link" to="/dashboard/admin/users" exact />
 				<b-menu-item icon="cog-outline" label="Settings" tag="nuxt-link" to="/dashboard/admin/settings" exact />
-				<!--<b-menu-item icon="chart-line" label="Statistics" tag="nuxt-link" to="/dashboard/admin/statistics" exact />-->
+				<b-menu-item icon="chart-line" label="Statistics" tag="nuxt-link" to="/dashboard/admin/statistics" exact />
 			</b-menu-item>
 			<b-menu-item
 				class="item"

src/site/components/statistics/byte.vue

@@ -1,6 +1,6 @@
 <template>
 	<div>
-		<div class="columns">
+		<div class="columns is-mobile">
 			<div class="column is-2">
 				{{ title }}
 			</div>

src/site/components/statistics/byteUsage.vue

@@ -1,6 +1,6 @@
 <template>
 	<div>
-		<div class="columns">
+		<div class="columns is-mobile">
 			<div class="column is-2">
 				{{ title }}
 			</div>

src/site/components/statistics/detailed.vue

@@ -1,11 +1,11 @@
 <template>
 	<div>
-		<div class="columns">
+		<div class="columns is-mobile">
 			<div class="column is-2">
 				{{ title }}
 			</div>
 			<div class="column">
-				<b-table :data="data || []" :mobile-cards="true">
+				<b-table v-if="data.length" :data="data || []" :mobile-cards="true" narrowed class="details">
 					<b-table-column v-slot="props" field="type" label="Type">
 						{{ props.row.key }}
 					</b-table-column>
@@ -13,6 +13,9 @@
 						{{ props.row.value }}
 					</b-table-column>
 				</b-table>
+				<template v-else>
+					-
+				</template>
 			</div>
 		</div>
 	</div>
@@ -31,3 +34,13 @@ export default {
 	}
 };
 </script>
+<style lang="scss" scoped>
+	.details ::v-deep .table-wrapper {
+		box-shadow: none;
+
+		.table {
+			border-radius: unset;
+			background: #2A2E3C;
+		}
+	}
+</style>

src/site/components/statistics/generic.vue

@@ -1,6 +1,6 @@
 <template>
 	<div>
-		<div class="columns">
+		<div class="columns is-mobile">
 			<div class="column is-2">
 				{{ title }}
 			</div>

src/site/components/statistics/time.vue

@@ -1,6 +1,6 @@
 <template>
 	<div>
-		<div class="columns">
+		<div class="columns is-mobile">
 			<div class="column is-2">
 				{{ title }}
 			</div>

src/site/pages/dashboard/admin/statistics.vue

@@ -14,8 +14,11 @@
 						<div :key="category"
 							class="stats-container">
 							<h2 class="title">
-								{{ category }}
+								{{ category }} <span v-if="stats[category].meta" class="is-size-7 is-pulled-right is-family-monospace has-text-grey-light">
+									generated on {{ stats[category].meta.generatedOn }}
+								</span>
 							</h2>
+
 							<template v-for="item in Object.keys(stats[category])">
 								<!-- If it's plain text or a number, just print it -->
 								<template v-if="typeof stats[category][item] === 'string' || typeof stats[category][item] === 'number'">
@@ -25,7 +28,7 @@
 								</template>
 
 								<!-- If it's an object then we need to do some magic -->
-								<template v-else-if="typeof stats[category][item] === 'object'">
+								<template v-else-if="typeof stats[category][item] === 'object' && stats[category][item].type !== 'hidden'">
 									<byteUsage v-if="stats[category][item].type === 'byteUsage'"
 										:key="item"
 										:title="item"

src/site/pages/dashboard/albums/index.vue

@@ -105,7 +105,7 @@ export default {
 
 	div.search-container {
 		padding: 1rem 2rem;
-		background-color: $base-2;
+		// background-color: $base-2;
 	}
 
 	div.column > h2.subtitle { padding-top: 1px; }