From 48e1f543368f389c96280a95dda7291b8784af31 Mon Sep 17 00:00:00 2001 From: jomo Date: Sun, 30 Nov 2014 02:48:54 +0100 Subject: [PATCH] add imagecleaner, fix #4 --- clean_images.sh | 24 --------------------- modules/clean_images.js | 44 +++++++++++++++++++++++++++++++++++++++ modules/config.example.js | 3 +++ modules/config.js | 3 +++ package.json | 3 ++- server.js | 7 ++++++- skins/faces/.gitkeep | 0 skins/helms/.gitkeep | 0 8 files changed, 58 insertions(+), 26 deletions(-) delete mode 100755 clean_images.sh create mode 100644 modules/clean_images.js delete mode 100644 skins/faces/.gitkeep delete mode 100644 skins/helms/.gitkeep diff --git a/clean_images.sh b/clean_images.sh deleted file mode 100755 index 723d795..0000000 --- a/clean_images.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# deletes old images on heroku -# heroku provides only 300 MB available disk space - -# number of files to delete (2 files ~ 400B) -amount="50000" # about 20MB - -# max free MB (on /) to trigger deletion -trigger="50" - -available=`df -m / | awk "NR==2 { print $4 }"` # MB available on / -if [ "$available" -le "$trigger" ]; then - echo "Deleting old images" - for file in `ls -1tr "/app/skins/faces" | head -n $amount`; do - rm -rf "/app/skins/faces/$file" - done - for file in `ls -1tr "/app/skins/helms" | head -n $amount`; do - rm -rf "/app/skins/helms/$file" - done - echo "done." -else - echo "More than $trigger MB are available ($available MB), not deleting!" -fi \ No newline at end of file diff --git a/modules/clean_images.js b/modules/clean_images.js new file mode 100644 index 0000000..bff7bda --- /dev/null +++ b/modules/clean_images.js @@ -0,0 +1,44 @@ +var logging = require("./logging"); +var config = require("./config"); +var df = require("node-df"); +var fs = require("fs"); + +var exp = {}; + +// uses `df` to get the available fisk space +// callback contains error, true|false +function should_clean(callback) { + df({ + file: __dirname + "/../" + config.faces_dir, + prefixMultiplier: 'KiB', + isDisplayPrefixMultiplier: false, + precision: 2 + }, function (err, response) { + var available = response[0].available; + console.log("ImageCleaner: " + available + "KB available"); + callback(err, available < config.cleaning_limit); + }); +} + +// check if disk limit reached +// then delete images +exp.run = function() { + should_clean(function(err, clean) { + if (err) { + logging.error(err); + } else if (clean) { + logging.warn("ImageCleaner: Disk limit reached! Cleaning images now"); + var skindir = __dirname + "/../" + config.faces_dir; + var helmdir = __dirname + "/../" + config.helms_dir; + var files = fs.readdirSync(skindir); + for (var i = 0; i < Math.min(files.length, config.cleaning_amount); i++) { + fs.unlink(skindir + files[i], function(){}); + fs.unlink(helmdir + files[i], function(){}); + } + } else { + logging.log("ImageCleaner: Nothing to clean"); + } + }); +}; + +module.exports = exp; \ No newline at end of file diff --git a/modules/config.example.js b/modules/config.example.js index 3085b47..e29028f 100644 --- a/modules/config.example.js +++ b/modules/config.example.js @@ -4,6 +4,9 @@ var config = { default_size: 160, // size to be used when no size given local_cache_time: 3600, // seconds until we will check if the image changed. should be > 60 to prevent mojang 429 response browser_cache_time: 3600, // seconds until browser will request image again + cleaning_interval: 1800, // seconds interval: deleting images if disk size at limit + cleaning_limit: 10240, // minumum required available KB on disk to trigger cleaning + cleaning_amount: 50000, // amount of avatar (and their helm) files to clean http_timeout: 1000, // ms until connection to mojang is dropped faces_dir: "skins/faces/", // directory where faces are kept. should have trailing "/" helms_dir: "skins/helms/", // directory where helms are kept. should have trailing "/" diff --git a/modules/config.js b/modules/config.js index 9b01641..adcdd4d 100644 --- a/modules/config.js +++ b/modules/config.js @@ -5,6 +5,9 @@ var config = { local_cache_time: 30, // seconds until we will check if the image changed. should be > 60 to prevent mojang 429 response browser_cache_time: 30, // seconds until browser will request image again http_timeout: 3000, // ms until connection to mojang is dropped + cleaning_interval: 3000, // ms interval: deleting images if disk size at limit + cleaning_limit: 10240, // minumum required available KB on disk to trigger cleaning + cleaning_amount: 50000, // amount of avatar (and their helm) files to clean faces_dir: "skins/faces/", // directory where faces are kept. should have trailing "/" helms_dir: "skins/helms/", // directory where helms are kept. should have trailing "/" debug_enabled: true // enables logging.debug diff --git a/package.json b/package.json index 22c555f..f10a536 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "mocha-lcov-reporter": "0.0.1", "morgan": "~1.3.0", "redis": "0.12.1", - "request": "2.45.0" + "request": "2.45.0", + "node-df": "0.1.1" } } diff --git a/server.js b/server.js index 95debcc..df1614e 100644 --- a/server.js +++ b/server.js @@ -1,9 +1,14 @@ #!/usr/bin/env node +var config = require("./modules/config"); var debug = require("debug")("crafatar"); +var clean = require("./modules/clean_images"); var app = require("./app"); app.set("port", process.env.PORT || 3000); var server = app.listen(app.get("port"), function() { debug("Crafatar server listening on port " + server.address().port); -}); \ No newline at end of file +}); + +// cleaning worker +setInterval(clean.run, config.cleaning_interval); \ No newline at end of file diff --git a/skins/faces/.gitkeep b/skins/faces/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/skins/helms/.gitkeep b/skins/helms/.gitkeep deleted file mode 100644 index e69de29..0000000