From da8ba52717912c45f96cb6a731eb4f097cc2bcc2 Mon Sep 17 00:00:00 2001 From: jomo Date: Sun, 2 Nov 2014 04:12:00 +0100 Subject: [PATCH] add redis caching, closes #3. More logs, closes #9 --- .gitignore | 1 + README.md | 7 ++- modules/cache.js | 30 +++++++++ modules/config.js | 1 + modules/helpers.js | 141 +++++++++++++++++++++++++++++------------- modules/networking.js | 5 ++ package.json | 3 +- routes/avatars.js | 6 ++ test/bulk.sh | 6 +- 9 files changed, 151 insertions(+), 49 deletions(-) create mode 100644 modules/cache.js diff --git a/.gitignore b/.gitignore index 0a8ce98..93f3af4 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ skins/ *.log node_modules/ .DS_Store +*.rdb diff --git a/README.md b/README.md index d4908d9..128a443 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,12 @@ Inspired by [Gravatar](https://gravatar.com) (hence the name) and [Minotar](http ## Usage -See the [API Usage](http://crafatar.com) +See the [API Usage](https://crafatar.com) ## Install * Clone the repository -* npm install -* npm start +* `npm install` +* `redis-server` +* `npm start` * Access [http://localhost:3000](http://localhost:3000) \ No newline at end of file diff --git a/modules/cache.js b/modules/cache.js new file mode 100644 index 0000000..a5912a5 --- /dev/null +++ b/modules/cache.js @@ -0,0 +1,30 @@ +var config = require("./config"); +var redis = require("redis").createClient(); +var fs = require("fs"); + +var exp = {}; + +// sets the timestamp for +uuid+ to now +exp.update_timestamp = function(uuid) { + console.log("cache: updating timestamp for " + uuid); + var time = new Date().getTime(); + redis.hmset(uuid, "t", time); +}; + +// create the key +uuid+, store +hash+ and time +exp.save_hash = function(uuid, hash) { + console.log("cache: saving hash for " + uuid); + var time = new Date().getTime(); + redis.hmset(uuid, "h", hash, "t", time); +}; + +// get a details object for +uuid+ +// {hash: "0123456789abcdef", time: 1414881524512} +// null when uuid unkown +exp.get_details = function(uuid, callback) { + redis.hgetall(uuid, function(err, data) { + callback(err, data); + }); +}; + +module.exports = exp; \ No newline at end of file diff --git a/modules/config.js b/modules/config.js index b1f9caa..c9b6725 100644 --- a/modules/config.js +++ b/modules/config.js @@ -2,6 +2,7 @@ var config = { min_size: 0, // < 0 will (obviously) cause crash max_size: 512, // too big values might lead to slow response time or DoS default_size: 180, // size to be used when no size given + local_cache_time: 3600, // seconds until we will check if the image changed browser_cache_time: 3600, // seconds until browser will request image again http_timeout: 1000, // ms until connection to mojang is dropped faces_dir: 'skins/faces/', // directory where faces are kept. should have trailing '/' diff --git a/modules/helpers.js b/modules/helpers.js index aa3ea90..f63a87a 100644 --- a/modules/helpers.js +++ b/modules/helpers.js @@ -1,15 +1,60 @@ var networking = require('./networking'); var config = require('./config'); +var cache = require('./cache'); var skins = require('./skins'); var fs = require('fs'); var valid_uuid = /^[0-9a-f]{32}$/; +var hash_pattern = /[0-9a-f]+$/; -var exp = {}; +function get_hash(url) { + return hash_pattern.exec(url)[0].toLowerCase(); +} + +// requests skin for +uuid+ and extracts face/helm if image hash in +details+ changed +// callback contains error, image hash +function store_images(uuid, details, callback) { + // get profile for +uuid+ + networking.get_profile(uuid, function(err, profile) { + if (err) { + callback(err, null); + } else { + var skinurl = skin_url(profile); + if (skinurl) { + console.log(skinurl); + // set file paths + var hash = get_hash(skinurl); + if (details && details.h == hash) { + // hash hasn't changed + console.log("hash has not changed"); + cache.update_timestamp(uuid); + callback(null, hash); + } else { + // hash has changed + console.log("new hash: " + hash); + var facepath = config.faces_dir + hash + ".png"; + var helmpath = config.helms_dir + hash + ".png"; + // download skin, extract face/helm + networking.skin_file(skinurl, facepath, helmpath, function(err) { + if (err) { + callback(err, null); + } else { + cache.save_hash(uuid, hash); + callback(null, hash); + } + }); + } + } else { + // profile found, but has no skin + callback(null, null); + } + } + }); +} // exracts the skin url of a +profile+ object // returns null when no url found (user has no skin) -exp.skin_url = function(profile) { +function skin_url(profile) { var url = null; if (profile && profile.properties) { profile.properties.forEach(function(prop) { @@ -21,8 +66,40 @@ exp.skin_url = function(profile) { }); } return url; -}; +} +// decides whether to get an image from disk or to download it +// callback contains error, status, hash +// the status gives information about how the image was received +// -1: error +// 1: found on disk +// 2: profile requested/found, skin downloaded from mojang servers +// 3: profile requested/found, but it has no skin +function get_image_hash(uuid, callback) { + cache.get_details(uuid, function(err, details) { + if (err) { + callback(err, -1, null); + } else { + if (details && details.t + config.local_cache_time >= new Date().getTime()) { + // uuid known + recently updated + console.log("uuid known & recently updated"); + callback(null, 1, details.h); + } else { + console.log("uuid not known or too old"); + store_images(uuid, details, function(err, hash) { + if (err) { + callback(err, -1, null); + } else { + console.log("hash: " + hash); + callback(null, (hash ? 2 : 3), hash); + } + }); + } + } + }); +} + +var exp = {}; // returns true if the +uuid+ is a valid uuid // the uuid may be not exist, however @@ -31,55 +108,31 @@ exp.uuid_valid = function(uuid) { }; // handles requests for +uuid+ images with +size+ -// callback is a function with 3 parameters: -// error, status, image buffer -// image is the user's face+helm when helm is true, or the face otherwise -// -// the status gives information about how the image was received -// -1: error -// 1: found on disk -// 2: profile requested/found, skin downloaded from mojang servers -// 3: profile requested/found, but it has no skin +// callback contains error, status, image buffer +// image is the user's face+helm when helm is true, or the face otherwise +// for status, see get_image_hash exp.get_avatar = function(uuid, helm, size, callback) { - var facepath = config.faces_dir + uuid + ".png"; - var helmpath = config.helms_dir + uuid + ".png"; - var filepath = helm ? helmpath : facepath; - - if (fs.existsSync(filepath)) { - // file found on disk - skins.resize_img(filepath, size, function(err, result) { - callback(err, 1, result); - }); - } else { - // download skin - networking.get_profile(uuid, function(err, profile) { - if (err) { - callback(err, -1, profile); - return; - } - var skinurl = exp.skin_url(profile); - - if (skinurl) { - networking.skin_file(skinurl, facepath, helmpath, function(err) { + console.log("\nrequest: " + uuid); + get_image_hash(uuid, function(err, status, hash) { + if (err) { + callback(err, -1, null); + } else { + if (hash) { + var filepath = (helm ? config.helms_dir : config.faces_dir) + hash + ".png"; + skins.resize_img(filepath, size, function(err, result) { if (err) { callback(err, -1, null); } else { - console.log('got skin'); - skins.resize_img(filepath, size, function(err, result) { - if (err) { - callback(err, -1, null); - } else { - callback(null, 2, result); - } - }); + callback(null, status, result); } }); } else { - // profile found, but has no skin - callback(null, 3, null); + // hash is null when uuid has no skin + callback(null, status, null); } - }); - } + } + }); + }; module.exports = exp; \ No newline at end of file diff --git a/modules/networking.js b/modules/networking.js index 69ba150..9c90a35 100644 --- a/modules/networking.js +++ b/modules/networking.js @@ -15,6 +15,7 @@ exp.get_profile = function(uuid, callback) { }, function (error, response, body) { if (!error && response.statusCode == 200) { // profile downloaded successfully + console.log("profile downloaded for " + uuid); callback(null, JSON.parse(body)); } else { if (error) { @@ -22,6 +23,7 @@ exp.get_profile = function(uuid, callback) { return; } else if (response.statusCode == 204 || response.statusCode == 404) { // we get 204 No Content when UUID doesn't exist (including 404 in case they change that) + console.log("uuid does not exist"); } else if (response.statusCode == 429) { // Too Many Requests console.warn("Too many requests for " + uuid); @@ -48,11 +50,14 @@ exp.skin_file = function(url, facename, helmname, callback) { }, function (error, response, body) { if (!error && response.statusCode == 200) { // skin downloaded successfully + console.log("skin downloaded."); skins.extract_face(body, facename, function(err) { if (err) { callback(err); } else { + console.log("face extracted."); skins.extract_helm(facename, body, helmname, function(err) { + console.log("helm extracted."); callback(err); }); } diff --git a/package.json b/package.json index 5e43530..4b37693 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "debug": "~2.0.0", "jade": "~1.6.0", "lwip": "0.0.5", - "request": "2.45.0" + "request": "2.45.0", + "redis": " 0.12.1" } } \ No newline at end of file diff --git a/routes/avatars.js b/routes/avatars.js index cf57705..d0732e3 100644 --- a/routes/avatars.js +++ b/routes/avatars.js @@ -26,6 +26,7 @@ router.get('/:uuid', function(req, res) { try { helpers.get_avatar(uuid, helm, size, function(err, status, image) { + console.log(uuid + " - " + status); if (err) { console.error(err); handle_404(def); @@ -33,6 +34,11 @@ router.get('/:uuid', function(req, res) { sendimage(200, status == 1, image); } else if (status == 3) { handle_404(def); + } else { + console.error("wat"); + console.error(err); + console.error(status); + handle_404(def); } }); } catch(e) { diff --git a/test/bulk.sh b/test/bulk.sh index 7851213..a69f833 100755 --- a/test/bulk.sh +++ b/test/bulk.sh @@ -4,5 +4,9 @@ rm -f "$dir/../skins/"*.png || exit 1 for uuid in `cat "$dir/uuids.txt"`; do uuid=`echo "$uuid" | tr -d '\r'` size=$(( ((RANDOM<<15)|RANDOM) % 514 - 1 )) # random number from -1 to 513 - curl -sS -o /dev/null -w "%{url_effective} %{http_code} %{time_total}s\\n" "http://127.0.0.1:3000/avatars/$uuid/$size" + helm="" + if [ "$(( ((RANDOM<<15)|RANDOM) % 2 ))" -eq "1" ]; then + helm="&helm" + fi + curl -sS -o /dev/null -w "%{url_effective} %{http_code} %{time_total}s\\n" "http://127.0.0.1:3000/avatars/$uuid?size=$size$helm" || exit 1 done