add redis caching, closes #3. More logs, closes #9

.gitignore

@@ -2,3 +2,4 @@ skins/
 *.log
 node_modules/
 .DS_Store
+*.rdb

README.md

@@ -5,11 +5,12 @@ Inspired by [Gravatar](https://gravatar.com) (hence the name) and [Minotar](http
 
 ## Usage
 
-See the [API Usage](http://crafatar.com)
+See the [API Usage](https://crafatar.com)
 
 ## Install
 
 * Clone the repository
-* npm install
+* `npm install`
-* npm start
+* `redis-server`
+* `npm start`
 * Access [http://localhost:3000](http://localhost:3000)

modules/cache.js

@@ -0,0 +1,30 @@
+var config = require("./config");
+var redis = require("redis").createClient();
+var fs = require("fs");
+
+var exp = {};
+
+// sets the timestamp for +uuid+ to now
+exp.update_timestamp = function(uuid) {
+  console.log("cache: updating timestamp for " + uuid);
+  var time = new Date().getTime();
+  redis.hmset(uuid, "t", time);
+};
+
+// create the key +uuid+, store +hash+ and time
+exp.save_hash = function(uuid, hash) {
+  console.log("cache: saving hash for " + uuid);
+  var time = new Date().getTime();
+  redis.hmset(uuid, "h", hash, "t", time);
+};
+
+// get a details object for +uuid+
+// {hash: "0123456789abcdef", time: 1414881524512}
+// null when uuid unkown
+exp.get_details = function(uuid, callback) {
+  redis.hgetall(uuid, function(err, data) {
+    callback(err, data);
+  });
+};
+
+module.exports = exp;

modules/config.js

@@ -2,6 +2,7 @@ var config = {
   min_size: 0,               // < 0 will (obviously) cause crash
   max_size: 512,             // too big values might lead to slow response time or DoS
   default_size: 180,         // size to be used when no size given
+  local_cache_time: 3600,    // seconds until we will check if the image changed
   browser_cache_time: 3600,  // seconds until browser will request image again
   http_timeout: 1000,        // ms until connection to mojang is dropped
   faces_dir: 'skins/faces/', // directory where faces are kept. should have trailing '/'

modules/helpers.js

@@ -1,15 +1,60 @@
 var networking = require('./networking');
 var config = require('./config');
+var cache = require('./cache');
 var skins = require('./skins');
 var fs = require('fs');
 
 var valid_uuid = /^[0-9a-f]{32}$/;
+var hash_pattern = /[0-9a-f]+$/;
 
-var exp = {};
+function get_hash(url) {
+  return hash_pattern.exec(url)[0].toLowerCase();
+}
+
+// requests skin for +uuid+ and extracts face/helm if image hash in +details+ changed
+// callback contains error, image hash
+function store_images(uuid, details, callback) {
+  // get profile for +uuid+
+  networking.get_profile(uuid, function(err, profile) {
+    if (err) {
+      callback(err, null);
+    } else {
+      var skinurl = skin_url(profile);
+      if (skinurl) {
+        console.log(skinurl);
+        // set file paths
+        var hash = get_hash(skinurl);
+        if (details && details.h == hash) {
+          // hash hasn't changed
+          console.log("hash has not changed");
+          cache.update_timestamp(uuid);
+          callback(null, hash);
+        } else {
+          // hash has changed
+          console.log("new hash: " + hash);
+          var facepath = config.faces_dir + hash + ".png";
+          var helmpath = config.helms_dir + hash + ".png";
+          // download skin, extract face/helm
+          networking.skin_file(skinurl, facepath, helmpath, function(err) {
+            if (err) {
+              callback(err, null);
+            } else {
+              cache.save_hash(uuid, hash);
+              callback(null, hash);
+            }
+          });
+        }
+      } else {
+        // profile found, but has no skin
+        callback(null, null);
+      }
+    }
+  });
+}
 
 // exracts the skin url of a +profile+ object
 // returns null when no url found (user has no skin)
-exp.skin_url = function(profile) {
+function skin_url(profile) {
   var url = null;
   if (profile && profile.properties) {
     profile.properties.forEach(function(prop) {
@@ -21,8 +66,40 @@ exp.skin_url = function(profile) {
     });
   }
   return url;
-};
+}
 
+// decides whether to get an image from disk or to download it
+// callback contains error, status, hash
+// the status gives information about how the image was received
+//  -1: error
+//   1: found on disk
+//   2: profile requested/found, skin downloaded from mojang servers
+//   3: profile requested/found, but it has no skin
+function get_image_hash(uuid, callback) {
+  cache.get_details(uuid, function(err, details) {
+    if (err) {
+      callback(err, -1, null);
+    } else {
+      if (details && details.t + config.local_cache_time >= new Date().getTime()) {
+        // uuid known + recently updated
+        console.log("uuid known & recently updated");
+        callback(null, 1, details.h);
+      } else {
+        console.log("uuid not known or too old");
+        store_images(uuid, details, function(err, hash) {
+          if (err) {
+            callback(err, -1, null);
+          } else {
+            console.log("hash: " + hash);
+            callback(null, (hash ? 2 : 3), hash);
+          }
+        });
+      }
+    }
+  });
+}
+
+var exp = {};
 
 // returns true if the +uuid+ is a valid uuid
 // the uuid may be not exist, however
@@ -31,55 +108,31 @@ exp.uuid_valid = function(uuid) {
 };
 
 // handles requests for +uuid+ images with +size+
-// callback is a function with 3 parameters:
+// callback contains error, status, image buffer
-//   error, status, image buffer
 // image is the user's face+helm when helm is true, or the face otherwise
-//
+// for status, see get_image_hash
-// the status gives information about how the image was received
-//  -1: error
-//   1: found on disk
-//   2: profile requested/found, skin downloaded from mojang servers
-//   3: profile requested/found, but it has no skin
 exp.get_avatar = function(uuid, helm, size, callback) {
-  var facepath = config.faces_dir + uuid + ".png";
+  console.log("\nrequest: " + uuid);
-  var helmpath = config.helms_dir + uuid + ".png";
+  get_image_hash(uuid, function(err, status, hash) {
-  var filepath = helm ? helmpath : facepath;
-
-  if (fs.existsSync(filepath)) {
-    // file found on disk
-    skins.resize_img(filepath, size, function(err, result) {
-      callback(err, 1, result);
-    });
-  } else {
-    // download skin
-    networking.get_profile(uuid, function(err, profile) {
-      if (err) {
-        callback(err, -1, profile);
-        return;
-      }
-      var skinurl = exp.skin_url(profile);
-
-      if (skinurl) {
-        networking.skin_file(skinurl, facepath, helmpath, function(err) {
     if (err) {
       callback(err, -1, null);
     } else {
-            console.log('got skin');
+      if (hash) {
+        var filepath = (helm ? config.helms_dir : config.faces_dir) + hash + ".png";
         skins.resize_img(filepath, size, function(err, result) {
           if (err) {
             callback(err, -1, null);
           } else {
-                callback(null, 2, result);
+            callback(null, status, result);
-              }
-            });
           }
         });
       } else {
-        // profile found, but has no skin
+        // hash is null when uuid has no skin
-        callback(null, 3, null);
+        callback(null, status, null);
+      }
     }
   });
-  }
+
 };
 
 module.exports = exp;

modules/networking.js

@@ -15,6 +15,7 @@ exp.get_profile = function(uuid, callback) {
   }, function (error, response, body) {
     if (!error && response.statusCode == 200) {
       // profile downloaded successfully
+      console.log("profile downloaded for " + uuid);
       callback(null, JSON.parse(body));
     } else {
       if (error) {
@@ -22,6 +23,7 @@ exp.get_profile = function(uuid, callback) {
         return;
       } else if (response.statusCode == 204 || response.statusCode == 404) {
         // we get 204 No Content when UUID doesn't exist (including 404 in case they change that)
+        console.log("uuid does not exist");
       } else if (response.statusCode == 429) {
         // Too Many Requests
         console.warn("Too many requests for " + uuid);
@@ -48,11 +50,14 @@ exp.skin_file = function(url, facename, helmname, callback) {
   }, function (error, response, body) {
     if (!error && response.statusCode == 200) {
       // skin downloaded successfully
+      console.log("skin downloaded.");
       skins.extract_face(body, facename, function(err) {
         if (err) {
           callback(err);
         } else {
+          console.log("face extracted.");
           skins.extract_helm(facename, body, helmname, function(err) {
+            console.log("helm extracted.");
             callback(err);
           });
         }

package.json

@@ -14,6 +14,7 @@
     "debug": "~2.0.0",
     "jade": "~1.6.0",
     "lwip": "0.0.5",
-    "request": "2.45.0"
+    "request": "2.45.0",
+    "redis": "  0.12.1"
   }
 }

routes/avatars.js

@@ -26,6 +26,7 @@ router.get('/:uuid', function(req, res) {
 
   try {
     helpers.get_avatar(uuid, helm, size, function(err, status, image) {
+      console.log(uuid + " - " + status);
       if (err) {
         console.error(err);
         handle_404(def);
@@ -33,6 +34,11 @@ router.get('/:uuid', function(req, res) {
         sendimage(200, status == 1, image);
       } else if (status == 3) {
         handle_404(def);
+      } else {
+        console.error("wat");
+        console.error(err);
+        console.error(status);
+        handle_404(def);
       }
     });
   } catch(e) {

test/bulk.sh

@@ -4,5 +4,9 @@ rm -f "$dir/../skins/"*.png || exit 1
 for uuid in `cat "$dir/uuids.txt"`; do
   uuid=`echo "$uuid" | tr -d '\r'`
   size=$(( ((RANDOM<<15)|RANDOM) % 514 - 1 )) # random number from -1 to 513
-  curl -sS -o /dev/null -w "%{url_effective} %{http_code} %{time_total}s\\n" "http://127.0.0.1:3000/avatars/$uuid/$size"
+  helm=""
+  if [ "$(( ((RANDOM<<15)|RANDOM) % 2 ))" -eq "1" ]; then
+    helm="&helm"
+  fi
+  curl -sS -o /dev/null -w "%{url_effective} %{http_code} %{time_total}s\\n" "http://127.0.0.1:3000/avatars/$uuid?size=$size$helm" || exit 1
 done