add rate limit option for sessionserver

any outgoing requests to the sessionserver
that would exceed the configured rate limit are skipped
to prevent being blocked by CloudFront

if a texture hash is cached but outdated, the cache ttl will be bumped
as if the request succeeded, in order to lower requests in the near future

config.js

@@ -50,6 +50,9 @@ var config = {
     debug_enabled: process.env.DEBUG === "true" || false,
     // set to false if you use an external logger that provides timestamps,
     log_time: process.env.LOG_TIME === "true" || true,
+    // rate limit per second for outgoing requests to the Mojang session server
+    // requests exceeding this limit are skipped and considered failed
+    sessions_rate_limit: parseInt(process.env.SESSIONS_RATE_LIMIT) || Infinity
   },
   sponsor: {
     sidebar: process.env.SPONSOR_SIDE,

lib/helpers.js

@@ -176,7 +176,7 @@ function store_images(rid, userId, cache_details, type, callback) {
             resume(userId, "cape", cache_err, null, false);
           });
         } else {
-          // an error occured, not caching. we can try in 60 seconds
+          // an error occured, not caching. we can try again in 60 seconds
           resume(userId, type, err, null, false);
         }
       } else {
@@ -242,7 +242,10 @@ exp.get_image_hash = function(rid, userId, type, callback) {
           if (store_err) {
             // we might have a cached hash although an error occured
             // (e.g. Mojang servers not reachable, using outdated hash)
-            cache.update_timestamp(rid, userId, true, function(err2) {
+
+            // when hitting the rate limit, let's pretend the request succeeded and bump the TTL
+            var ratelimited = store_err.code === "RATELIMIT";
+            cache.update_timestamp(rid, userId, !ratelimited, function(err2) {
               callback(err2 || store_err, -1, cache_details && cached_hash, slim);
             });
           } else {

lib/networking.js

@@ -1,81 +1,119 @@
-var http_code = require("http").STATUS_CODES;
 var logging = require("./logging");
 var request = require("request");
 var config = require("../config");
 var skins = require("./skins");
+var http = require("http");
 require("./object-patch");
 
 var session_url = "https://sessionserver.mojang.com/session/minecraft/profile/";
 var textures_url = "https://textures.minecraft.net/texture/";
 
+// count requests made to session_url in the last 1000ms
+var session_requests = [];
+
 var exp = {};
 
+function req_count() {
+  var index = session_requests.findIndex((i) => i >= Date.now() - 1000);
+  if (index >= 0) {
+    return session_requests.length - index;
+  } else {
+    return 0;
+  }
+}
+
+exp.resetCounter = function() {
+  var count = req_count();
+  if (count) {
+    var logfunc = count >= config.server.sessions_rate_limit ? logging.warn : logging.debug;
+    logfunc('Clearing old session requests (count was ' + count + ')');
+    session_requests.splice(0, session_requests.length - count);
+  } else {
+    session_requests = []
+  }
+}
+
 // performs a GET request to the +url+
 // +options+ object includes these options:
 //   encoding (string), default is to return a buffer
 // callback: the body, response,
 // and error buffer. get_from helper method is available
 exp.get_from_options = function(rid, url, options, callback) {
-  request.get({
-    url: url,
-    headers: {
-      "User-Agent": "Crafatar (+https://crafatar.com)"
-    },
-    timeout: config.server.http_timeout,
-    followRedirect: false,
-    encoding: options.encoding || null,
-  }, function(error, response, body) {
-    // log url + code + description
-    var code = response && response.statusCode;
+  var session_req = url.startsWith(session_url);
 
-    var logfunc = code && (code < 400 || code === 404) ? logging.debug : logging.warn;
-    logfunc(rid, url, code || error && error.code, http_code[code]);
-
-    // not necessarily used
-    var e = new Error(code);
+  // This is to prevent being blocked by CloudFront for exceeding the rate limit
+  if (session_req && req_count() >= config.server.sessions_rate_limit) {
+    var e = new Error("Skipped, rate limit exceeded");
     e.name = "HTTP";
-    e.code = "HTTPERROR";
+    e.code = "RATELIMIT";
 
-    switch (code) {
-      case 200:
-      case 301:
-      case 302: // never seen, but mojang might use it in future
-      case 307: // never seen, but mojang might use it in future
-      case 308: // never seen, but mojang might use it in future
-        // these are okay
-        break;
-      case 204: // no content, used like 404 by mojang. making sure it really has no content
-      case 404:
-        // can be cached as null
-        body = null;
-        break;
-      case 403: // Blocked by CloudFront :(
-      case 429: // this shouldn't usually happen, but occasionally does
-      case 500:
-      case 502: // CloudFront can't reach mojang origin
-      case 503:
-      case 504:
-        // we don't want to cache this
-        error = error || e;
-        body = null;
-        break;
-      default:
-        if (!error) {
-          // Probably 500 or the likes
-          logging.error(rid, "Unexpected response:", code, body);
-        }
-        error = error || e;
-        body = null;
-        break;
-    }
+    var response = new http.IncomingMessage();
+    response.statusCode = 403;
 
-    if (body && !body.length) {
-      // empty response
-      body = null;
-    }
+    callback(null, response, e);
+  } else {
+    session_req && session_requests.push(Date.now());
+    request.get({
+      url: url,
+      headers: {
+        "User-Agent": "Crafatar (+https://crafatar.com)"
+      },
+      timeout: config.server.http_timeout,
+      followRedirect: false,
+      encoding: options.encoding || null,
+    }, function(error, response, body) {
+      // log url + code + description
+      var code = response && response.statusCode;
 
-    callback(body, response, error);
-  });
+      var logfunc = code && (code < 400 || code === 404) ? logging.debug : logging.warn;
+      logfunc(rid, url, code || error && error.code, http.STATUS_CODES[code]);
+
+      // not necessarily used
+      var e = new Error(code);
+      e.name = "HTTP";
+      e.code = "HTTPERROR";
+
+      switch (code) {
+        case 200:
+        case 301:
+        case 302: // never seen, but mojang might use it in future
+        case 307: // never seen, but mojang might use it in future
+        case 308: // never seen, but mojang might use it in future
+          // these are okay
+          break;
+        case 204: // no content, used like 404 by mojang. making sure it really has no content
+        case 404:
+          // can be cached as null
+          body = null;
+          break;
+        case 403: // Blocked by CloudFront :(
+        case 429: // this shouldn't usually happen, but occasionally does
+        case 500:
+        case 502: // CloudFront can't reach mojang origin
+        case 503:
+        case 504:
+          // we don't want to cache this
+          error = error || e;
+          body = null;
+          break;
+        default:
+          if (!error) {
+            // Probably 500 or the likes
+            logging.error(rid, "Unexpected response:", code, body);
+          }
+          error = error || e;
+          body = null;
+          break;
+      }
+
+      if (body && !body.length) {
+        // empty response
+        body = null;
+      }
+
+      callback(body, response, error);
+    });
+  }
 };
 
 // helper method for get_from_options, no options required

lib/response.js

@@ -13,7 +13,7 @@ var human_status = {
 
 
 // print these, but without stacktrace
-var silent_errors = ["ETIMEDOUT", "ESOCKETTIMEDOUT", "ECONNRESET", "EHOSTUNREACH", "ECONNREFUSED", "HTTPERROR"];
+var silent_errors = ["ETIMEDOUT", "ESOCKETTIMEDOUT", "ECONNRESET", "EHOSTUNREACH", "ECONNREFUSED", "HTTPERROR", "RATELIMIT"];
 
 // handles HTTP responses
 // +request+ a http.IncomingMessage

test/test.js

@@ -695,6 +695,18 @@ describe("Crafatar", function() {
         });
       });
     });
+
+    it("CloudFront rate limit is handled", function(done) {
+      var original_rate_limit = config.server.sessions_rate_limit;
+      config.server.sessions_rate_limit = 1;
+      networking.get_profile(rid(), uuid, function() {
+        networking.get_profile(rid(), uuid, function(err, profile) {
+          assert.strictEqual(err.code, "RATELIMIT");
+          config.server.sessions_rate_limit = original_rate_limit;
+          done();
+        });
+      });
+    });
   });
 
   after(function(done) {

www.js

@@ -1,3 +1,4 @@
+var networking = require("./lib/networking");
 var logging = require("./lib/logging");
 var config = require("./config");
 
@@ -6,4 +7,6 @@ process.on("uncaughtException", function(err) {
   process.exit(1);
 });
 
+setInterval(networking.resetCounter, 1000);
+
 require("./lib/server.js").boot();