From 15a4f17560eb22697177343f5112910aece927db Mon Sep 17 00:00:00 2001 From: jomo Date: Sun, 29 Mar 2020 01:46:15 +0100 Subject: [PATCH] add rate limit option for sessionserver any outgoing requests to the sessionserver that would exceed the configured rate limit are skipped to prevent being blocked by CloudFront if a texture hash is cached but outdated, the cache ttl will be bumped as if the request succeeded, in order to lower requests in the near future --- config.js | 3 + lib/helpers.js | 7 ++- lib/networking.js | 150 +++++++++++++++++++++++++++++----------------- lib/response.js | 2 +- test/test.js | 12 ++++ www.js | 3 + 6 files changed, 118 insertions(+), 59 deletions(-) diff --git a/config.js b/config.js index 7c5baec..0c9bd2c 100644 --- a/config.js +++ b/config.js @@ -50,6 +50,9 @@ var config = { debug_enabled: process.env.DEBUG === "true" || false, // set to false if you use an external logger that provides timestamps, log_time: process.env.LOG_TIME === "true" || true, + // rate limit per second for outgoing requests to the Mojang session server + // requests exceeding this limit are skipped and considered failed + sessions_rate_limit: parseInt(process.env.SESSIONS_RATE_LIMIT) || Infinity }, sponsor: { sidebar: process.env.SPONSOR_SIDE, diff --git a/lib/helpers.js b/lib/helpers.js index 5afe2fa..fdbe479 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -176,7 +176,7 @@ function store_images(rid, userId, cache_details, type, callback) { resume(userId, "cape", cache_err, null, false); }); } else { - // an error occured, not caching. we can try in 60 seconds + // an error occured, not caching. we can try again in 60 seconds resume(userId, type, err, null, false); } } else { @@ -242,7 +242,10 @@ exp.get_image_hash = function(rid, userId, type, callback) { if (store_err) { // we might have a cached hash although an error occured // (e.g. Mojang servers not reachable, using outdated hash) - cache.update_timestamp(rid, userId, true, function(err2) { + + // when hitting the rate limit, let's pretend the request succeeded and bump the TTL + var ratelimited = store_err.code === "RATELIMIT"; + cache.update_timestamp(rid, userId, !ratelimited, function(err2) { callback(err2 || store_err, -1, cache_details && cached_hash, slim); }); } else { diff --git a/lib/networking.js b/lib/networking.js index d4b2626..47ae317 100644 --- a/lib/networking.js +++ b/lib/networking.js @@ -1,81 +1,119 @@ -var http_code = require("http").STATUS_CODES; var logging = require("./logging"); var request = require("request"); var config = require("../config"); var skins = require("./skins"); +var http = require("http"); require("./object-patch"); var session_url = "https://sessionserver.mojang.com/session/minecraft/profile/"; var textures_url = "https://textures.minecraft.net/texture/"; +// count requests made to session_url in the last 1000ms +var session_requests = []; + var exp = {}; +function req_count() { + var index = session_requests.findIndex((i) => i >= Date.now() - 1000); + if (index >= 0) { + return session_requests.length - index; + } else { + return 0; + } +} + +exp.resetCounter = function() { + var count = req_count(); + if (count) { + var logfunc = count >= config.server.sessions_rate_limit ? logging.warn : logging.debug; + logfunc('Clearing old session requests (count was ' + count + ')'); + session_requests.splice(0, session_requests.length - count); + } else { + session_requests = [] + } +} + // performs a GET request to the +url+ // +options+ object includes these options: // encoding (string), default is to return a buffer // callback: the body, response, // and error buffer. get_from helper method is available exp.get_from_options = function(rid, url, options, callback) { - request.get({ - url: url, - headers: { - "User-Agent": "Crafatar (+https://crafatar.com)" - }, - timeout: config.server.http_timeout, - followRedirect: false, - encoding: options.encoding || null, - }, function(error, response, body) { - // log url + code + description - var code = response && response.statusCode; + var session_req = url.startsWith(session_url); - var logfunc = code && (code < 400 || code === 404) ? logging.debug : logging.warn; - logfunc(rid, url, code || error && error.code, http_code[code]); - - // not necessarily used - var e = new Error(code); + // This is to prevent being blocked by CloudFront for exceeding the rate limit + if (session_req && req_count() >= config.server.sessions_rate_limit) { + var e = new Error("Skipped, rate limit exceeded"); e.name = "HTTP"; - e.code = "HTTPERROR"; + e.code = "RATELIMIT"; - switch (code) { - case 200: - case 301: - case 302: // never seen, but mojang might use it in future - case 307: // never seen, but mojang might use it in future - case 308: // never seen, but mojang might use it in future - // these are okay - break; - case 204: // no content, used like 404 by mojang. making sure it really has no content - case 404: - // can be cached as null - body = null; - break; - case 403: // Blocked by CloudFront :( - case 429: // this shouldn't usually happen, but occasionally does - case 500: - case 502: // CloudFront can't reach mojang origin - case 503: - case 504: - // we don't want to cache this - error = error || e; - body = null; - break; - default: - if (!error) { - // Probably 500 or the likes - logging.error(rid, "Unexpected response:", code, body); - } - error = error || e; - body = null; - break; - } + var response = new http.IncomingMessage(); + response.statusCode = 403; - if (body && !body.length) { - // empty response - body = null; - } + callback(null, response, e); + } else { + session_req && session_requests.push(Date.now()); + request.get({ + url: url, + headers: { + "User-Agent": "Crafatar (+https://crafatar.com)" + }, + timeout: config.server.http_timeout, + followRedirect: false, + encoding: options.encoding || null, + }, function(error, response, body) { + // log url + code + description + var code = response && response.statusCode; - callback(body, response, error); - }); + var logfunc = code && (code < 400 || code === 404) ? logging.debug : logging.warn; + logfunc(rid, url, code || error && error.code, http.STATUS_CODES[code]); + + // not necessarily used + var e = new Error(code); + e.name = "HTTP"; + e.code = "HTTPERROR"; + + switch (code) { + case 200: + case 301: + case 302: // never seen, but mojang might use it in future + case 307: // never seen, but mojang might use it in future + case 308: // never seen, but mojang might use it in future + // these are okay + break; + case 204: // no content, used like 404 by mojang. making sure it really has no content + case 404: + // can be cached as null + body = null; + break; + case 403: // Blocked by CloudFront :( + case 429: // this shouldn't usually happen, but occasionally does + case 500: + case 502: // CloudFront can't reach mojang origin + case 503: + case 504: + // we don't want to cache this + error = error || e; + body = null; + break; + default: + if (!error) { + // Probably 500 or the likes + logging.error(rid, "Unexpected response:", code, body); + } + error = error || e; + body = null; + break; + } + + if (body && !body.length) { + // empty response + body = null; + } + + callback(body, response, error); + }); + } }; // helper method for get_from_options, no options required diff --git a/lib/response.js b/lib/response.js index 93de301..daf08cb 100644 --- a/lib/response.js +++ b/lib/response.js @@ -13,7 +13,7 @@ var human_status = { // print these, but without stacktrace -var silent_errors = ["ETIMEDOUT", "ESOCKETTIMEDOUT", "ECONNRESET", "EHOSTUNREACH", "ECONNREFUSED", "HTTPERROR"]; +var silent_errors = ["ETIMEDOUT", "ESOCKETTIMEDOUT", "ECONNRESET", "EHOSTUNREACH", "ECONNREFUSED", "HTTPERROR", "RATELIMIT"]; // handles HTTP responses // +request+ a http.IncomingMessage diff --git a/test/test.js b/test/test.js index 3afcf24..11c60f3 100644 --- a/test/test.js +++ b/test/test.js @@ -695,6 +695,18 @@ describe("Crafatar", function() { }); }); }); + + it("CloudFront rate limit is handled", function(done) { + var original_rate_limit = config.server.sessions_rate_limit; + config.server.sessions_rate_limit = 1; + networking.get_profile(rid(), uuid, function() { + networking.get_profile(rid(), uuid, function(err, profile) { + assert.strictEqual(err.code, "RATELIMIT"); + config.server.sessions_rate_limit = original_rate_limit; + done(); + }); + }); + }); }); after(function(done) { diff --git a/www.js b/www.js index 328099c..891145e 100644 --- a/www.js +++ b/www.js @@ -1,3 +1,4 @@ +var networking = require("./lib/networking"); var logging = require("./lib/logging"); var config = require("./config"); @@ -6,4 +7,6 @@ process.on("uncaughtException", function(err) { process.exit(1); }); +setInterval(networking.resetCounter, 1000); + require("./lib/server.js").boot(); \ No newline at end of file