From 04b5b7f578a1cac6a31cefa0a567b276a0463d3a Mon Sep 17 00:00:00 2001 From: zefie Date: Sun, 8 Aug 2021 18:47:50 -0400 Subject: [PATCH 01/10] fix possible loss of trust chain --- .../ServiceVault/wtv-head-waiter/login-stage-two.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login-stage-two.js b/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login-stage-two.js index 531fceeb..521011cb 100644 --- a/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login-stage-two.js +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login-stage-two.js @@ -31,11 +31,10 @@ if (socket.ssid !== null) { } } -if (!ssid_sessions[socket.ssid].getSessionData("registered") && (!request_headers.query.guest_login || !minisrv_config.config.allow_guests)) gourl = "wtv-register:/splash"; +if (!ssid_sessions[socket.ssid].getSessionData("registered") && (!request_headers.query.guest_login || !minisrv_config.config.allow_guests)) gourl = "wtv-register:/splash?"; if (gourl) { headers = `200 OK -Connection: Close wtv-open-isp-disabled: false `; if (!ssid_sessions[socket.ssid].getSessionData("registered") && (!request_headers.query.guest_login || !minisrv_config.config.allow_guests)) { @@ -44,7 +43,7 @@ wtv-ticket: ${wtvsec_login.ticket_b64} ${getServiceString('wtv-register')} ${getServiceString('wtv-head-waiter')} ${getServiceString('wtv-star')} -wtv-boot-url: wtv-register:/splash +wtv-boot-url: wtv-register:/splash? ` } headers += `wtv-visit: ${gourl} From f0861e2f21581933c3033c1aa0136425e46a979f Mon Sep 17 00:00:00 2001 From: zefie Date: Sun, 8 Aug 2021 19:40:52 -0400 Subject: [PATCH 02/10] improve session retention --- zefie_wtvp_minisrv/WTVClientSessionData.js | 49 ++++++++++++++++------ zefie_wtvp_minisrv/app.js | 37 ++++++++++------ 2 files changed, 61 insertions(+), 25 deletions(-) diff --git a/zefie_wtvp_minisrv/WTVClientSessionData.js b/zefie_wtvp_minisrv/WTVClientSessionData.js index bf1099dd..082f6e97 100644 --- a/zefie_wtvp_minisrv/WTVClientSessionData.js +++ b/zefie_wtvp_minisrv/WTVClientSessionData.js @@ -1,3 +1,5 @@ +const { lib } = require('crypto-js'); + class WTVClientSessionData { fs = require('fs'); @@ -38,7 +40,8 @@ class WTVClientSessionData { } } - constructor(hide_ssid_in_logs, session_storage_directory) { + constructor(ssid, hide_ssid_in_logs, session_storage_directory) { + this.ssid = ssid; if (hide_ssid_in_logs) this.hide_ssid_in_logs = hide_ssid_in_logs; if (!session_storage_directory) session_storage_directory = __dirname + "/SessionStore"; this.session_storage = session_storage_directory; @@ -172,11 +175,13 @@ class WTVClientSessionData { return outstring; } - loadSessionData() { + loadSessionData(raw_data = false) { try { if (this.fs.lstatSync(this.session_storage + this.path.sep + this.ssid + ".json")) { - var session_data_file = this.fs.readFileSync(this.session_storage + this.path.sep + this.ssid + ".json", 'Utf8'); - var session_data = JSON.parse(session_data_file); + var json_data = this.fs.readFileSync(this.session_storage + this.path.sep + this.ssid + ".json", 'Utf8') + if (raw_data) return json_data; + + var session_data = JSON.parse(json_data); this.session_store = session_data; return true; } @@ -188,14 +193,17 @@ class WTVClientSessionData { } saveSessionData() { - if (!this.session_store.registered) { - var temp_store = this.session_store; - this.loadSessionData(); - this.session_store = Object.assign(this.session_store, temp_store); - } + // load data from disk and merge new data + var temp_store = this.session_store; + if (this.loadSessionData()) this.session_store = Object.assign(this.session_store, temp_store); + else this.session_store = temp_store; + temp_store = null; + try { - var store_data = JSON.stringify(this.session_store); - this.fs.writeFileSync(this.session_storage + this.path.sep + this.ssid + ".json", store_data, "Utf8"); + // only save if file has changed + var json_save_data = JSON.stringify(this.session_store); + var json_load_data = loadSessionData(true); + if (json_save_data != json_load_data) this.fs.writeFileSync(this.session_storage + this.path.sep + this.ssid + ".json", JSON.stringify(this.session_store), "Utf8"); return true; } catch (e) { console.error(" # Error saving session data for", this.filterSSID(this.ssid), e); @@ -213,16 +221,31 @@ class WTVClientSessionData { return this.saveSessionData(); } + SaveIfRegistered() { + if (this.isRegistered()) this.saveSessionData(); + } + + isRegistered() { + var self = this; + var ssid_match = false; + this.fs.readdirSync(this.session_storage).forEach(file => { + if (!file.match(/.*\.json/ig)) return; + if (ssid_match) return; + if (file.split('.')[0] == self.ssid) ssid_match = true; + }); + return ssid_match; + } unregisterBox() { try { if (this.fs.lstatSync(this.session_storage + this.path.sep + this.ssid + ".json")) { - return this.fs.unlinkSync(this.session_storage + this.path.sep + this.ssid + ".json"); + this.fs.unlinkSync(this.session_storage + this.path.sep + this.ssid + ".json"); this.session_store = {}; + return true; } } catch (e) { // Don't log error 'file not found', it just means the client isn't registered yet - if (e.code != "ENOENT") console.error(" # Error deleting session data for", this.filterSSID(this.ssid), e); + console.error(" # Error deleting session data for", this.filterSSID(this.ssid), e); return false; } } diff --git a/zefie_wtvp_minisrv/app.js b/zefie_wtvp_minisrv/app.js index 95752d97..e88752a1 100644 --- a/zefie_wtvp_minisrv/app.js +++ b/zefie_wtvp_minisrv/app.js @@ -279,8 +279,8 @@ function filterSSID(obj) { return obj.substr(0, 6) + ('*').repeat(9); } } else { - if (obj["wtv-client-serial-number"]) { - var ssid = obj["wtv-client-serial-number"]; + if (makeSafeSSID(obj["wtv-client-serial-number"])) { + var ssid = makeSafeSSID(obj["wtv-client-serial-number"]); if (ssid.substr(0, 8) == "MSTVSIMU") { obj["wtv-client-serial-number"] = ssid.substr(0, 10) + ('*').repeat(10) + ssid.substr(20); } else if (ssid.substr(0, 5) == "1SEGA") { @@ -296,6 +296,12 @@ function filterSSID(obj) { } } +function makeSafeSSID(ssid) { + ssid = ssid.replace(/[^a-zA-Z0-9]/g, ""); + if (ssid.length == 0) ssid = null; + return ssid; +} + function makeSafePath(base, target) { target.replace(/[\|\&\;\$\%\@\"\<\>\+\,\\]/g, ""); if (path.sep != "/") target = target.replace(/\//g, path.sep); @@ -356,8 +362,11 @@ async function processURL(socket, request_headers) { if (shortURL.indexOf(':/') >= 0 && shortURL.indexOf('://') < 0) { var ssid = socket.ssid; if (ssid == null) { - ssid = request_headers["wtv-client-serial-number"]; + // prevent possible injection attacks via SSID and filesystem SessionStore + ssid = makeSafeSSID(request_headers["wtv-client-serial-number"]); + if (ssid == "") ssid = null; } + var reqverb = "Request"; if (request_headers.encrypted || request_headers.secure) { reqverb = "Encrypted " + reqverb; @@ -757,7 +766,7 @@ function isUnencryptedString(string, verbose = false) { } function filterSSID(ssid) { - var WTVCSD = new WTVClientSessionData(minisrv_config.config.hide_ssid_in_logs); + var WTVCSD = new WTVClientSessionData(null,minisrv_config.config.hide_ssid_in_logs); return WTVCSD.filterSSID(ssid); } @@ -827,14 +836,17 @@ async function processRequest(socket, data_hex, skipSecure = false, encryptedReq if (!headers) return; - if (headers["wtv-client-serial-number"] != null) { - socket.ssid = headers["wtv-client-serial-number"]; - if (!ssid_sessions[socket.ssid]) { - ssid_sessions[socket.ssid] = new WTVClientSessionData(minisrv_config.config.hide_ssid_in_logs); + if (headers["wtv-client-serial-number"] != null && socket.ssid == null) { + socket.ssid = makeSafeSSID(headers["wtv-client-serial-number"]); + if (socket.ssid != null) { + if (!ssid_sessions[socket.ssid]) { + ssid_sessions[socket.ssid] = new WTVClientSessionData(socket.ssid,minisrv_config.config.hide_ssid_in_logs); + ssid_sessions[socket.ssid].SaveIfRegistered(); + } + if (!ssid_sessions[socket.ssid].data_store.sockets) ssid_sessions[socket.ssid].data_store.sockets = new Set(); + ssid_sessions[socket.ssid].ssid = socket.ssid; + ssid_sessions[socket.ssid].data_store.sockets.add(socket); } - if (!ssid_sessions[socket.ssid].data_store.sockets) ssid_sessions[socket.ssid].data_store.sockets = new Set(); - ssid_sessions[socket.ssid].ssid = socket.ssid; - ssid_sessions[socket.ssid].data_store.sockets.add(socket); } var ip2long = function (ip) { @@ -920,7 +932,8 @@ async function processRequest(socket, data_hex, skipSecure = false, encryptedReq if (headers["wtv-capability-flags"] != null) { if (!ssid_sessions[socket.ssid]) { - ssid_sessions[socket.ssid] = new WTVClientSessionData(minisrv_config.config.hide_ssid_in_logs); + ssid_sessions[socket.ssid] = new WTVClientSessionData(socket.ssid,minisrv_config.config.hide_ssid_in_logs); + ssid_sessions[socket.ssid].SaveIfRegistered(); } if (!ssid_sessions[socket.ssid].capabilities) ssid_sessions[socket.ssid].capabilities = new WTVClientCapabilities(headers["wtv-capability-flags"]); } From 4befbc08a56ea0a7e18470f67c9dd3bdadec3bb1 Mon Sep 17 00:00:00 2001 From: zefie Date: Sun, 8 Aug 2021 19:41:04 -0400 Subject: [PATCH 03/10] add username to home --- zefie_wtvp_minisrv/ServiceVault/wtv-home/home.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-home/home.js b/zefie_wtvp_minisrv/ServiceVault/wtv-home/home.js index 214d201b..76b32623 100644 --- a/zefie_wtvp_minisrv/ServiceVault/wtv-home/home.js +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-home/home.js @@ -24,9 +24,11 @@ function go() { location.href=document.access.url.value; } -Welcome to `+ z_title + `
-`; -if (minisrv_config.config.git_commit) data += "" + "  ".repeat(32) + "git revision " + minisrv_config.config.git_commit + "
"; +Welcome to ${z_title}`; +if (ssid_sessions[socket.ssid].getSessionData("registered")) data += ", " + ssid_sessions[socket.ssid].getSessionData("subscriber_username") + "!"; +data += "
"; +if (minisrv_config.config.git_commit) data += "" + "  ".repeat(35) + "git revision " + minisrv_config.config.git_commit + "
"; + data += ` Encryption Status: ${cryptstatus}
Connection Speed: &rate; From 304968a3641107cf9afd077771a179ec326ee67a Mon Sep 17 00:00:00 2001 From: zefie Date: Sun, 8 Aug 2021 19:41:35 -0400 Subject: [PATCH 04/10] version increment for dev --- zefie_wtvp_minisrv/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zefie_wtvp_minisrv/package.json b/zefie_wtvp_minisrv/package.json index 6ae035e2..863f93a8 100644 --- a/zefie_wtvp_minisrv/package.json +++ b/zefie_wtvp_minisrv/package.json @@ -1,6 +1,6 @@ { "name": "zefie_wtvp_minisrv", - "version": "0.9.13", + "version": "0.9.14", "description": "WebTV Service (WTVP) Emulation Server", "main": "app.js", "homepage": "https://github.com/zefie/zefie_wtvp_minisrv", From 7a09c5f3b12eb39ba6343e05512a7c12d6543836 Mon Sep 17 00:00:00 2001 From: zefie Date: Sun, 8 Aug 2021 20:06:03 -0400 Subject: [PATCH 05/10] use wtv-head-waiter:/relogin for boot url - viewer seems to retain only wtv-* and wtv-head-waiter, so lets try to be closer to protocol and boot with a wtv-head-waiter address instead of wtv-1800 - we still handle via wtv-1800 but we accept wtv-head-waiter:/relogin and send the client on its way to the relogin path --- .../ServiceVault/wtv-1800/preregister.js | 2 +- .../wtv-head-waiter/login-stage-two.js | 8 ++++---- .../ServiceVault/wtv-head-waiter/login.js | 6 +++--- .../ServiceVault/wtv-head-waiter/relogin.js | 20 +++++++++++++++++++ zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj | 3 +++ 5 files changed, 31 insertions(+), 8 deletions(-) create mode 100644 zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/relogin.js diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-1800/preregister.js b/zefie_wtvp_minisrv/ServiceVault/wtv-1800/preregister.js index 49ae3d46..fd2bfc70 100644 --- a/zefie_wtvp_minisrv/ServiceVault/wtv-1800/preregister.js +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-1800/preregister.js @@ -144,7 +144,7 @@ if (ssid_sessions[socket.ssid].data_store.wtvsec_login) { headers += getServiceString('wtv-flashrom') + "\n"; if (bf0app_update) headers += "wtv-boot-url: " + gourl + "\n"; else { - headers += "wtv-boot-url: wtv-1800:/preregister?relogin=true"; + headers += "wtv-boot-url: wtv-head-waiter:/relogin?relogin=true"; if (request_headers.query.guest_login) headers += "&guest_login=true"; headers += "\n"; } diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login-stage-two.js b/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login-stage-two.js index 521011cb..aaff9e62 100644 --- a/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login-stage-two.js +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login-stage-two.js @@ -43,7 +43,7 @@ wtv-ticket: ${wtvsec_login.ticket_b64} ${getServiceString('wtv-register')} ${getServiceString('wtv-head-waiter')} ${getServiceString('wtv-star')} -wtv-boot-url: wtv-register:/splash? +wtv-boot-url: wtv-head-waiter:/relogin? ` } headers += `wtv-visit: ${gourl} @@ -110,11 +110,11 @@ wtv-connection-timeout: 90 wtv-show-time-enabled: true wtv-fader-timeout: 900 wtv-tourist-enabled: true` - headers += "\nwtv-relogin-url: wtv-1800:/preregister?relogin=true"; + headers += "\nwtv-relogin-url: wtv-head-waiter:/relogin?relogin=true"; if (request_headers.query.guest_login) headers += "&guest_login=true"; - headers += "\nwtv-reconnect-url: wtv-1800:/preregister?reconnect=true"; + headers += "\nwtv-reconnect-url: wtv-head-waiter:/relogin?reconnect=true"; if (request_headers.query.guest_login) headers += "&guest_login=true"; - headers += "\nwtv-boot-url: wtv-1800:/preregister?relogin=true"; + headers += "\nwtv-boot-url: wtv-head-waiter:/relogin?relogin=true"; if (request_headers.query.guest_login) headers += "&guest_login=true"; headers += "\nwtv-allow-dsc: true"; headers += "\nwtv-home-url: wtv-home:/home?"; diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login.js b/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login.js index ae847962..9f44b498 100644 --- a/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login.js +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login.js @@ -53,8 +53,8 @@ wtv-expire-all: wtv-head-waiter: wtv-log-url: wtv-log:/log`; if (challenge_header != "") headers += "\n" + challenge_header; headers += ` -wtv-relogin-url: wtv-1800:/preregister?relogin=true -wtv-reconnect-url: wtv-1800:/preregister?reconnect=true +wtv-relogin-url: wtv-head-waiter:/relogin?relogin=true +wtv-reconnect-url: wwtv-head-waiter:/relogin?reconnect=true wtv-visit: ${gourl} Content-type: text/html`; data = ''; @@ -66,7 +66,7 @@ Connection: Keep-Alive Expires: Wed, 09 Oct 1991 22:00:00 GMT wtv-expire-all: wtv-head-waiter: wtv-expire-all: wtv-1800: -wtv-visit: wtv-1800:/preregister?relogin=true +wtv-visit: wtv-head-waiter:/relogin?relogin=true Content-type: text/html`; data = ''; } \ No newline at end of file diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/relogin.js b/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/relogin.js new file mode 100644 index 00000000..52cb2023 --- /dev/null +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/relogin.js @@ -0,0 +1,20 @@ +var gourl = "wtv-1800:/preregister?"; +if (request_headers.query.relogin) gourl += "relogin=true"; +else if (request_headers.query.reconnect) gourl += "reconnect=true"; + +if (request_headers.query.guest_login) { + if (request_headers.query.relogin || request_headers.query.reconnect) gourl += "&"; + gourl += "guest_login=true"; + if (request_headers.query.skip_splash) gourl += "&skip_splash=true"; +} + +headers = `200 OK +Connection: Keep-Alive +Expires: Wed, 09 Oct 1991 22:00:00 GMT +wtv-expire-all: wtv-head-waiter: +wtv-expire-all: wtv-1800: +wtv-service: reset +${getServiceString('wtv-1800')} +wtv-visit: ${gourl} +Content-type: text/html`; +data = ''; \ No newline at end of file diff --git a/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj b/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj index e5f9c67b..cd03b72f 100644 --- a/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj +++ b/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj @@ -71,6 +71,9 @@ + + Code + From 81d604ea77f39266f4d0883a9bcdbf6aaf5876cd Mon Sep 17 00:00:00 2001 From: zefie Date: Sun, 8 Aug 2021 20:52:50 -0400 Subject: [PATCH 06/10] add missing pesky period --- zefie_wtvp_minisrv/ServiceVault/wtv-register/register.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-register/register.js b/zefie_wtvp_minisrv/ServiceVault/wtv-register/register.js index aecdd722..12092688 100644 --- a/zefie_wtvp_minisrv/ServiceVault/wtv-register/register.js +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-register/register.js @@ -40,7 +40,7 @@ Welcome ENCTYPE="x-www-form-encoded" METHOD="POST"> -Welcome to the ${minisrv_config.config.service_name} Mini Service, operated by ${minisrv_config.config.service_owner} +Welcome to the ${minisrv_config.config.service_name} Mini Service, operated by ${minisrv_config.config.service_owner}. The next screens will lead you through a quick setup process for using this service.

Press the "Continue" button below to begin setup.

From e917132437f4b0180a3ab7dd6e1587ecb5006802 Mon Sep 17 00:00:00 2001 From: zefie Date: Sun, 8 Aug 2021 21:47:42 -0400 Subject: [PATCH 07/10] fix f0861e2f21581933c3033c1aa0136425e46a979f --- zefie_wtvp_minisrv/WTVClientSessionData.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zefie_wtvp_minisrv/WTVClientSessionData.js b/zefie_wtvp_minisrv/WTVClientSessionData.js index 082f6e97..550fa206 100644 --- a/zefie_wtvp_minisrv/WTVClientSessionData.js +++ b/zefie_wtvp_minisrv/WTVClientSessionData.js @@ -202,7 +202,7 @@ class WTVClientSessionData { try { // only save if file has changed var json_save_data = JSON.stringify(this.session_store); - var json_load_data = loadSessionData(true); + var json_load_data = this.loadSessionData(true); if (json_save_data != json_load_data) this.fs.writeFileSync(this.session_storage + this.path.sep + this.ssid + ".json", JSON.stringify(this.session_store), "Utf8"); return true; } catch (e) { From b6be54dc035f3769667b97a557e053c3fddfc74e Mon Sep 17 00:00:00 2001 From: zefie Date: Sun, 8 Aug 2021 22:16:31 -0400 Subject: [PATCH 08/10] correct branding in TOS --- .../ServiceVault/wtv-register/ServeLegal.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-register/ServeLegal.js b/zefie_wtvp_minisrv/ServiceVault/wtv-register/ServeLegal.js index c14144b4..44a7168f 100644 --- a/zefie_wtvp_minisrv/ServiceVault/wtv-register/ServeLegal.js +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-register/ServeLegal.js @@ -249,17 +249,17 @@ WARRANTY OF ANY KIND. THE INFORMATION, SOFTWARE, PRODUCTS, AND SERVICES INCLUDED THROUGH THE ${minisrv_config.config.service_name.toUpperCase()} SERVICE MAY INCLUDE INACCURACIES OR TYPOGRAPHICAL ERRORS. ADVICE RECEIVED VIA THE ${minisrv_config.config.service_name.toUpperCase()} SERVICE SHOULD NOT BE RELIED UPON FOR PERSONAL, MEDICAL, LEGAL OR FINANCIAL DECISIONS AND YOU SHOULD CONSULT AN APPROPRIATE -PROFESSIONAL FOR SPECIFIC ADVICE TAILORED TO YOUR SITUATION. MICROSOFT, +PROFESSIONAL FOR SPECIFIC ADVICE TAILORED TO YOUR SITUATION. ${WTVRegister.getServiceOperator().toUpperCase()}, ITS RESELLERS, DISTRIBUTORS AND/OR SUPPLIERS DO NOT WARRANT THAT ACCESS TO OR USE OF THE ${minisrv_config.config.service_name.toUpperCase()} SERVICE WILL BE UNINTERRUPTED OR ERROR-FREE, THAT MEMBERS WILL BE ABLE TO ACCESS THE ${minisrv_config.config.service_name.toUpperCase()} SERVICE AT ANY TIME OR IN ANY GEOGRAPHIC AREA, OR THAT THE ${minisrv_config.config.service_name.toUpperCase()} SERVICE OR ${WTVRegister.getServiceOperator().toUpperCase()} SOFTWARE OR SERVICES WILL MEET ANY -PARTICULAR CRITERIA OF PERFORMANCE OR QUALITY. MICROSOFT, ITS RESELLERS, +PARTICULAR CRITERIA OF PERFORMANCE OR QUALITY. ${WTVRegister.getServiceOperator().toUpperCase()}, ITS RESELLERS, DISTRIBUTORS AND/OR SUPPLIERS HEREBY DISCLAIM ALL WARRANTIES AND CONDITIONS WITH REGARD TO THE ${minisrv_config.config.service_name.toUpperCase()} SERVICE AND ALL RELATED SOFTWARE, INFORMATION, PRODUCTS, SERVICES AND GRAPHICS, INCLUDING ALL IMPLIED WARRANTIES AND CONDITIONS OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, WORKMANLIKE -EFFORT, TITLE, AND NON-INFRINGEMENT. IN NO EVENT SHALL MICROSOFT, ITS +EFFORT, TITLE, AND NON-INFRINGEMENT. IN NO EVENT SHALL ${WTVRegister.getServiceOperator().toUpperCase()}, ITS RESELLERS, DISTRIBUTORS AND/OR SUPPLIERS BE LIABLE FOR ANY DIRECT, INDIRECT, PUNITIVE, INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER, INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF USE, DATA, OR @@ -281,7 +281,7 @@ DO NOT ALLOW THE EXCLUSION OR LIMITATION OF LIABILITY FOR CONSEQUENTIAL OR INCIDENTAL DAMAGES, THE ABOVE LIMITATION MAY NOT APPLY TO YOU. IF YOU ARE DISSATISFIED WITH ANY PORTION OF THE ${minisrv_config.config.service_name.toUpperCase()} SERVICE, OR WITH ANY OF THESE TERMS OF USE, YOUR SOLE AND EXCLUSIVE REMEDY IS TO NOT REGISTER FOR A ${minisrv_config.config.service_name.toUpperCase()} -SERVICE ACCOUNT OR TO TERMINATE YOUR ${minisrv_config.config.service_name.toUpperCase()} SERVICE ACCOUNT. MICROSOFT MAY, +SERVICE ACCOUNT OR TO TERMINATE YOUR ${minisrv_config.config.service_name.toUpperCase()} SERVICE ACCOUNT. ${WTVRegister.getServiceOperator().toUpperCase()} MAY, IN ITS SOLE DISCRETION AND WITHOUT PRIOR NOTICE (I) RESTRICT OR LIMIT ACCESS TO THE ${minisrv_config.config.service_name.toUpperCase()} SERVICE; (II) TERMINATE A USER ACCOUNT OR USER SESSIONS AT ANY TIME; OR (III) DISCONTINUE OR MODIFY ANY OR ALL ASPECTS OF THE ${minisrv_config.config.service_name.toUpperCase()} SERVICE OR ITS SERVICES. From 337eecc9ebbbcd511d93b77fb59c2b0ad6298a88 Mon Sep 17 00:00:00 2001 From: zefie Date: Sun, 8 Aug 2021 22:22:14 -0400 Subject: [PATCH 09/10] this might be crazy but it appears to help a bit (at least on the ServeLegal page) --- zefie_wtvp_minisrv/WTVLzpf.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/zefie_wtvp_minisrv/WTVLzpf.js b/zefie_wtvp_minisrv/WTVLzpf.js index 27b09cf8..f2847708 100644 --- a/zefie_wtvp_minisrv/WTVLzpf.js +++ b/zefie_wtvp_minisrv/WTVLzpf.js @@ -15,12 +15,12 @@ class WTVLzpf { current_length = 0; current_literal = 0; - flag = 0xFFFF; + flag = 0xFFFFFFFF; working_data = 0; match_index = 0; type_index = 0; checksum = 0; - flag_table = new Uint16Array(0x1000) + flag_table = new Uint32Array(0x1000) ring_buffer = new Uint8Array(0x2000) encoded_data = []; @@ -283,13 +283,13 @@ class WTVLzpf { clear() { this.current_length = 0; this.current_literal = 0; - this.flag = 0xFFFF; + this.flag = 0xFFFFFFFF; this.working_data = 0; this.match_index = 0; this.type_index = 0; this.checksum = 0; this.ring_buffer.fill(0x00, 0, 0x2000) - this.flag_table.fill(0xFFFF, 0, 0x1000); + this.flag_table.fill(0xFFFFFFFF, 0, 0x1000); this.encoded_data = []; } @@ -364,28 +364,28 @@ class WTVLzpf { this.type_index = 3; } else { this.match_index = (this.match_index + 1) & 0x1FFF; - this.flag = (this.flag + 1) & 0x1FFF; + this.flag = (this.flag + 1) & 0x1FFFFFFF; this.checksum = (this.checksum + byte) & 0xFFFF; this.working_data = ((this.working_data * 0x0100) + byte) & 0xFFFFFFFF; i++; } } else { - this.flag = 0xFFFF; + this.flag = 0xFFFFFFFF; if (i >= 3) { flags_index = (this.working_data >>> 0x0B ^ this.working_data) & 0x0FFF; this.flag = this.flag_table[flags_index]; - this.flag_table[flags_index] = i & 0x1FFF; + this.flag_table[flags_index] = i & 0x1FFFFFFF; } else { this.type_index++; } - if (this.flag == 0xFFFF) { + if (this.flag == 0xFFFFFFFF) { code_length = this.nomatchEncode[byte][1]; code = this.nomatchEncode[byte][0] << 0x10; } else if (byte == this.ring_buffer[this.flag] && compress_data) { this.match_index = 1; - this.flag = (this.flag + 1) & 0x1FFF; + this.flag = (this.flag + 1) & 0x1FFFFFFF; this.type_index = 4; } else { code_length = this.nomatchEncode[byte][1] + 1; @@ -427,7 +427,7 @@ class WTVLzpf { var flags_index = (this.working_data >>> 0x0B ^ this.working_data) & 0x0FFF; var flag = this.flag_table[flags_index]; - if (flag == 0xFFFF) { + if (flag == 0xFFFFFFFF) { this.EncodeLiteral(0x10, 0x00990000); } else { this.EncodeLiteral(0x11, 0x004c8000); From 72f1c989c6a388e4e1e722e30dc9a8674d6dead2 Mon Sep 17 00:00:00 2001 From: zefie Date: Sun, 8 Aug 2021 22:25:42 -0400 Subject: [PATCH 10/10] add gzip compression for LC2 and newer --- zefie_wtvp_minisrv/app.js | 161 ++++++++++++++++++++------------- zefie_wtvp_minisrv/config.json | 1 + 2 files changed, 99 insertions(+), 63 deletions(-) diff --git a/zefie_wtvp_minisrv/app.js b/zefie_wtvp_minisrv/app.js index e88752a1..c0b50f1b 100644 --- a/zefie_wtvp_minisrv/app.js +++ b/zefie_wtvp_minisrv/app.js @@ -2,6 +2,7 @@ const fs = require('fs'); const path = require('path'); +const zlib = require('zlib'); const http = require('http'); const https = require('https'); const strftime = require('strftime'); // used externally by service scripts @@ -100,68 +101,70 @@ function doErrorPage(code, data = null) { function getConType(path) { var file_ext = getFileExt(path).toLowerCase(); + var wtv_mime_type = ""; + var modern_mime_type = ""; // process WebTV overrides, fall back to generic mime lookup switch (file_ext) { case "aif": - return "audio/x-aif"; + wtv_mime_type = "audio/x-aif"; case "aifc": - return "audio/x-aifc"; + wtv_mime_type = "audio/x-aifc"; case "aiff": - return "audio/x-aiff"; + wtv_mime_type = "audio/x-aiff"; case "ani": - return "x-wtv-animation"; + wtv_mime_type = "x-wtv-animation"; case "brom": - return "binary/x-wtv-bootrom"; + wtv_mime_type = "binary/x-wtv-bootrom"; case "cdf": - return "application/netcdf"; + wtv_mime_type = "application/netcdf"; case "dat": - return "binary/cache-data"; + wtv_mime_type = "binary/cache-data"; case "dl": - return "wtv/download-list"; + wtv_mime_type = "wtv/download-list"; case "gsm": - return "audio/x-gsm"; + wtv_mime_type = "audio/x-gsm"; case "gz": - return "application/gzip"; + wtv_mime_type = "application/gzip"; case "ini": - return "wtv/jack-configuration"; + wtv_mime_type = "wtv/jack-configuration"; case "mips-code": - return "code/x-wtv-code-mips"; + wtv_mime_type = "code/x-wtv-code-mips"; case "o": - return "binary/x-wtv-approm"; + wtv_mime_type = "binary/x-wtv-approm"; case "ram": - return "audio/x-pn-realaudio"; + wtv_mime_type = "audio/x-pn-realaudio"; case "rom": - return "binary/x-wtv-flashblock"; + wtv_mime_type = "binary/x-wtv-flashblock"; case "rsp": - return "wtv/jack-response"; + wtv_mime_type = "wtv/jack-response"; case "swa": case "swf": - return "application/x-shockwave-flash"; + wtv_mime_type = "application/x-shockwave-flash"; case "srf": case "spl": - return "wtv/jack-data"; + wtv_mime_type = "wtv/jack-data"; case "ttf": - return "wtv/jack-fonts"; + wtv_mime_type = "wtv/jack-fonts"; case "tvch": - return "wtv/tv-channels"; + wtv_mime_type = "wtv/tv-channels"; case "tvl": - return "wtv/tv-listings"; + wtv_mime_type = "wtv/tv-listings"; case "tvsl": - return "wtv/tv-smartlinks"; + wtv_mime_type = "wtv/tv-smartlinks"; case "wad": - return "binary/doom-data"; + wtv_mime_type = "binary/doom-data"; case "mp2": case "hsb": case "rmf": case "s3m": case "mod": case "xm": - return "application/Music"; + wtv_mime_type = "application/Music"; } - // if we reach here, its not a WebTV specific override - // or we are not yet aware of said override - return mime.lookup(path); + modern_mime_type = mime.lookup(path); + if (wtv_mime_type == "") wtv_mime_type = modern_mime_type; + return new Array(wtv_mime_type, modern_mime_type); } async function processPath(socket, service_vault_file_path, request_headers = new Array(), service_name) { @@ -180,9 +183,10 @@ async function processPath(socket, service_vault_file_path, request_headers = ne service_vault_found = true; request_is_async = true; if (!zquiet) console.log(" * Found " + service_vault_file_path + " to handle request (Direct File Mode) [Socket " + socket.id + "]"); - var contype = getConType(service_vault_file_path); + var contypes = getConType(service_vault_file_path); headers = "200 OK\n" - headers += "Content-Type: " + contype; + headers += "Content-Type: " + contypes[0] + "\n"; + headers += "wtv-modern-content-type" + contypes[1]; fs.readFile(service_vault_file_path, null, function (err, data) { sendToClient(socket, headers, data); }); @@ -585,11 +589,11 @@ async function sendToClient(socket, headers_obj, data) { headers_obj = moveObjectElement('Connection', 'http_response', headers_obj); } - var clen = 0; + var content_length = 0; if (typeof data.length !== 'undefined') { - clen = data.length; + content_length = data.length; } else if (typeof data.byteLength !== 'undefined') { - clen = data.byteLength; + content_length = data.byteLength; } // fix captialization @@ -601,28 +605,80 @@ async function sendToClient(socket, headers_obj, data) { // if box can do compression, see if its worth enabling if (ssid_sessions[socket.ssid].capabilities) { - if (ssid_sessions[socket.ssid].capabilities['client-can-receive-compressed-data'] && minisrv_config.config.enable_lzpf_compression) { - compress_data = shouldWeCompress(headers_obj["Content-Type"]); + if (ssid_sessions[socket.ssid].capabilities['client-can-receive-compressed-data']) { + var compression_type = 1; // lzpf + if (ssid_sessions[socket.ssid].get("wtv-client-rom-type") != "bf0app") { + var compression_type = 2; // gzip + } + + // mostly for debugging + if (minisrv_config.config.force_compression_type == "lzpf") compression_type = 1; + if (minisrv_config.config.force_compression_type == "gzip") compression_type = 2; + + // should we bother to compress? + if (!headers_obj["Content-Encoding"]) { + if (typeof (headers_obj["Content-Type"]) != 'undefined') { + var content_type = (typeof (headers_obj["wtv-modern-content-type"]) != 'undefined') ? headers_obj["wtv-modern-content-type"] : headers_obj["Content-Type"]; + // both lzpf and gzip + if (content_type.match(/^text\//) && headers_obj["Content-Type"] != "text/tellyscript") compress_data = true; + else if (content_type.match(/^application\/(x-?)javascript$/)) compress_data = true; + else if (content_type == "application/json") compress_data = true; + if (compression_type == 2) { + // gzip only + if (content_type.match(/^audio\/(x-)?[midi|wav]/)) compress_data = true; // midi & wav + if (content_type.match(/^audio\/(x-)?[s3m|mod|xm]/)) compress_data = true; // s3m, mod, xm + if (content_type.match(/^audio\/(x-)?[midi|wav]/)) compress_data = true; // midi & wav + } + } + } + if (headers_obj["wtv-modern-content-type"]) delete headers_obj["wtv-modern-content-type"]; } } // compress if needed - if (compress_data && clen > 0) { - content_length = clen; + if (compress_data && compression_type && content_length > 0) { + var uncompressed_content_length = content_length; + switch (compression_type) { + case 1: + // wtv-lzpf implementation + if (minisrv_config.config.enable_lzpf_compression || minisrv_config.config.force_compression_type) { + headers_obj["wtv-lzpf"] = 0; + var wtvcomp = new WTVLzpf(); + data = wtvcomp.Compress(data); + wtvcomp = null; // Makes the garbage gods happy so it cleans up our mess + } + break; - headers_obj["wtv-lzpf"] = 0; + case 2: + // zlib gzip implementation + if (minisrv_config.config.enable_gzip_compression || minisrv_config.config.force_compression_type) { + headers_obj['Content-Encoding'] = 'gzip'; + data = zlib.gzipSync(data, { + 'level': 9 + }); + } + break; + } - var wtvcomp = new WTVLzpf(); - data = wtvcomp.Compress(data); - - wtvcomp = null; // Makes the garbage gods happy so it cleans up our mess + var compressed_content_length = 0; + if (content_length == 0 || compression_type != 1) { + // ultimately send compressed content length + compressed_content_length = data.byteLength; + content_length = compressed_content_length; + } else { + // ultimately send original content length if lzpf + compressed_content_length = data.byteLength; + } + var compression_percentage = ((compressed_content_length / uncompressed_content_length) * 100).toFixed(1).toString() + "%"; + if (uncompressed_content_length != compressed_content_length) if (zdebug) console.log(" # Compression stats: Orig Size:", uncompressed_content_length, "~ Comp Size:", compressed_content_length, "~ Ratio:", compression_percentage); } + // encrypt if needed if (socket_sessions[socket.id].secure == true) { headers_obj["wtv-encrypted"] = 'true'; headers_obj = moveObjectElement('wtv-encrypted', 'Connection', headers_obj); - if (clen > 0 && socket_sessions[socket.id].wtvsec) { + if (content_length > 0 && socket_sessions[socket.id].wtvsec) { if (!zquiet) console.log(" * Encrypting response to client ...") var enc_data = socket_sessions[socket.id].wtvsec.Encrypt(1, data); data = enc_data; @@ -634,14 +690,6 @@ async function sendToClient(socket, headers_obj, data) { if (headers_obj["Content-Length"]) delete headers_obj["Content-Length"]; if (headers_obj["Content-length"]) delete headers_obj["Content-length"]; - if (content_length == 0) { - if (typeof data.length !== 'undefined') { - content_length = data.length; - } else if (typeof data.byteLength !== 'undefined') { - content_length = data.byteLength; - } - } - headers_obj["Content-length"] = content_length; if (ssid_sessions[socket.ssid]) { @@ -713,19 +761,6 @@ async function sendToClient(socket, headers_obj, data) { } } -function shouldWeCompress(content_type) { - if (typeof (content_type) != 'undefined') { - if ((content_type.match(/^text\//) && content_type != "text/tellyscript") || - content_type.match(/^application\/(x-?)javascript$/) || - content_type.match(/^audio\/(x-)?midi/) || - content_type.match(/^audio\/(x-)?wav/) || - content_type == "application/json") { - return true; - } - } - return false; -} - function concatArrayBuffer(buffer1, buffer2) { var tmp = new Uint8Array(buffer1.byteLength + buffer2.byteLength); tmp.set(new Uint8Array(buffer1), 0); diff --git a/zefie_wtvp_minisrv/config.json b/zefie_wtvp_minisrv/config.json index 474b2f44..1ce17dd8 100644 --- a/zefie_wtvp_minisrv/config.json +++ b/zefie_wtvp_minisrv/config.json @@ -15,6 +15,7 @@ "verbosity": 2, "error_log_file": "errors.log", "enable_lzpf_compression": false, + "enable_gzip_compression": true, "allow_guests": true }, "services": {