From ad16a04c55eaf97e70559d247f1dc6d37ff98c01 Mon Sep 17 00:00:00 2001 From: zefie Date: Fri, 6 Aug 2021 12:18:30 -0400 Subject: [PATCH 1/9] update: do not delete WTVSec on last socket, instead recreate on prereg update: clean up SSID session data only if client is not seen for 3 minutes update: add shouldWeCompress() function update: tweak lzpf (still corrupted) update: rename wtv-setup:/get to wtv-setup:/get-settings update: add additional headers to wtv-setup:/get-settings update: add initial blank wtv-music:/get-playlist --- .../ServiceVault/wtv-1800/preregister.js | 7 +- .../wtv-head-waiter/login-stage-two.js | 13 ++- .../ServiceVault/wtv-head-waiter/login.js | 79 ++++++++----------- .../ServiceVault/wtv-music/get-playlist.js | 4 + .../wtv-setup/{get.js => get-settings.js} | 5 ++ zefie_wtvp_minisrv/WTVClientSessionData.js | 9 +++ zefie_wtvp_minisrv/WTVLzpf.js | 31 +++++--- zefie_wtvp_minisrv/app.js | 70 +++++++++++----- zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj | 5 +- 9 files changed, 137 insertions(+), 86 deletions(-) create mode 100644 zefie_wtvp_minisrv/ServiceVault/wtv-music/get-playlist.js rename zefie_wtvp_minisrv/ServiceVault/wtv-setup/{get.js => get-settings.js} (82%) diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-1800/preregister.js b/zefie_wtvp_minisrv/ServiceVault/wtv-1800/preregister.js index 4a9518e3..9956c53d 100644 --- a/zefie_wtvp_minisrv/ServiceVault/wtv-1800/preregister.js +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-1800/preregister.js @@ -18,16 +18,17 @@ if (socket.ssid) { if (i > 0 && zdebug) console.log(" # Closed", i, "previous sockets for", filterSSID(socket.ssid)); } } - if (ssid_sessions[socket.ssid].data_store.wtvsec_login) { + if (ssid_sessions[socket.ssid].data_store.wtvsec_login) { + if (zdebug) console.log(" # Recreating primary WTVSec login instance for", filterSSID(socket.ssid)); delete ssid_sessions[socket.ssid].data_store.wtvsec_login; } ssid_sessions[socket.ssid].data_store.wtvsec_login = new WTVSec(); ssid_sessions[socket.ssid].data_store.wtvsec_login.IssueChallenge(); - ssid_sessions[socket.ssid].data_store.wtvsec_login.set_incarnation(request_headers["wtv-incarnation"]); + ssid_sessions[socket.ssid].data_store.wtvsec_login.set_incarnation(request_headers["wtv-incarnation"] || 1); } else { console.log(" * Something bad happened (we don't know the client ssid???)"); - var errpage = doErrorCode(400) + var errpage = doErrorPage(400) headers = errpage[0]; data = errpage[1]; } diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login-stage-two.js b/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login-stage-two.js index 6f587ad3..d6d9ca61 100644 --- a/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login-stage-two.js +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login-stage-two.js @@ -79,7 +79,11 @@ wtv-service: reset wtv-boot-url: wtv-1800:/preregister?relogin=true wtv-human-name: ${nickname} ${ssid_sessions[socket.ssid].setIRCNick(nickname)} -wtv-home-url: wtv-home:/home? +wtv-home-url: wtv-home:/home?` + if (ssid_sessions[socket.ssid].get('wtv-need-upgrade') != 'true' && !request_headers.query.reconnect) { + headers += "\nwtv-settings-url: wtv-setup:/get-settings"; + } +headers += ` wtv-domain: wtv.zefie.com wtv-inactive-timeout: 0 wtv-connection-timeout: 90 @@ -94,11 +98,6 @@ wtv-log-url: wtv-log:/log wtv-demo-mode: 0 wtv-wink-deferrer-retries: 3 wtv-offline-mail-enable: false -wtv-name-server: 8.8.8.8 -`; - if (ssid_sessions[socket.ssid].get('wtv-need-upgrade') != 'true' && !request_headers.query.reconnect) { - headers += "wtv-settings-url: wtv-setup:/get\n"; - } - headers += `wtv-visit: wtv-home:/splash? +wtv-visit: wtv-home:/splash? Content-Type: text/html`; } \ No newline at end of file diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login.js b/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login.js index df2b929c..d8e7768e 100644 --- a/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login.js +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-head-waiter/login.js @@ -1,72 +1,59 @@ -var challenge_response, challenge_header = ''; +var challenge_response, challenge_header = ""; var gourl = "wtv-head-waiter:/login-stage-two?"; if (request_headers.query.relogin) gourl += "relogin=true"; if (request_headers.query.reconnect) gourl += "reconnect=true"; +var send_to_relogin = true; -if (socket.ssid !== null) { - var wtvsec_login = ssid_sessions[socket.ssid].get("wtvsec_login"); - if (request_headers["wtv-ticket"]) { - if (wtvsec_login.ticket_b64 == null) { - if (request_headers["wtv-ticket"].length > 8) { - wtvsec_login.DecodeTicket(request_headers["wtv-ticket"]); - wtvsec_login.ticket_b64 = request_headers["wtv-ticket"]; - } - } - } else { - if (wtvsec_login) { - challenge_response = wtvsec_login.challenge_response; - var client_challenge_response = request_headers["wtv-challenge-response"] || null; - if (challenge_response && client_challenge_response) { - if (challenge_response.toString(CryptoJS.enc.Base64).substring(0, 85) == client_challenge_response.substring(0, 85)) { - console.log(" * wtv-challenge-response success for " + socket.ssid); - wtvsec_login.PrepareTicket(); - } else { - challenge_header = "wtv-challenge: " + wtvsec_login.IssueChallenge(); +if (socket.ssid) { + if (ssid_sessions[socket.ssid]) { + if (request_headers["wtv-ticket"]) { + if (ssid_sessions[socket.ssid].data_store.wtvsec_login.ticket_b64 == null) { + if (request_headers["wtv-ticket"].length > 8) { + ssid_sessions[socket.ssid].data_store.wtvsec_login.DecodeTicket(request_headers["wtv-ticket"]); + ssid_sessions[socket.ssid].data_store.wtvsec_login.ticket_b64 = request_headers["wtv-ticket"]; + send_to_relogin = false; } - } else { - challenge_header = "wtv-challenge: " + wtvsec_login.IssueChallenge(); } } else { - wtvsec_login = new WTVSec(); - - } + if (ssid_sessions[socket.ssid].data_store.wtvsec_login) { + var client_challenge_response = request_headers["wtv-challenge-response"] || null; + if (challenge_response && client_challenge_response) { + if (challenge_response.toString(CryptoJS.enc.Base64).substring(0, 85) == client_challenge_response.substring(0, 85)) { + console.log(" * wtv-challenge-response success for " + socket.ssid); + ssid_sessions[socket.ssid].data_store.wtvsec_login.PrepareTicket(); + send_to_relogin = false; + } else { + challenge_header = "wtv-challenge: " + ssid_sessions[socket.ssid].data_store.wtvsec_login.IssueChallenge(); + send_to_relogin = false; + } + } else { + challenge_header = "wtv-challenge: " + ssid_sessions[socket.ssid].data_store.wtvsec_login.IssueChallenge(); + send_to_relogin = false; + } + } + } } } -/* -if (request_headers) { - var cookiedata = {}; - Object.keys(request_headers).forEach(function (k) { - switch (k) { - case "wtv-capability-flags": - case "wtv-system-version": - case "wtv-client-rom-type": - case "wtv-client-bootrom-version": - case "wtv-system-chipversion": - case "wtv-system-sysconfig": - case "wtv-system-cpuspeed": - cookiedata[k] = request_headers[k]; - break; - } - }); -} -*/ +if (!send_to_relogin) { -if (challenge_header != '') { headers = `200 OK Connection: Keep-Alive Expires: Wed, 09 Oct 1991 22:00:00 GMT wtv-expire-all: wtv-head-waiter: `+ getServiceString('wtv-log') + ` -wtv-log-url: wtv-log:/log -${challenge_header} +wtv-log-url: wtv-log:/log`; + if (challenge_header != "") headers += "\n" + challenge_header; + headers += ` wtv-relogin-url: wtv-1800:/preregister?relogin=true wtv-reconnect-url: wtv-1800:/preregister?reconnect=true wtv-visit: ${gourl} Content-type: text/html`; data = ''; + } else { + headers = `200 OK Connection: Keep-Alive Expires: Wed, 09 Oct 1991 22:00:00 GMT diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-music/get-playlist.js b/zefie_wtvp_minisrv/ServiceVault/wtv-music/get-playlist.js new file mode 100644 index 00000000..e42519c9 --- /dev/null +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-music/get-playlist.js @@ -0,0 +1,4 @@ +headers = `200 OK +Content-Type: text/html`; + +data = ''; \ No newline at end of file diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-setup/get.js b/zefie_wtvp_minisrv/ServiceVault/wtv-setup/get-settings.js similarity index 82% rename from zefie_wtvp_minisrv/ServiceVault/wtv-setup/get.js rename to zefie_wtvp_minisrv/ServiceVault/wtv-setup/get-settings.js index 827bd585..87468535 100644 --- a/zefie_wtvp_minisrv/ServiceVault/wtv-setup/get.js +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-setup/get-settings.js @@ -1,4 +1,9 @@ headers = `200 OK +wtv-backgroundmusic-load-playlist: wtv-music:/get-playlist +wtv-printer-model: -1,-1 +wtv-printer-pen: 0,0,1,0 +wtv-printer-setup: 0,0,1,0 +wtv-language-header: en-US,en Content-Type: text/html` var settings_obj = new Array(); diff --git a/zefie_wtvp_minisrv/WTVClientSessionData.js b/zefie_wtvp_minisrv/WTVClientSessionData.js index ba7b5dbf..65d6f94e 100644 --- a/zefie_wtvp_minisrv/WTVClientSessionData.js +++ b/zefie_wtvp_minisrv/WTVClientSessionData.js @@ -51,6 +51,15 @@ class WTVClientSessionData { return false; } + currentConnections() { + if (this.data_store) { + if (this.data_store.sockets) { + return this.data_store.sockets.size; + } + } + return 0; + } + get(key = null) { if (typeof (this.data_store) === 'undefined') return null; else if (key === null) return this.data_store; diff --git a/zefie_wtvp_minisrv/WTVLzpf.js b/zefie_wtvp_minisrv/WTVLzpf.js index 1c95cd51..380354a6 100644 --- a/zefie_wtvp_minisrv/WTVLzpf.js +++ b/zefie_wtvp_minisrv/WTVLzpf.js @@ -12,8 +12,9 @@ class WTVLzpf { current_length = 0 current_literal = 0 + compressed_offset = 0 flag_table = new Uint16Array(0x1000) - compressed_data = [] + compressed_data = null; nomatchEncode = [ [0x0000, 0x10], [0x0001, 0x10], [0x0002, 0x10], @@ -274,8 +275,9 @@ class WTVLzpf { clear() { this.current_length = 0; this.current_literal = 0; + this.compressed_offset = 0; this.flag_table.fill(0xFFFF, 0, 0x1000); - this.compressed_data = []; + this.compressed_data = null; } /** @@ -294,7 +296,7 @@ class WTVLzpf { while (this.current_length > 7) { //console.log("add", this.current_literal >>> 0, code >>> 0, byte, type) - this.compressed_data.push((this.current_literal >>> 0x18) & 0xFF); + this.compressed_offset = this.compressed_data.writeUInt8((this.current_literal >>> 0x18) & 0xFF, this.compressed_offset); this.current_length -= 8; this.current_literal = (this.current_literal << 8) & 0xFFFFFFFF; @@ -312,12 +314,19 @@ class WTVLzpf { this.clear(); if (uncompressed_data.words) { - uncompressed_data = new Buffer.from(this.wordArrayToUint8Array(uncompressed_data)); - } else { + // if its a wordArray convert it to a Buffer + + // Barrow function from WTVSec class + const WTVSec = require("./WTVSec.js"); + var wtvsec = new WTVSec(); + uncompressed_data = new Buffer.from(wtvsec.wordArrayToUint8Array(uncompressed_data)); + } else if (!uncompressed_data.byteLength) { + // otherwise if its not already a Buffer, convert it to one uncompressed_data = new Buffer.from(uncompressed_data); } - var uncompressed_len = uncompressed_data.length; + var uncompressed_len = uncompressed_data.byteLength; + this.compressed_data = new Buffer.alloc(uncompressed_len); var i = 0; var sum = 0; @@ -403,10 +412,14 @@ class WTVLzpf { this.EncodeLiteral(0x08, (sum << 0x18) & 0xFFFFFFFF); // End - this.compressed_data.push(this.current_literal >>> 0x18); - this.compressed_data.push(0x20); + this.compressed_offset = this.compressed_data.writeUInt8(this.current_literal >>> 0x18, this.compressed_offset); + this.compressed_offset = this.compressed_data.writeUInt8(0x20, this.compressed_offset); - return new Buffer.from(this.compressed_data); + var output_buffer = new Buffer.alloc(this.compressed_offset); + this.compressed_data.copy(output_buffer, 0, 0, this.compressed_offset) + this.compressed_data = null; + + return output_buffer; } } diff --git a/zefie_wtvp_minisrv/app.js b/zefie_wtvp_minisrv/app.js index 042c9b65..9368b09b 100644 --- a/zefie_wtvp_minisrv/app.js +++ b/zefie_wtvp_minisrv/app.js @@ -448,7 +448,7 @@ async function sendToClient(socket, headers_obj, data, compress_data = false) { return; } var wtv_connection_close = headers_obj["wtv-connection-close"]; - if (typeof(headers_obj["wtv-connection-close"]) != 'undefined') delete headers_obj["wtv-connection-close"]; + if (typeof (headers_obj["wtv-connection-close"]) != 'undefined') delete headers_obj["wtv-connection-close"]; // add Connection header if missing, default to Keep-Alive if (!headers_obj.Connection) { @@ -468,12 +468,27 @@ async function sendToClient(socket, headers_obj, data, compress_data = false) { compress_data = true; } + // fix captialization + if (headers_obj["Content-type"]) { + headers_obj["Content-Type"] = headers_obj["Content-type"]; + delete headers_obj["Content-type"]; + } + + + // if box can do compression, see if its worth enabling + if (ssid_sessions[socket.ssid].capabilities) { + if (ssid_sessions[socket.ssid].capabilities['client-can-receive-compressed-data']) { + compress_data = shouldWeCompress(headers_obj["Content-Type"]); + } + } + // compress if needed if (compress_data && clen > 0) { - headers_obj["wtv-lzpf"] = "0"; - - var lzpf = new WTVLzpf(); - data = lzpf.Compress(data); + headers_obj["wtv-lzpf"] = 0; + var wtvcomp = new WTVLzpf(); + var uncomp_data = data; + data = wtvcomp.Compress(uncomp_data); + uncomp_data, wtvcomp = null; } // encrypt if needed @@ -487,17 +502,11 @@ async function sendToClient(socket, headers_obj, data, compress_data = false) { } } - // fix captialization - if (headers_obj["Content-length"]) { - delete headers_obj["Content-length"]; - } - - if (headers_obj["Content-type"]) { - headers_obj["Content-Type"] = headers_obj["Content-type"]; - delete headers_obj["Content-type"]; - } - // calculate content length + // make sure we are using our Content-length and not one set in a script. + if (headers_obj["Content-Length"]) delete headers_obj["Content-Length"]; + if (headers_obj["Content-length"]) delete headers_obj["Content-length"]; + // On the WNI server this is the length before compression but we're using the length after compression. // It matches the HTTP spec anyway so leaving. if (typeof data.length !== 'undefined') { @@ -575,6 +584,19 @@ async function sendToClient(socket, headers_obj, data, compress_data = false) { } } +function shouldWeCompress(content_type) { + if (typeof (content_type) != 'undefined') { + if ((content_type.match(/^text\//) && content_type != "text/tellyscript") || + content_type.match(/^application\/(x-?)javascript$/) || + content_type.match(/^audio\/(x-)?midi/) || + content_type.match(/^audio\/(x-)?wav/) || + content_type == "application/json") { + return true; + } + } + return false; +} + function concatArrayBuffer(buffer1, buffer2) { var tmp = new Uint8Array(buffer1.byteLength + buffer2.byteLength); tmp.set(new Uint8Array(buffer1), 0); @@ -1091,14 +1113,22 @@ async function cleanupSocket(socket) { if (socket.ssid) { ssid_sessions[socket.ssid].data_store.sockets.delete(socket); - if (ssid_sessions[socket.ssid].data_store.sockets.size === 0 && ssid_sessions[socket.ssid].data_store.wtvsec_login) { - // if last socket for SSID disconnected, destroy login session - if (!zquiet) console.log(" * Last socket from WebTV SSID", filterSSID(socket.ssid),"disconnected, cleaning up primary WTVSec instance for this SSID"); - ssid_sessions[socket.ssid].delete("wtvsec_login"); - + if (ssid_sessions[socket.ssid].currentConnections() === 0) { // clean up possible minibrowser session data if (ssid_sessions[socket.ssid].get("wtv-needs-upgrade")) ssid_sessions[socket.ssid].delete("wtv-needs-upgrade"); if (ssid_sessions[socket.ssid].get("wtv-used-8675309")) ssid_sessions[socket.ssid].delete("wtv-used-8675309"); + + // set timer to destroy entirety of session data if client does not return in X time + var timeout = 180000; // timeout is in milliseconds, default 180000 (3 min) .. be sure to allow time for dialup reconnections + + if (!ssid_sessions[socket.ssid].data_store.socket_check) { + ssid_sessions[socket.ssid].data_store.socket_check = setTimeout(function (ssid) { + if (ssid_sessions[ssid].currentConnections() === 0) { + if (!zquiet) console.log(" * WebTV SSID", filterSSID(ssid), " has not been seen in", (timeout / 1000), "seconds, cleaning up session data for this SSID"); + delete ssid_sessions[ssid]; + } + }, timeout, socket.ssid); + } } } socket.end(); diff --git a/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj b/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj index d8923f44..eabe471a 100644 --- a/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj +++ b/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj @@ -186,7 +186,10 @@ - + + Code + + Code From 3da637dfb01232eb07039f5a571fc29ae39fa25f Mon Sep 17 00:00:00 2001 From: zefie Date: Fri, 6 Aug 2021 16:20:44 -0400 Subject: [PATCH 2/9] update: compression system: make asynchronous --- zefie_wtvp_minisrv/WTVLzpf.js | 38 +++++++++++++++++++++++------------ zefie_wtvp_minisrv/app.js | 23 +++++++++++++++++---- 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/zefie_wtvp_minisrv/WTVLzpf.js b/zefie_wtvp_minisrv/WTVLzpf.js index 380354a6..1a25127d 100644 --- a/zefie_wtvp_minisrv/WTVLzpf.js +++ b/zefie_wtvp_minisrv/WTVLzpf.js @@ -1,3 +1,5 @@ +var EventEmitter = require('events').EventEmitter; + /** * Pure-JS implementation of WebTV's LZPF compression * @@ -7,12 +9,14 @@ * By: Eric MacDonald (eMac) */ -class WTVLzpf { - // Note: currentlty doesn't offer streaming support but this is good enough to meet perf demands at the scale we're at. +class WTVLzpf extends EventEmitter { + // Note: currentlty doesn't offer optimal streaming support but this is good enough to meet perf demands at the scale we're at. current_length = 0 current_literal = 0 compressed_offset = 0 + total_compressed = 0; + chunk_size = 65535; flag_table = new Uint16Array(0x1000) compressed_data = null; @@ -261,23 +265,28 @@ class WTVLzpf { /** * Initialize the Lzpf class. * + * @param chunk_size {Number} Set the size of the compressed data chunks to fire off with 'data' event. (suggested betweet 2048 and 65535); * @returns {undefined} */ - constructor() { + constructor(chunk_size = 0) { + super(); // for extended class + if (chunk_size > 0) this.chunk_size = chunk_size; this.clear(); } /** * Sets starting values for the compression algorithm. * + * @param length {Number} size of compressed data Buffer to allocate * @returns {undefined} */ - clear() { + clear(length = 0) { this.current_length = 0; this.current_literal = 0; + this.total_compressed = 0; this.compressed_offset = 0; this.flag_table.fill(0xFFFF, 0, 0x1000); - this.compressed_data = null; + this.compressed_data = (length > 0) ? null : Buffer.alloc(length); } /** @@ -288,7 +297,7 @@ class WTVLzpf { * * @returns {undefined} */ - EncodeLiteral(code_length, code) { + async EncodeLiteral(code_length, code) { // Using >>> to stick with unsigned integers without making a mess with casting. this.current_literal |= code >>> (this.current_length & 0x1F); @@ -310,9 +319,8 @@ class WTVLzpf { * * @returns {Buffer} Lzpf compression data */ - Compress(uncompressed_data) { + async Compress(uncompressed_data) { this.clear(); - if (uncompressed_data.words) { // if its a wordArray convert it to a Buffer @@ -326,7 +334,8 @@ class WTVLzpf { } var uncompressed_len = uncompressed_data.byteLength; - this.compressed_data = new Buffer.alloc(uncompressed_len); + this.clear(uncompressed_len); + this.compressed_data = new Buffer.alloc((uncompressed_len >= this.chunk_size) ? this.chunk_size : uncompressed_len); var i = 0; var sum = 0; @@ -385,6 +394,11 @@ class WTVLzpf { if (code_length > 0) { this.EncodeLiteral(code_length, code); } + if (this.compressed_offset == this.chunk_size) { + this.total_compressed += this.compressed_offset; + this.emit('data', this.compressed_data, this.compressed_offset, (this.total_compressed - this.compressed_offset), false); + this.compressed_offset = 0; + } } // Finish up. This would normally be in an Lzpf_Finish method. @@ -415,11 +429,9 @@ class WTVLzpf { this.compressed_offset = this.compressed_data.writeUInt8(this.current_literal >>> 0x18, this.compressed_offset); this.compressed_offset = this.compressed_data.writeUInt8(0x20, this.compressed_offset); - var output_buffer = new Buffer.alloc(this.compressed_offset); - this.compressed_data.copy(output_buffer, 0, 0, this.compressed_offset) - this.compressed_data = null; + this.total_compressed += this.compressed_offset; - return output_buffer; + this.emit('data', this.compressed_data, this.compressed_offset, (this.total_compressed - this.compressed_offset), this.total_compressed); } } diff --git a/zefie_wtvp_minisrv/app.js b/zefie_wtvp_minisrv/app.js index 9368b09b..b22385f4 100644 --- a/zefie_wtvp_minisrv/app.js +++ b/zefie_wtvp_minisrv/app.js @@ -463,6 +463,7 @@ async function sendToClient(socket, headers_obj, data, compress_data = false) { clen = data.byteLength; } + // If wtv-lzpf is in the header then force compression if (headers_obj["wtv-lzpf"]) { compress_data = true; @@ -483,14 +484,28 @@ async function sendToClient(socket, headers_obj, data, compress_data = false) { } // compress if needed - if (compress_data && clen > 0) { + if (compress_data && clen > 0 && !headers_obj['minisrv-already-compressed']) { + if (zdebug) console.log(" # Uncompressed data length:", clen); headers_obj["wtv-lzpf"] = 0; var wtvcomp = new WTVLzpf(); - var uncomp_data = data; - data = wtvcomp.Compress(uncomp_data); - uncomp_data, wtvcomp = null; + var compressed_data = new Buffer.alloc(clen); + wtvcomp.on('data', (data, length, offset, complete) => { + data.copy(compressed_data, offset, 0, length); + if (complete !== false) { + data = new Buffer.alloc(complete); + compressed_data.copy(data, 0, 0, compressed_data.byteLength); + compressed_data, wtvcomp = null; + headers_obj['minisrv-already-compressed'] = true; + sendToClient(socket, headers_obj, data); + } + }); + wtvcomp.Compress(data); + return; } + if (headers_obj['minisrv-already-compressed']) delete headers_obj['minisrv-already-compressed']; + + // encrypt if needed if (socket_sessions[socket.id].secure == true) { headers_obj["wtv-encrypted"] = 'true'; From 6a286d6851f7918f954348b619dfc30762cd5e82 Mon Sep 17 00:00:00 2001 From: Eric MacDonald Date: Sat, 7 Aug 2021 14:52:09 -0400 Subject: [PATCH 3/9] Change lookback to ring buffer --- zefie_wtvp_minisrv/WTVLzpf.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/zefie_wtvp_minisrv/WTVLzpf.js b/zefie_wtvp_minisrv/WTVLzpf.js index 1a25127d..4bd1e574 100644 --- a/zefie_wtvp_minisrv/WTVLzpf.js +++ b/zefie_wtvp_minisrv/WTVLzpf.js @@ -18,6 +18,7 @@ class WTVLzpf extends EventEmitter { total_compressed = 0; chunk_size = 65535; flag_table = new Uint16Array(0x1000) + ring_buffer = new Uint8Array(0x2000) compressed_data = null; nomatchEncode = [ @@ -285,6 +286,7 @@ class WTVLzpf extends EventEmitter { this.current_literal = 0; this.total_compressed = 0; this.compressed_offset = 0; + this.ring_buffer.fill(0x00, 0, 0x2000) this.flag_table.fill(0xFFFF, 0, 0x1000); this.compressed_data = (length > 0) ? null : Buffer.alloc(length); } @@ -349,16 +351,17 @@ class WTVLzpf extends EventEmitter { var code = -1; var byte = uncompressed_data.readUInt8(i); + this.ring_buffer[i & 0x1FFF] = byte; if(match_index > 0) { - if (byte != uncompressed_data.readUInt8(flag) || match_index > 0x0127) { + if (byte != this.ring_buffer[flag] || match_index > 0x0127) { code_length = this.matchEncode[match_index][1]; code = this.matchEncode[match_index][0]; match_index = 0; type_index = 3; } else { - match_index++; - flag++; + match_index = (match_index + 1) & 0x1FFF; + flag = (flag + 1) & 0x1FFF; sum = (sum + byte) & 0xFFFF; working_data = ((working_data * 0x0100) + byte) & 0xFFFFFFFF; i++; @@ -369,7 +372,7 @@ class WTVLzpf extends EventEmitter { if (i >= 3) { flags_index = (working_data >>> 0x0B ^ working_data) & 0x0FFF; flag = this.flag_table[flags_index]; - this.flag_table[flags_index] = i; + this.flag_table[flags_index] = i & 0x1FFF; } else { type_index++; } @@ -377,9 +380,9 @@ class WTVLzpf extends EventEmitter { if (flag == 0xFFFF) { code_length = this.nomatchEncode[byte][1]; code = this.nomatchEncode[byte][0] << 0x10; - } else if (byte == uncompressed_data.readUInt8(flag)) { + } else if (byte == this.ring_buffer[flag]) { match_index = 1; - flag++; + flag = (flag + 1) & 0x1FFF; type_index = 4; } else { code_length = this.nomatchEncode[byte][1] + 1; From 7a69d3e8da3120566a10e26313e2d645021c0890 Mon Sep 17 00:00:00 2001 From: Eric MacDonald Date: Sat, 7 Aug 2021 15:11:18 -0400 Subject: [PATCH 4/9] Remove Zefie's chunking and async stuff. Will revisit later. Fixing alg first. --- zefie_wtvp_minisrv/WTVLzpf.js | 44 ++++++++++------------------------- zefie_wtvp_minisrv/app.js | 22 ++++++------------ 2 files changed, 19 insertions(+), 47 deletions(-) diff --git a/zefie_wtvp_minisrv/WTVLzpf.js b/zefie_wtvp_minisrv/WTVLzpf.js index 4bd1e574..d9cc6f24 100644 --- a/zefie_wtvp_minisrv/WTVLzpf.js +++ b/zefie_wtvp_minisrv/WTVLzpf.js @@ -9,17 +9,15 @@ var EventEmitter = require('events').EventEmitter; * By: Eric MacDonald (eMac) */ -class WTVLzpf extends EventEmitter { +class WTVLzpf { // Note: currentlty doesn't offer optimal streaming support but this is good enough to meet perf demands at the scale we're at. current_length = 0 current_literal = 0 - compressed_offset = 0 - total_compressed = 0; chunk_size = 65535; flag_table = new Uint16Array(0x1000) ring_buffer = new Uint8Array(0x2000) - compressed_data = null; + compressed_data = []; nomatchEncode = [ [0x0000, 0x10], [0x0001, 0x10], [0x0002, 0x10], @@ -266,29 +264,25 @@ class WTVLzpf extends EventEmitter { /** * Initialize the Lzpf class. * - * @param chunk_size {Number} Set the size of the compressed data chunks to fire off with 'data' event. (suggested betweet 2048 and 65535); * @returns {undefined} */ - constructor(chunk_size = 0) { - super(); // for extended class - if (chunk_size > 0) this.chunk_size = chunk_size; + constructor() { this.clear(); } /** * Sets starting values for the compression algorithm. * - * @param length {Number} size of compressed data Buffer to allocate * @returns {undefined} */ - clear(length = 0) { + clear() { this.current_length = 0; this.current_literal = 0; this.total_compressed = 0; this.compressed_offset = 0; this.ring_buffer.fill(0x00, 0, 0x2000) this.flag_table.fill(0xFFFF, 0, 0x1000); - this.compressed_data = (length > 0) ? null : Buffer.alloc(length); + this.compressed_data = []; } /** @@ -306,8 +300,7 @@ class WTVLzpf extends EventEmitter { this.current_length += code_length; while (this.current_length > 7) { - //console.log("add", this.current_literal >>> 0, code >>> 0, byte, type) - this.compressed_offset = this.compressed_data.writeUInt8((this.current_literal >>> 0x18) & 0xFF, this.compressed_offset); + this.compressed_data.push((this.current_literal >>> 0x18) & 0xFF); this.current_length -= 8; this.current_literal = (this.current_literal << 8) & 0xFFFFFFFF; @@ -321,23 +314,17 @@ class WTVLzpf extends EventEmitter { * * @returns {Buffer} Lzpf compression data */ - async Compress(uncompressed_data) { + Compress(uncompressed_data) { this.clear(); - if (uncompressed_data.words) { - // if its a wordArray convert it to a Buffer - // Barrow function from WTVSec class - const WTVSec = require("./WTVSec.js"); - var wtvsec = new WTVSec(); + if (uncompressed_data.words) { uncompressed_data = new Buffer.from(wtvsec.wordArrayToUint8Array(uncompressed_data)); } else if (!uncompressed_data.byteLength) { // otherwise if its not already a Buffer, convert it to one uncompressed_data = new Buffer.from(uncompressed_data); } - var uncompressed_len = uncompressed_data.byteLength; - this.clear(uncompressed_len); - this.compressed_data = new Buffer.alloc((uncompressed_len >= this.chunk_size) ? this.chunk_size : uncompressed_len); + var uncompressed_len = uncompressed_data.length; var i = 0; var sum = 0; @@ -397,11 +384,6 @@ class WTVLzpf extends EventEmitter { if (code_length > 0) { this.EncodeLiteral(code_length, code); } - if (this.compressed_offset == this.chunk_size) { - this.total_compressed += this.compressed_offset; - this.emit('data', this.compressed_data, this.compressed_offset, (this.total_compressed - this.compressed_offset), false); - this.compressed_offset = 0; - } } // Finish up. This would normally be in an Lzpf_Finish method. @@ -429,12 +411,10 @@ class WTVLzpf extends EventEmitter { this.EncodeLiteral(0x08, (sum << 0x18) & 0xFFFFFFFF); // End - this.compressed_offset = this.compressed_data.writeUInt8(this.current_literal >>> 0x18, this.compressed_offset); - this.compressed_offset = this.compressed_data.writeUInt8(0x20, this.compressed_offset); + this.compressed_data.push(this.current_literal >>> 0x18); + this.compressed_data.push(0x20); - this.total_compressed += this.compressed_offset; - - this.emit('data', this.compressed_data, this.compressed_offset, (this.total_compressed - this.compressed_offset), this.total_compressed); + return Buffer.from(this.compressed_data); } } diff --git a/zefie_wtvp_minisrv/app.js b/zefie_wtvp_minisrv/app.js index b22385f4..4b06389a 100644 --- a/zefie_wtvp_minisrv/app.js +++ b/zefie_wtvp_minisrv/app.js @@ -484,23 +484,15 @@ async function sendToClient(socket, headers_obj, data, compress_data = false) { } // compress if needed - if (compress_data && clen > 0 && !headers_obj['minisrv-already-compressed']) { - if (zdebug) console.log(" # Uncompressed data length:", clen); + if (compress_data && clen > 0) { headers_obj["wtv-lzpf"] = 0; + var wtvcomp = new WTVLzpf(); - var compressed_data = new Buffer.alloc(clen); - wtvcomp.on('data', (data, length, offset, complete) => { - data.copy(compressed_data, offset, 0, length); - if (complete !== false) { - data = new Buffer.alloc(complete); - compressed_data.copy(data, 0, 0, compressed_data.byteLength); - compressed_data, wtvcomp = null; - headers_obj['minisrv-already-compressed'] = true; - sendToClient(socket, headers_obj, data); - } - }); - wtvcomp.Compress(data); - return; + data = wtvcomp.Compress(data); + + console.log("data", data) + + wtvcomp = null; // Makes the garbage gods happy so it cleans up our mess } if (headers_obj['minisrv-already-compressed']) delete headers_obj['minisrv-already-compressed']; From 9fc9636b2c6c9426bb50c1bb3f50ce79d920178c Mon Sep 17 00:00:00 2001 From: Eric MacDonald Date: Sat, 7 Aug 2021 15:23:09 -0400 Subject: [PATCH 5/9] More removing Zefie's chunking --- zefie_wtvp_minisrv/WTVLzpf.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/zefie_wtvp_minisrv/WTVLzpf.js b/zefie_wtvp_minisrv/WTVLzpf.js index d9cc6f24..891b7b51 100644 --- a/zefie_wtvp_minisrv/WTVLzpf.js +++ b/zefie_wtvp_minisrv/WTVLzpf.js @@ -14,7 +14,6 @@ class WTVLzpf { current_length = 0 current_literal = 0 - chunk_size = 65535; flag_table = new Uint16Array(0x1000) ring_buffer = new Uint8Array(0x2000) compressed_data = []; @@ -278,8 +277,6 @@ class WTVLzpf { clear() { this.current_length = 0; this.current_literal = 0; - this.total_compressed = 0; - this.compressed_offset = 0; this.ring_buffer.fill(0x00, 0, 0x2000) this.flag_table.fill(0xFFFF, 0, 0x1000); this.compressed_data = []; From 1c8e0adbafcb7612f23534db0dee42daa07bab55 Mon Sep 17 00:00:00 2001 From: Eric MacDonald Date: Sat, 7 Aug 2021 15:24:24 -0400 Subject: [PATCH 6/9] More removing Zefie's asyc stuff --- zefie_wtvp_minisrv/WTVLzpf.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zefie_wtvp_minisrv/WTVLzpf.js b/zefie_wtvp_minisrv/WTVLzpf.js index 891b7b51..a508475d 100644 --- a/zefie_wtvp_minisrv/WTVLzpf.js +++ b/zefie_wtvp_minisrv/WTVLzpf.js @@ -290,7 +290,7 @@ class WTVLzpf { * * @returns {undefined} */ - async EncodeLiteral(code_length, code) { + EncodeLiteral(code_length, code) { // Using >>> to stick with unsigned integers without making a mess with casting. this.current_literal |= code >>> (this.current_length & 0x1F); From eb6fc5f89a94cc9bcaf53d57bf433c65adffd233 Mon Sep 17 00:00:00 2001 From: Eric MacDonald Date: Sat, 7 Aug 2021 15:29:20 -0400 Subject: [PATCH 7/9] Initial code for Lzpf stream compression --- zefie_wtvp_minisrv/WTVLzpf.js | 142 +++++++++++++++++++++++----------- 1 file changed, 95 insertions(+), 47 deletions(-) diff --git a/zefie_wtvp_minisrv/WTVLzpf.js b/zefie_wtvp_minisrv/WTVLzpf.js index a508475d..7146ca15 100644 --- a/zefie_wtvp_minisrv/WTVLzpf.js +++ b/zefie_wtvp_minisrv/WTVLzpf.js @@ -14,6 +14,10 @@ class WTVLzpf { current_length = 0 current_literal = 0 + working_data = 0; + match_index = 0; + type_index = 0; + checksum = 0; flag_table = new Uint16Array(0x1000) ring_buffer = new Uint8Array(0x2000) compressed_data = []; @@ -277,11 +281,26 @@ class WTVLzpf { clear() { this.current_length = 0; this.current_literal = 0; + this.working_data = 0; + this.match_index = 0; + this.type_index = 0; + this.checksum = 0; this.ring_buffer.fill(0x00, 0, 0x2000) this.flag_table.fill(0xFFFF, 0, 0x1000); this.compressed_data = []; } + /** + * Appends a byte to the end of the compressed byte array. Re-allocates as needed + * + * @param byte {Number} char code of the byte to be added. + * + * @returns {undefined} + */ + AddByte(byte) { + this.compressed_data.push(byte); + } + /** * Encodes a literal onto the compressed byte array. * @@ -297,13 +316,72 @@ class WTVLzpf { this.current_length += code_length; while (this.current_length > 7) { - this.compressed_data.push((this.current_literal >>> 0x18) & 0xFF); + this.AddByte((this.current_literal >>> 0x18) & 0xFF); this.current_length -= 8; this.current_literal = (this.current_literal << 8) & 0xFFFFFFFF; } } + /** + * Starts a compression stream + * + * @returns {undefined} Lzpf compression data + */ + Begin() { + this.clear(); + } + + /** + * Compress a block of data. Used for streamed chunks. + * + * @param uncompressed_data {String} data to compress + * + * @returns {Buffer} Lzpf compression data + */ + CompressBlock(uncompressed_data) { + } + + /** + * Ends a compression stream. + * + * @param type_index {Number} the end type used to finalize + * + * @returns {Buffer} Lzpf compression data + */ + Finalize() { + var code_length = -1 + var code = -1 + + if (this.type_index == 2) { + this.EncodeLiteral(0x10, 0x00990000); + } else if (this.type_index >= 3) { + if (this.type_index == 4) { + code_length = this.matchEncode[this.match_index][1]; + code = this.matchEncode[this.match_index][0]; + this.EncodeLiteral(code_length, code); + } + + var flags_index = (this.working_data >>> 0x0B ^ this.working_data) & 0x0FFF; + var flag = this.flag_table[flags_index]; + if (flag == 0xFFFF) { + this.EncodeLiteral(0x10, 0x00990000); + } else { + this.EncodeLiteral(0x11, 0x004c8000); + } + } + + // Below is just metadata. The compressed block is complete. + + // Encode checksum + this.EncodeLiteral(0x08, (this.checksum << 0x10) & 0xFFFFFFFF); + this.EncodeLiteral(0x08, (this.checksum << 0x18) & 0xFFFFFFFF); + + // End + this.AddByte((this.current_literal >>> 0x18) & 0xFF); + this.AddByte(0x20); + } + /** * Compress data using WebTV's Lzpf compression algorithm and adds the footer to the end. * @@ -324,12 +402,8 @@ class WTVLzpf { var uncompressed_len = uncompressed_data.length; var i = 0; - var sum = 0; - var working_data = 0; var flag = 0xFFFF; var flags_index = 0; - var match_index = 0; - var type_index = 0; while(i < uncompressed_len) { var code_length = -1; var code = -1; @@ -337,44 +411,44 @@ class WTVLzpf { var byte = uncompressed_data.readUInt8(i); this.ring_buffer[i & 0x1FFF] = byte; - if(match_index > 0) { - if (byte != this.ring_buffer[flag] || match_index > 0x0127) { - code_length = this.matchEncode[match_index][1]; - code = this.matchEncode[match_index][0]; - match_index = 0; - type_index = 3; + if (this.match_index > 0) { + if (byte != this.ring_buffer[flag] || this.match_index > 0x0127) { + code_length = this.matchEncode[this.match_index][1]; + code = this.matchEncode[this.match_index][0]; + this.match_index = 0; + this.type_index = 3; } else { - match_index = (match_index + 1) & 0x1FFF; + this.match_index = (this.match_index + 1) & 0x1FFF; flag = (flag + 1) & 0x1FFF; - sum = (sum + byte) & 0xFFFF; - working_data = ((working_data * 0x0100) + byte) & 0xFFFFFFFF; + this.checksum = (this.checksum + byte) & 0xFFFF; + this.working_data = ((this.working_data * 0x0100) + byte) & 0xFFFFFFFF; i++; } } else { flag = 0xFFFF; if (i >= 3) { - flags_index = (working_data >>> 0x0B ^ working_data) & 0x0FFF; + flags_index = (this.working_data >>> 0x0B ^ this.working_data) & 0x0FFF; flag = this.flag_table[flags_index]; this.flag_table[flags_index] = i & 0x1FFF; } else { - type_index++; + this.type_index++; } if (flag == 0xFFFF) { code_length = this.nomatchEncode[byte][1]; code = this.nomatchEncode[byte][0] << 0x10; } else if (byte == this.ring_buffer[flag]) { - match_index = 1; + this.match_index = 1; flag = (flag + 1) & 0x1FFF; - type_index = 4; + this.type_index = 4; } else { code_length = this.nomatchEncode[byte][1] + 1; code = this.nomatchEncode[byte][0] << 0x0F; } - sum = (sum + byte) & 0xFFFF; - working_data = ((working_data * 0x0100) + byte) & 0xFFFFFFFF; + this.checksum = (this.checksum + byte) & 0xFFFF; + this.working_data = ((this.working_data * 0x0100) + byte) & 0xFFFFFFFF; i++; } @@ -383,33 +457,7 @@ class WTVLzpf { } } - // Finish up. This would normally be in an Lzpf_Finish method. - if(type_index == 2) { - this.EncodeLiteral(0x10, 0x00990000); - } else if(type_index >= 3) { - if(type_index == 4) { - code_length = this.matchEncode[match_index][1]; - code = this.matchEncode[match_index][0]; - this.EncodeLiteral(code_length, code); - } - - flags_index = (working_data >>> 0x0B ^ working_data) & 0x0FFF; - if (flags_index == 0xFFFF) { - this.EncodeLiteral(0x10, 0x00990000); - } else { - this.EncodeLiteral(0x11, 0x004c8000); - } - } - - // Below is just metadata. The compressed block is complete. - - // Encode checksum - this.EncodeLiteral(0x08, (sum << 0x10) & 0xFFFFFFFF); - this.EncodeLiteral(0x08, (sum << 0x18) & 0xFFFFFFFF); - - // End - this.compressed_data.push(this.current_literal >>> 0x18); - this.compressed_data.push(0x20); + this.Finalize(); return Buffer.from(this.compressed_data); } From e6958fb2c3f372d6fdc9798c21023af0caf87f4a Mon Sep 17 00:00:00 2001 From: Eric MacDonald Date: Sat, 7 Aug 2021 16:10:40 -0400 Subject: [PATCH 8/9] 'Working' Lzpf stream compression --- zefie_wtvp_minisrv/WTVLzpf.js | 146 +++++++++++++++++----------------- 1 file changed, 75 insertions(+), 71 deletions(-) diff --git a/zefie_wtvp_minisrv/WTVLzpf.js b/zefie_wtvp_minisrv/WTVLzpf.js index 7146ca15..66cfe72f 100644 --- a/zefie_wtvp_minisrv/WTVLzpf.js +++ b/zefie_wtvp_minisrv/WTVLzpf.js @@ -12,8 +12,9 @@ var EventEmitter = require('events').EventEmitter; class WTVLzpf { // Note: currentlty doesn't offer optimal streaming support but this is good enough to meet perf demands at the scale we're at. - current_length = 0 - current_literal = 0 + current_length = 0; + current_literal = 0; + flag = 0xFFFF; working_data = 0; match_index = 0; type_index = 0; @@ -281,6 +282,7 @@ class WTVLzpf { clear() { this.current_length = 0; this.current_literal = 0; + this.flag = 0xFFFF; this.working_data = 0; this.match_index = 0; this.type_index = 0; @@ -340,6 +342,73 @@ class WTVLzpf { * @returns {Buffer} Lzpf compression data */ CompressBlock(uncompressed_data) { + this.compressed_data = []; + + if (uncompressed_data.words) { + uncompressed_data = new Buffer.from(wtvsec.wordArrayToUint8Array(uncompressed_data)); + } else if (!uncompressed_data.byteLength) { + // otherwise if its not already a Buffer, convert it to one + uncompressed_data = new Buffer.from(uncompressed_data); + } + + var uncompressed_len = uncompressed_data.length; + + var i = 0; + var flags_index = 0; + while (i < uncompressed_len) { + var code_length = -1; + var code = -1; + + var byte = uncompressed_data.readUInt8(i); + this.ring_buffer[i & 0x1FFF] = byte; + + if (this.match_index > 0) { + if (byte != this.ring_buffer[this.flag] || this.match_index > 0x0127) { + code_length = this.matchEncode[this.match_index][1]; + code = this.matchEncode[this.match_index][0]; + this.match_index = 0; + this.type_index = 3; + } else { + this.match_index = (this.match_index + 1) & 0x1FFF; + this.flag = (this.flag + 1) & 0x1FFF; + this.checksum = (this.checksum + byte) & 0xFFFF; + this.working_data = ((this.working_data * 0x0100) + byte) & 0xFFFFFFFF; + i++; + } + } else { + this.flag = 0xFFFF; + + if (i >= 3) { + flags_index = (this.working_data >>> 0x0B ^ this.working_data) & 0x0FFF; + this.flag = this.flag_table[flags_index]; + this.flag_table[flags_index] = i & 0x1FFF; + } else { + this.type_index++; + } + + if (this.flag == 0xFFFF) { + code_length = this.nomatchEncode[byte][1]; + code = this.nomatchEncode[byte][0] << 0x10; + } else if (byte == this.ring_buffer[this.flag]) { + this.match_index = 1; + this.flag = (this.flag + 1) & 0x1FFF; + this.type_index = 4; + } else { + code_length = this.nomatchEncode[byte][1] + 1; + code = this.nomatchEncode[byte][0] << 0x0F; + } + + this.checksum = (this.checksum + byte) & 0xFFFF; + this.working_data = ((this.working_data * 0x0100) + byte) & 0xFFFFFFFF; + i++; + } + + if (code_length > 0) { + this.EncodeLiteral(code_length, code); + } + } + + return Buffer.from(this.compressed_data); } /** @@ -349,7 +418,7 @@ class WTVLzpf { * * @returns {Buffer} Lzpf compression data */ - Finalize() { + Finish() { var code_length = -1 var code = -1 @@ -390,74 +459,9 @@ class WTVLzpf { * @returns {Buffer} Lzpf compression data */ Compress(uncompressed_data) { - this.clear(); - - if (uncompressed_data.words) { - uncompressed_data = new Buffer.from(wtvsec.wordArrayToUint8Array(uncompressed_data)); - } else if (!uncompressed_data.byteLength) { - // otherwise if its not already a Buffer, convert it to one - uncompressed_data = new Buffer.from(uncompressed_data); - } - - var uncompressed_len = uncompressed_data.length; - - var i = 0; - var flag = 0xFFFF; - var flags_index = 0; - while(i < uncompressed_len) { - var code_length = -1; - var code = -1; - - var byte = uncompressed_data.readUInt8(i); - this.ring_buffer[i & 0x1FFF] = byte; - - if (this.match_index > 0) { - if (byte != this.ring_buffer[flag] || this.match_index > 0x0127) { - code_length = this.matchEncode[this.match_index][1]; - code = this.matchEncode[this.match_index][0]; - this.match_index = 0; - this.type_index = 3; - } else { - this.match_index = (this.match_index + 1) & 0x1FFF; - flag = (flag + 1) & 0x1FFF; - this.checksum = (this.checksum + byte) & 0xFFFF; - this.working_data = ((this.working_data * 0x0100) + byte) & 0xFFFFFFFF; - i++; - } - } else { - flag = 0xFFFF; - - if (i >= 3) { - flags_index = (this.working_data >>> 0x0B ^ this.working_data) & 0x0FFF; - flag = this.flag_table[flags_index]; - this.flag_table[flags_index] = i & 0x1FFF; - } else { - this.type_index++; - } - - if (flag == 0xFFFF) { - code_length = this.nomatchEncode[byte][1]; - code = this.nomatchEncode[byte][0] << 0x10; - } else if (byte == this.ring_buffer[flag]) { - this.match_index = 1; - flag = (flag + 1) & 0x1FFF; - this.type_index = 4; - } else { - code_length = this.nomatchEncode[byte][1] + 1; - code = this.nomatchEncode[byte][0] << 0x0F; - } - - this.checksum = (this.checksum + byte) & 0xFFFF; - this.working_data = ((this.working_data * 0x0100) + byte) & 0xFFFFFFFF; - i++; - } - - if (code_length > 0) { - this.EncodeLiteral(code_length, code); - } - } - - this.Finalize(); + this.Begin(); + this.CompressBlock(uncompressed_data) + this.Finish(); return Buffer.from(this.compressed_data); } From 36a2aebe174b760ed01fb93b41130c0772d1a835 Mon Sep 17 00:00:00 2001 From: Eric MacDonald Date: Sat, 7 Aug 2021 16:43:48 -0400 Subject: [PATCH 9/9] Remove wtv-lzpf detection to do stream comp outside sendToClient --- zefie_wtvp_minisrv/app.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/zefie_wtvp_minisrv/app.js b/zefie_wtvp_minisrv/app.js index 4b06389a..5da330d1 100644 --- a/zefie_wtvp_minisrv/app.js +++ b/zefie_wtvp_minisrv/app.js @@ -463,12 +463,6 @@ async function sendToClient(socket, headers_obj, data, compress_data = false) { clen = data.byteLength; } - - // If wtv-lzpf is in the header then force compression - if (headers_obj["wtv-lzpf"]) { - compress_data = true; - } - // fix captialization if (headers_obj["Content-type"]) { headers_obj["Content-Type"] = headers_obj["Content-type"];