diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-music/get-playlist.js b/zefie_wtvp_minisrv/ServiceVault/wtv-music/get-playlist.js index 0f1575a6..28e9685d 100644 --- a/zefie_wtvp_minisrv/ServiceVault/wtv-music/get-playlist.js +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-music/get-playlist.js @@ -31,4 +31,5 @@ wtv-backgroundmusic-add: wtv-music:/MusicCache/headspace/RMF/underground/renegad wtv-backgroundmusic-add: wtv-music:/MusicCache/headspace/RMF/moods/affectionate.rmf wtv-backgroundmusic-add: wtv-music:/MusicCache/headspace/RMF/moods/sociable.rmf`; -*/ \ No newline at end of file +*/ +data = ''; diff --git a/zefie_wtvp_minisrv/WTVLzpf.js b/zefie_wtvp_minisrv/WTVLzpf.js index 1c95cd51..66cfe72f 100644 --- a/zefie_wtvp_minisrv/WTVLzpf.js +++ b/zefie_wtvp_minisrv/WTVLzpf.js @@ -1,3 +1,5 @@ +var EventEmitter = require('events').EventEmitter; + /** * Pure-JS implementation of WebTV's LZPF compression * @@ -8,12 +10,18 @@ */ class WTVLzpf { - // Note: currentlty doesn't offer streaming support but this is good enough to meet perf demands at the scale we're at. + // Note: currentlty doesn't offer optimal streaming support but this is good enough to meet perf demands at the scale we're at. - current_length = 0 - current_literal = 0 + current_length = 0; + current_literal = 0; + flag = 0xFFFF; + working_data = 0; + match_index = 0; + type_index = 0; + checksum = 0; flag_table = new Uint16Array(0x1000) - compressed_data = [] + ring_buffer = new Uint8Array(0x2000) + compressed_data = []; nomatchEncode = [ [0x0000, 0x10], [0x0001, 0x10], [0x0002, 0x10], @@ -274,10 +282,27 @@ class WTVLzpf { clear() { this.current_length = 0; this.current_literal = 0; + this.flag = 0xFFFF; + this.working_data = 0; + this.match_index = 0; + this.type_index = 0; + this.checksum = 0; + this.ring_buffer.fill(0x00, 0, 0x2000) this.flag_table.fill(0xFFFF, 0, 0x1000); this.compressed_data = []; } + /** + * Appends a byte to the end of the compressed byte array. Re-allocates as needed + * + * @param byte {Number} char code of the byte to be added. + * + * @returns {undefined} + */ + AddByte(byte) { + this.compressed_data.push(byte); + } + /** * Encodes a literal onto the compressed byte array. * @@ -293,14 +318,139 @@ class WTVLzpf { this.current_length += code_length; while (this.current_length > 7) { - //console.log("add", this.current_literal >>> 0, code >>> 0, byte, type) - this.compressed_data.push((this.current_literal >>> 0x18) & 0xFF); + this.AddByte((this.current_literal >>> 0x18) & 0xFF); this.current_length -= 8; this.current_literal = (this.current_literal << 8) & 0xFFFFFFFF; } } + /** + * Starts a compression stream + * + * @returns {undefined} Lzpf compression data + */ + Begin() { + this.clear(); + } + + /** + * Compress a block of data. Used for streamed chunks. + * + * @param uncompressed_data {String} data to compress + * + * @returns {Buffer} Lzpf compression data + */ + CompressBlock(uncompressed_data) { + this.compressed_data = []; + + if (uncompressed_data.words) { + uncompressed_data = new Buffer.from(wtvsec.wordArrayToUint8Array(uncompressed_data)); + } else if (!uncompressed_data.byteLength) { + // otherwise if its not already a Buffer, convert it to one + uncompressed_data = new Buffer.from(uncompressed_data); + } + + var uncompressed_len = uncompressed_data.length; + + var i = 0; + var flags_index = 0; + while (i < uncompressed_len) { + var code_length = -1; + var code = -1; + + var byte = uncompressed_data.readUInt8(i); + this.ring_buffer[i & 0x1FFF] = byte; + + if (this.match_index > 0) { + if (byte != this.ring_buffer[this.flag] || this.match_index > 0x0127) { + code_length = this.matchEncode[this.match_index][1]; + code = this.matchEncode[this.match_index][0]; + this.match_index = 0; + this.type_index = 3; + } else { + this.match_index = (this.match_index + 1) & 0x1FFF; + this.flag = (this.flag + 1) & 0x1FFF; + this.checksum = (this.checksum + byte) & 0xFFFF; + this.working_data = ((this.working_data * 0x0100) + byte) & 0xFFFFFFFF; + i++; + } + } else { + this.flag = 0xFFFF; + + if (i >= 3) { + flags_index = (this.working_data >>> 0x0B ^ this.working_data) & 0x0FFF; + this.flag = this.flag_table[flags_index]; + this.flag_table[flags_index] = i & 0x1FFF; + } else { + this.type_index++; + } + + if (this.flag == 0xFFFF) { + code_length = this.nomatchEncode[byte][1]; + code = this.nomatchEncode[byte][0] << 0x10; + } else if (byte == this.ring_buffer[this.flag]) { + this.match_index = 1; + this.flag = (this.flag + 1) & 0x1FFF; + this.type_index = 4; + } else { + code_length = this.nomatchEncode[byte][1] + 1; + code = this.nomatchEncode[byte][0] << 0x0F; + } + + this.checksum = (this.checksum + byte) & 0xFFFF; + this.working_data = ((this.working_data * 0x0100) + byte) & 0xFFFFFFFF; + i++; + } + + if (code_length > 0) { + this.EncodeLiteral(code_length, code); + } + } + + return Buffer.from(this.compressed_data); + } + + /** + * Ends a compression stream. + * + * @param type_index {Number} the end type used to finalize + * + * @returns {Buffer} Lzpf compression data + */ + Finish() { + var code_length = -1 + var code = -1 + + if (this.type_index == 2) { + this.EncodeLiteral(0x10, 0x00990000); + } else if (this.type_index >= 3) { + if (this.type_index == 4) { + code_length = this.matchEncode[this.match_index][1]; + code = this.matchEncode[this.match_index][0]; + this.EncodeLiteral(code_length, code); + } + + var flags_index = (this.working_data >>> 0x0B ^ this.working_data) & 0x0FFF; + var flag = this.flag_table[flags_index]; + if (flag == 0xFFFF) { + this.EncodeLiteral(0x10, 0x00990000); + } else { + this.EncodeLiteral(0x11, 0x004c8000); + } + } + + // Below is just metadata. The compressed block is complete. + + // Encode checksum + this.EncodeLiteral(0x08, (this.checksum << 0x10) & 0xFFFFFFFF); + this.EncodeLiteral(0x08, (this.checksum << 0x18) & 0xFFFFFFFF); + + // End + this.AddByte((this.current_literal >>> 0x18) & 0xFF); + this.AddByte(0x20); + } + /** * Compress data using WebTV's Lzpf compression algorithm and adds the footer to the end. * @@ -309,104 +459,11 @@ class WTVLzpf { * @returns {Buffer} Lzpf compression data */ Compress(uncompressed_data) { - this.clear(); + this.Begin(); + this.CompressBlock(uncompressed_data) + this.Finish(); - if (uncompressed_data.words) { - uncompressed_data = new Buffer.from(this.wordArrayToUint8Array(uncompressed_data)); - } else { - uncompressed_data = new Buffer.from(uncompressed_data); - } - - var uncompressed_len = uncompressed_data.length; - - var i = 0; - var sum = 0; - var working_data = 0; - var flag = 0xFFFF; - var flags_index = 0; - var match_index = 0; - var type_index = 0; - while(i < uncompressed_len) { - var code_length = -1; - var code = -1; - - var byte = uncompressed_data.readUInt8(i); - - if(match_index > 0) { - if (byte != uncompressed_data.readUInt8(flag) || match_index > 0x0127) { - code_length = this.matchEncode[match_index][1]; - code = this.matchEncode[match_index][0]; - match_index = 0; - type_index = 3; - } else { - match_index++; - flag++; - sum = (sum + byte) & 0xFFFF; - working_data = ((working_data * 0x0100) + byte) & 0xFFFFFFFF; - i++; - } - } else { - flag = 0xFFFF; - - if (i >= 3) { - flags_index = (working_data >>> 0x0B ^ working_data) & 0x0FFF; - flag = this.flag_table[flags_index]; - this.flag_table[flags_index] = i; - } else { - type_index++; - } - - if (flag == 0xFFFF) { - code_length = this.nomatchEncode[byte][1]; - code = this.nomatchEncode[byte][0] << 0x10; - } else if (byte == uncompressed_data.readUInt8(flag)) { - match_index = 1; - flag++; - type_index = 4; - } else { - code_length = this.nomatchEncode[byte][1] + 1; - code = this.nomatchEncode[byte][0] << 0x0F; - } - - sum = (sum + byte) & 0xFFFF; - working_data = ((working_data * 0x0100) + byte) & 0xFFFFFFFF; - i++; - } - - if (code_length > 0) { - this.EncodeLiteral(code_length, code); - } - } - - // Finish up. This would normally be in an Lzpf_Finish method. - if(type_index == 2) { - this.EncodeLiteral(0x10, 0x00990000); - } else if(type_index >= 3) { - if(type_index == 4) { - code_length = this.matchEncode[match_index][1]; - code = this.matchEncode[match_index][0]; - this.EncodeLiteral(code_length, code); - } - - flags_index = (working_data >>> 0x0B ^ working_data) & 0x0FFF; - if (flags_index == 0xFFFF) { - this.EncodeLiteral(0x10, 0x00990000); - } else { - this.EncodeLiteral(0x11, 0x004c8000); - } - } - - // Below is just metadata. The compressed block is complete. - - // Encode checksum - this.EncodeLiteral(0x08, (sum << 0x10) & 0xFFFFFFFF); - this.EncodeLiteral(0x08, (sum << 0x18) & 0xFFFFFFFF); - - // End - this.compressed_data.push(this.current_literal >>> 0x18); - this.compressed_data.push(0x20); - - return new Buffer.from(this.compressed_data); + return Buffer.from(this.compressed_data); } } diff --git a/zefie_wtvp_minisrv/app.js b/zefie_wtvp_minisrv/app.js index 5fea9ed7..222716d3 100644 --- a/zefie_wtvp_minisrv/app.js +++ b/zefie_wtvp_minisrv/app.js @@ -522,19 +522,35 @@ async function sendToClient(socket, headers_obj, data, compress_data = false) { clen = data.byteLength; } - // If wtv-lzpf is in the header then force compression - if (headers_obj["wtv-lzpf"]) { - compress_data = true; + // fix captialization + if (headers_obj["Content-type"]) { + headers_obj["Content-Type"] = headers_obj["Content-type"]; + delete headers_obj["Content-type"]; + } + + + // if box can do compression, see if its worth enabling + if (ssid_sessions[socket.ssid].capabilities) { + if (ssid_sessions[socket.ssid].capabilities['client-can-receive-compressed-data']) { + compress_data = shouldWeCompress(headers_obj["Content-Type"]); + } } // compress if needed if (compress_data && clen > 0) { - headers_obj["wtv-lzpf"] = "0"; + headers_obj["wtv-lzpf"] = 0; - var lzpf = new WTVLzpf(); - data = lzpf.Compress(data); + var wtvcomp = new WTVLzpf(); + data = wtvcomp.Compress(data); + + console.log("data", data) + + wtvcomp = null; // Makes the garbage gods happy so it cleans up our mess } + if (headers_obj['minisrv-already-compressed']) delete headers_obj['minisrv-already-compressed']; + + // encrypt if needed if (socket_sessions[socket.id].secure == true) { headers_obj["wtv-encrypted"] = 'true'; @@ -546,17 +562,11 @@ async function sendToClient(socket, headers_obj, data, compress_data = false) { } } - // fix captialization - if (headers_obj["Content-length"]) { - delete headers_obj["Content-length"]; - } - - if (headers_obj["Content-type"]) { - headers_obj["Content-Type"] = headers_obj["Content-type"]; - delete headers_obj["Content-type"]; - } - // calculate content length + // make sure we are using our Content-length and not one set in a script. + if (headers_obj["Content-Length"]) delete headers_obj["Content-Length"]; + if (headers_obj["Content-length"]) delete headers_obj["Content-length"]; + // On the WNI server this is the length before compression but we're using the length after compression. // It matches the HTTP spec anyway so leaving. if (typeof data.length !== 'undefined') {