zefie/minisrv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'lzpf-compression'

Browse Source 
- fix issues caused by my rebasing
This commit is contained in:
zefie
2021-08-07 17:27:28 -04:00
parent 2d1893e450 36a2aebe17
commit 00f80c9ea5
3 changed files with 188 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
zefie_wtvp_minisrv/ServiceVault/wtv-music/get-playlist.js
View File

@@ -31,4 +31,5 @@ wtv-backgroundmusic-add: wtv-music:/MusicCache/headspace/RMF/underground/renegad
wtv-backgroundmusic-add: wtv-music:/MusicCache/headspace/RMF/moods/affectionate.rmf
wtv-backgroundmusic-add: wtv-music:/MusicCache/headspace/RMF/moods/sociable.rmf`;
*/
*/
data = '';

263
zefie_wtvp_minisrv/WTVLzpf.js
View File

@@ -1,3 +1,5 @@
var EventEmitter = require('events').EventEmitter;
/**
* Pure-JS implementation of WebTV's LZPF compression
*
@@ -8,12 +10,18 @@
*/
class WTVLzpf {
// Note: currentlty doesn't offer streaming support but this is good enough to meet perf demands at the scale we're at.
// Note: currentlty doesn't offer optimal streaming support but this is good enough to meet perf demands at the scale we're at.
current_length = 0
current_literal = 0
current_length = 0;
current_literal = 0;
flag = 0xFFFF;
working_data = 0;
match_index = 0;
type_index = 0;
checksum = 0;
flag_table = new Uint16Array(0x1000)
compressed_data = []
ring_buffer = new Uint8Array(0x2000)
compressed_data = [];
nomatchEncode = [
[0x0000, 0x10], [0x0001, 0x10], [0x0002, 0x10],
@@ -274,10 +282,27 @@ class WTVLzpf {
clear() {
this.current_length = 0;
this.current_literal = 0;
this.flag = 0xFFFF;
this.working_data = 0;
this.match_index = 0;
this.type_index = 0;
this.checksum = 0;
this.ring_buffer.fill(0x00, 0, 0x2000)
this.flag_table.fill(0xFFFF, 0, 0x1000);
this.compressed_data = [];
}
/**
* Appends a byte to the end of the compressed byte array. Re-allocates as needed
*
* @param byte {Number} char code of the byte to be added.
*
* @returns {undefined}
*/
AddByte(byte) {
this.compressed_data.push(byte);
}
/**
* Encodes a literal onto the compressed byte array.
*
@@ -293,14 +318,139 @@ class WTVLzpf {
this.current_length += code_length;
while (this.current_length > 7) {
//console.log("add", this.current_literal >>> 0, code >>> 0, byte, type)
this.compressed_data.push((this.current_literal >>> 0x18) & 0xFF);
this.AddByte((this.current_literal >>> 0x18) & 0xFF);
this.current_length -= 8;
this.current_literal = (this.current_literal << 8) & 0xFFFFFFFF;
}
}
/**
* Starts a compression stream
*
* @returns {undefined} Lzpf compression data
*/
Begin() {
this.clear();
}
/**
* Compress a block of data. Used for streamed chunks.
*
* @param uncompressed_data {String} data to compress
*
* @returns {Buffer} Lzpf compression data
*/
CompressBlock(uncompressed_data) {
this.compressed_data = [];
if (uncompressed_data.words) {
uncompressed_data = new Buffer.from(wtvsec.wordArrayToUint8Array(uncompressed_data));
} else if (!uncompressed_data.byteLength) {
// otherwise if its not already a Buffer, convert it to one
uncompressed_data = new Buffer.from(uncompressed_data);
}
var uncompressed_len = uncompressed_data.length;
var i = 0;
var flags_index = 0;
while (i < uncompressed_len) {
var code_length = -1;
var code = -1;
var byte = uncompressed_data.readUInt8(i);
this.ring_buffer[i & 0x1FFF] = byte;
if (this.match_index > 0) {
if (byte != this.ring_buffer[this.flag] || this.match_index > 0x0127) {
code_length = this.matchEncode[this.match_index][1];
code = this.matchEncode[this.match_index][0];
this.match_index = 0;
this.type_index = 3;
} else {
this.match_index = (this.match_index + 1) & 0x1FFF;
this.flag = (this.flag + 1) & 0x1FFF;
this.checksum = (this.checksum + byte) & 0xFFFF;
this.working_data = ((this.working_data * 0x0100) + byte) & 0xFFFFFFFF;
i++;
}
} else {
this.flag = 0xFFFF;
if (i >= 3) {
flags_index = (this.working_data >>> 0x0B ^ this.working_data) & 0x0FFF;
this.flag = this.flag_table[flags_index];
this.flag_table[flags_index] = i & 0x1FFF;
} else {
this.type_index++;
}
if (this.flag == 0xFFFF) {
code_length = this.nomatchEncode[byte][1];
code = this.nomatchEncode[byte][0] << 0x10;
} else if (byte == this.ring_buffer[this.flag]) {
this.match_index = 1;
this.flag = (this.flag + 1) & 0x1FFF;
this.type_index = 4;
} else {
code_length = this.nomatchEncode[byte][1] + 1;
code = this.nomatchEncode[byte][0] << 0x0F;
}
this.checksum = (this.checksum + byte) & 0xFFFF;
this.working_data = ((this.working_data * 0x0100) + byte) & 0xFFFFFFFF;
i++;
}
if (code_length > 0) {
this.EncodeLiteral(code_length, code);
}
}
return Buffer.from(this.compressed_data);
}
/**
* Ends a compression stream.
*
* @param type_index {Number} the end type used to finalize
*
* @returns {Buffer} Lzpf compression data
*/
Finish() {
var code_length = -1
var code = -1
if (this.type_index == 2) {
this.EncodeLiteral(0x10, 0x00990000);
} else if (this.type_index >= 3) {
if (this.type_index == 4) {
code_length = this.matchEncode[this.match_index][1];
code = this.matchEncode[this.match_index][0];
this.EncodeLiteral(code_length, code);
}
var flags_index = (this.working_data >>> 0x0B ^ this.working_data) & 0x0FFF;
var flag = this.flag_table[flags_index];
if (flag == 0xFFFF) {
this.EncodeLiteral(0x10, 0x00990000);
} else {
this.EncodeLiteral(0x11, 0x004c8000);
}
}
// Below is just metadata. The compressed block is complete.
// Encode checksum
this.EncodeLiteral(0x08, (this.checksum << 0x10) & 0xFFFFFFFF);
this.EncodeLiteral(0x08, (this.checksum << 0x18) & 0xFFFFFFFF);
// End
this.AddByte((this.current_literal >>> 0x18) & 0xFF);
this.AddByte(0x20);
}
/**
* Compress data using WebTV's Lzpf compression algorithm and adds the footer to the end.
*
@@ -309,104 +459,11 @@ class WTVLzpf {
* @returns {Buffer} Lzpf compression data
*/
Compress(uncompressed_data) {
this.clear();
this.Begin();
this.CompressBlock(uncompressed_data)
this.Finish();
if (uncompressed_data.words) {
uncompressed_data = new Buffer.from(this.wordArrayToUint8Array(uncompressed_data));
} else {
uncompressed_data = new Buffer.from(uncompressed_data);
}
var uncompressed_len = uncompressed_data.length;
var i = 0;
var sum = 0;
var working_data = 0;
var flag = 0xFFFF;
var flags_index = 0;
var match_index = 0;
var type_index = 0;
while(i < uncompressed_len) {
var code_length = -1;
var code = -1;
var byte = uncompressed_data.readUInt8(i);
if(match_index > 0) {
if (byte != uncompressed_data.readUInt8(flag) || match_index > 0x0127) {
code_length = this.matchEncode[match_index][1];
code = this.matchEncode[match_index][0];
match_index = 0;
type_index = 3;
} else {
match_index++;
flag++;
sum = (sum + byte) & 0xFFFF;
working_data = ((working_data * 0x0100) + byte) & 0xFFFFFFFF;
i++;
}
} else {
flag = 0xFFFF;
if (i >= 3) {
flags_index = (working_data >>> 0x0B ^ working_data) & 0x0FFF;
flag = this.flag_table[flags_index];
this.flag_table[flags_index] = i;
} else {
type_index++;
}
if (flag == 0xFFFF) {
code_length = this.nomatchEncode[byte][1];
code = this.nomatchEncode[byte][0] << 0x10;
} else if (byte == uncompressed_data.readUInt8(flag)) {
match_index = 1;
flag++;
type_index = 4;
} else {
code_length = this.nomatchEncode[byte][1] + 1;
code = this.nomatchEncode[byte][0] << 0x0F;
}
sum = (sum + byte) & 0xFFFF;
working_data = ((working_data * 0x0100) + byte) & 0xFFFFFFFF;
i++;
}
if (code_length > 0) {
this.EncodeLiteral(code_length, code);
}
}
// Finish up. This would normally be in an Lzpf_Finish method.
if(type_index == 2) {
this.EncodeLiteral(0x10, 0x00990000);
} else if(type_index >= 3) {
if(type_index == 4) {
code_length = this.matchEncode[match_index][1];
code = this.matchEncode[match_index][0];
this.EncodeLiteral(code_length, code);
}
flags_index = (working_data >>> 0x0B ^ working_data) & 0x0FFF;
if (flags_index == 0xFFFF) {
this.EncodeLiteral(0x10, 0x00990000);
} else {
this.EncodeLiteral(0x11, 0x004c8000);
}
}
// Below is just metadata. The compressed block is complete.
// Encode checksum
this.EncodeLiteral(0x08, (sum << 0x10) & 0xFFFFFFFF);
this.EncodeLiteral(0x08, (sum << 0x18) & 0xFFFFFFFF);
// End
this.compressed_data.push(this.current_literal >>> 0x18);
this.compressed_data.push(0x20);
return new Buffer.from(this.compressed_data);
return Buffer.from(this.compressed_data);
}
}

42
zefie_wtvp_minisrv/app.js
View File

@@ -522,19 +522,35 @@ async function sendToClient(socket, headers_obj, data, compress_data = false) {
clen = data.byteLength;
}
// If wtv-lzpf is in the header then force compression
if (headers_obj["wtv-lzpf"]) {
compress_data = true;
// fix captialization
if (headers_obj["Content-type"]) {
headers_obj["Content-Type"] = headers_obj["Content-type"];
delete headers_obj["Content-type"];
}
// if box can do compression, see if its worth enabling
if (ssid_sessions[socket.ssid].capabilities) {
if (ssid_sessions[socket.ssid].capabilities['client-can-receive-compressed-data']) {
compress_data = shouldWeCompress(headers_obj["Content-Type"]);
}
}
// compress if needed
if (compress_data && clen > 0) {
headers_obj["wtv-lzpf"] = "0";
headers_obj["wtv-lzpf"] = 0;
var lzpf = new WTVLzpf();
data = lzpf.Compress(data);
var wtvcomp = new WTVLzpf();
data = wtvcomp.Compress(data);
console.log("data", data)
wtvcomp = null; // Makes the garbage gods happy so it cleans up our mess
}
if (headers_obj['minisrv-already-compressed']) delete headers_obj['minisrv-already-compressed'];
// encrypt if needed
if (socket_sessions[socket.id].secure == true) {
headers_obj["wtv-encrypted"] = 'true';
@@ -546,17 +562,11 @@ async function sendToClient(socket, headers_obj, data, compress_data = false) {
}
}
// fix captialization
if (headers_obj["Content-length"]) {
delete headers_obj["Content-length"];
}
if (headers_obj["Content-type"]) {
headers_obj["Content-Type"] = headers_obj["Content-type"];
delete headers_obj["Content-type"];
}
// calculate content length
// make sure we are using our Content-length and not one set in a script.
if (headers_obj["Content-Length"]) delete headers_obj["Content-Length"];
if (headers_obj["Content-length"]) delete headers_obj["Content-length"];
// On the WNI server this is the length before compression but we're using the length after compression.
// It matches the HTTP spec anyway so leaving.
if (typeof data.length !== 'undefined') {