Merge branch 'lzpf-compression'

- fix issues caused by my rebasing

zefie_wtvp_minisrv/ServiceVault/wtv-music/get-playlist.js

@@ -31,4 +31,5 @@ wtv-backgroundmusic-add: wtv-music:/MusicCache/headspace/RMF/underground/renegad
 wtv-backgroundmusic-add: wtv-music:/MusicCache/headspace/RMF/moods/affectionate.rmf
 wtv-backgroundmusic-add: wtv-music:/MusicCache/headspace/RMF/moods/sociable.rmf`;
 
-*/
+*/
+data = '';

zefie_wtvp_minisrv/WTVLzpf.js

@@ -1,3 +1,5 @@
+var EventEmitter = require('events').EventEmitter;
+
 /**
 * Pure-JS implementation of WebTV's LZPF compression
 *
@@ -8,12 +10,18 @@
 */
 
 class WTVLzpf {
-    // Note: currentlty doesn't offer streaming support but this is good enough to meet perf demands at the scale we're at.
+    // Note: currentlty doesn't offer optimal streaming support but this is good enough to meet perf demands at the scale we're at.
 
-    current_length = 0
+    current_length = 0;
-    current_literal = 0
+    current_literal = 0;
+    flag = 0xFFFF;
+    working_data = 0;
+    match_index = 0;
+    type_index = 0;
+    checksum = 0;
     flag_table = new Uint16Array(0x1000)
-    compressed_data = []
+    ring_buffer = new Uint8Array(0x2000)
+    compressed_data = [];
 
     nomatchEncode = [
         [0x0000, 0x10], [0x0001, 0x10], [0x0002, 0x10], 
@@ -274,10 +282,27 @@ class WTVLzpf {
     clear() {
         this.current_length = 0;
         this.current_literal = 0;
+        this.flag = 0xFFFF;
+        this.working_data = 0;
+        this.match_index = 0;
+        this.type_index = 0;
+        this.checksum = 0;
+        this.ring_buffer.fill(0x00, 0, 0x2000)
         this.flag_table.fill(0xFFFF, 0, 0x1000);
         this.compressed_data = [];
     }
 
+    /**
+     * Appends a byte to the end of the compressed byte array.  Re-allocates as needed
+     *
+     * @param byte {Number} char code of the byte to be added.
+     *
+     * @returns {undefined}
+     */
+    AddByte(byte) {
+        this.compressed_data.push(byte);
+    }
+
     /**
      * Encodes a literal onto the compressed byte array.
      *
@@ -293,14 +318,139 @@ class WTVLzpf {
         this.current_length += code_length;
 
         while (this.current_length > 7) {
-            //console.log("add", this.current_literal >>> 0, code >>> 0, byte, type)
+            this.AddByte((this.current_literal >>> 0x18) & 0xFF);
-            this.compressed_data.push((this.current_literal >>> 0x18) & 0xFF);
 
             this.current_length -= 8;
             this.current_literal = (this.current_literal << 8) & 0xFFFFFFFF;
         }
     }
 
+    /**
+     * Starts a compression stream
+     *
+     * @returns {undefined} Lzpf compression data
+     */
+    Begin() {
+        this.clear();
+    }
+
+    /**
+     * Compress a block of data.  Used for streamed chunks.
+     *
+     * @param uncompressed_data {String} data to compress
+     *
+     * @returns {Buffer} Lzpf compression data
+     */
+    CompressBlock(uncompressed_data) {
+        this.compressed_data = [];
+
+        if (uncompressed_data.words) {
+            uncompressed_data = new Buffer.from(wtvsec.wordArrayToUint8Array(uncompressed_data));
+        } else if (!uncompressed_data.byteLength) {
+            // otherwise if its not already a Buffer, convert it to one
+            uncompressed_data = new Buffer.from(uncompressed_data);
+        }
+
+        var uncompressed_len = uncompressed_data.length;
+
+        var i = 0;
+        var flags_index = 0;
+        while (i < uncompressed_len) {
+            var code_length = -1;
+            var code = -1;
+
+            var byte = uncompressed_data.readUInt8(i);
+            this.ring_buffer[i & 0x1FFF] = byte;
+
+            if (this.match_index > 0) {
+                if (byte != this.ring_buffer[this.flag] || this.match_index > 0x0127) {
+                    code_length = this.matchEncode[this.match_index][1];
+                    code = this.matchEncode[this.match_index][0];
+                    this.match_index = 0;
+                    this.type_index = 3;
+                } else {
+                    this.match_index = (this.match_index + 1) & 0x1FFF;
+                    this.flag = (this.flag + 1) & 0x1FFF;
+                    this.checksum = (this.checksum + byte) & 0xFFFF;
+                    this.working_data = ((this.working_data * 0x0100) + byte) & 0xFFFFFFFF;
+                    i++;
+                }
+            } else {
+                this.flag = 0xFFFF;
+
+                if (i >= 3) {
+                    flags_index = (this.working_data >>> 0x0B ^ this.working_data) & 0x0FFF;
+                    this.flag = this.flag_table[flags_index];
+                    this.flag_table[flags_index] = i & 0x1FFF;
+                } else {
+                    this.type_index++;
+                }
+
+                if (this.flag == 0xFFFF) {
+                    code_length = this.nomatchEncode[byte][1];
+                    code = this.nomatchEncode[byte][0] << 0x10;
+                } else if (byte == this.ring_buffer[this.flag]) {
+                    this.match_index = 1;
+                    this.flag = (this.flag + 1) & 0x1FFF;
+                    this.type_index = 4;
+                } else {
+                    code_length = this.nomatchEncode[byte][1] + 1;
+                    code = this.nomatchEncode[byte][0] << 0x0F;
+                }
+
+                this.checksum = (this.checksum + byte) & 0xFFFF;
+                this.working_data = ((this.working_data * 0x0100) + byte) & 0xFFFFFFFF;
+                i++;
+            }
+
+            if (code_length > 0) {
+                this.EncodeLiteral(code_length, code);
+            }
+        }
+
+        return Buffer.from(this.compressed_data);
+    }
+
+    /**
+     * Ends a compression stream.
+     *
+     * @param type_index {Number} the end type used to finalize
+     *
+     * @returns {Buffer} Lzpf compression data
+     */
+    Finish() {
+        var code_length = -1
+        var code = -1
+
+        if (this.type_index == 2) {
+            this.EncodeLiteral(0x10, 0x00990000);
+        } else if (this.type_index >= 3) {
+            if (this.type_index == 4) {
+                code_length = this.matchEncode[this.match_index][1];
+                code = this.matchEncode[this.match_index][0];
+                this.EncodeLiteral(code_length, code);
+            }
+
+            var flags_index = (this.working_data >>> 0x0B ^ this.working_data) & 0x0FFF;
+            var flag = this.flag_table[flags_index];
+            if (flag == 0xFFFF) {
+                this.EncodeLiteral(0x10, 0x00990000);
+            } else {
+                this.EncodeLiteral(0x11, 0x004c8000);
+            }
+        }
+
+        // Below is just metadata.  The compressed block is complete.
+
+        // Encode checksum
+        this.EncodeLiteral(0x08, (this.checksum << 0x10) & 0xFFFFFFFF);
+        this.EncodeLiteral(0x08, (this.checksum << 0x18) & 0xFFFFFFFF);
+
+        // End
+        this.AddByte((this.current_literal >>> 0x18) & 0xFF);
+        this.AddByte(0x20);
+    }
+
     /**
      * Compress data using WebTV's Lzpf compression algorithm and adds the footer to the end.
      *
@@ -309,104 +459,11 @@ class WTVLzpf {
      * @returns {Buffer} Lzpf compression data
      */
     Compress(uncompressed_data) {
-        this.clear();
+        this.Begin();
+        this.CompressBlock(uncompressed_data)
+        this.Finish();
 
-        if (uncompressed_data.words) {
+        return Buffer.from(this.compressed_data);
-            uncompressed_data = new Buffer.from(this.wordArrayToUint8Array(uncompressed_data));
-        } else {
-            uncompressed_data = new Buffer.from(uncompressed_data);
-        }
-
-        var uncompressed_len = uncompressed_data.length;
-
-        var i = 0;
-        var sum = 0;
-        var working_data = 0;
-        var flag = 0xFFFF;
-        var flags_index = 0;
-        var match_index = 0;
-        var type_index = 0;
-        while(i < uncompressed_len) {
-            var code_length = -1;
-            var code = -1;
-
-            var byte = uncompressed_data.readUInt8(i);
-
-            if(match_index > 0) {
-                if (byte != uncompressed_data.readUInt8(flag) || match_index > 0x0127) {
-                    code_length = this.matchEncode[match_index][1];
-                    code = this.matchEncode[match_index][0];
-                    match_index = 0;
-                    type_index = 3;
-                } else {
-                    match_index++;
-                    flag++;
-                    sum = (sum + byte) & 0xFFFF;
-                    working_data = ((working_data * 0x0100) + byte) & 0xFFFFFFFF;
-                    i++;
-                }
-            } else {
-                flag = 0xFFFF;
-
-                if (i >= 3) {
-                    flags_index = (working_data >>> 0x0B ^ working_data) & 0x0FFF;
-                    flag = this.flag_table[flags_index];
-                    this.flag_table[flags_index] = i;
-                } else {
-                    type_index++;
-                }
-
-                if (flag == 0xFFFF) {
-                    code_length = this.nomatchEncode[byte][1];
-                    code = this.nomatchEncode[byte][0] << 0x10;
-                } else if (byte == uncompressed_data.readUInt8(flag)) {
-                    match_index = 1;
-                    flag++;
-                    type_index = 4;
-                } else {
-                    code_length = this.nomatchEncode[byte][1] + 1;
-                    code = this.nomatchEncode[byte][0] << 0x0F;
-                }
-
-                sum = (sum + byte) & 0xFFFF;
-                working_data = ((working_data * 0x0100) + byte) & 0xFFFFFFFF;
-                i++;
-            }
-
-            if (code_length > 0) {
-                this.EncodeLiteral(code_length, code);
-            }
-        }
-
-        // Finish up.  This would normally be in an Lzpf_Finish method.
-        if(type_index == 2) {
-            this.EncodeLiteral(0x10, 0x00990000);
-        } else if(type_index >= 3) {
-            if(type_index == 4) {
-                code_length = this.matchEncode[match_index][1];
-                code = this.matchEncode[match_index][0];
-                this.EncodeLiteral(code_length, code);
-            }
-
-            flags_index = (working_data >>> 0x0B ^ working_data) & 0x0FFF;
-            if (flags_index == 0xFFFF) {
-                this.EncodeLiteral(0x10, 0x00990000);
-            } else {
-                this.EncodeLiteral(0x11, 0x004c8000);
-            }
-        }
-
-        // Below is just metadata.  The compressed block is complete.
-
-        // Encode checksum
-        this.EncodeLiteral(0x08, (sum << 0x10) & 0xFFFFFFFF);
-        this.EncodeLiteral(0x08, (sum << 0x18) & 0xFFFFFFFF);
-
-        // End
-        this.compressed_data.push(this.current_literal >>> 0x18);
-        this.compressed_data.push(0x20);
-
-        return new Buffer.from(this.compressed_data);
     }
 }
 

zefie_wtvp_minisrv/app.js

@@ -522,19 +522,35 @@ async function sendToClient(socket, headers_obj, data, compress_data = false) {
         clen = data.byteLength;
     }
 
-    // If wtv-lzpf is in the header then force compression
+    // fix captialization
-    if (headers_obj["wtv-lzpf"]) {
+    if (headers_obj["Content-type"]) {
-        compress_data = true;
+        headers_obj["Content-Type"] = headers_obj["Content-type"];
+        delete headers_obj["Content-type"];
+    }
+
+
+    // if box can do compression, see if its worth enabling
+    if (ssid_sessions[socket.ssid].capabilities) {
+        if (ssid_sessions[socket.ssid].capabilities['client-can-receive-compressed-data']) {
+            compress_data = shouldWeCompress(headers_obj["Content-Type"]);
+        }
     }
 
     // compress if needed
     if (compress_data && clen > 0) {
-        headers_obj["wtv-lzpf"] = "0";
+        headers_obj["wtv-lzpf"] = 0;
 
-        var lzpf = new WTVLzpf();
+        var wtvcomp = new WTVLzpf();
-        data = lzpf.Compress(data);
+        data = wtvcomp.Compress(data);
+
+        console.log("data", data)
+
+        wtvcomp = null; // Makes the garbage gods happy so it cleans up our mess
     }
 
+    if (headers_obj['minisrv-already-compressed']) delete headers_obj['minisrv-already-compressed'];
+
+
     // encrypt if needed
     if (socket_sessions[socket.id].secure == true) {
         headers_obj["wtv-encrypted"] = 'true';
@@ -546,17 +562,11 @@ async function sendToClient(socket, headers_obj, data, compress_data = false) {
         }
     }
 
-    // fix captialization
-    if (headers_obj["Content-length"]) {
-        delete headers_obj["Content-length"];
-    }
-
-    if (headers_obj["Content-type"]) {
-        headers_obj["Content-Type"] = headers_obj["Content-type"];
-        delete headers_obj["Content-type"];
-    }
-
     // calculate content length
+    // make sure we are using our Content-length and not one set in a script.
+    if (headers_obj["Content-Length"]) delete headers_obj["Content-Length"];
+    if (headers_obj["Content-length"]) delete headers_obj["Content-length"];
+
     // On the WNI server this is the length before compression but we're using the length after compression.
     // It matches the HTTP spec anyway so leaving.
     if (typeof data.length !== 'undefined') {