v0.9.5

- major update: app.js: rewrite socket handling to stream data (fix issues POSTing with shoddy dialup, namely fix wtv-update:/sync issues) - update: wtv-home:/home: remove broken irc test, add links to DiskHax and VFatHax, remove URL Accessor form from MiniBrowser because it doesn't work - update: wtv-update:/sync: skip file if the Content-length matches the client and the client file is equal or newer, since client is not storing wtv-checksum for some reason. Client file time is set to when the client received the file last, and not actually when the file was modified on our end
2021-07-23 15:59:02 -04:00
parent d2a6ea0097
commit 6ced1443f0
6 changed files with 253 additions and 107 deletions
--- a/user_config_README.md
+++ b/user_config_README.md
@@ -11,6 +11,14 @@ Some values are available that are not defined in `config.json` by default. I wi
 ```
 You can set the image to be loaded in the top left in place of the WebTV or MSN logo, as well as the main Splash image shown on login.
 If an absolute path (`wtv-url:/`, `file://` url, or `http(s)://` url) is not passed, the server will search for the specified filename in `wtv-star/images` of any Service Vault. You'll want to keep the filesizes low.
+```
+		"post_debug": true
+```
+If you would like to see debug information about realtime bytes received from a client POST request, set `post_debug` to true.
+```
+		"post_percentages": [ 0, 25, 50, 100]
+```
+If you would like to see progress updates on client POST requests, you can define which percentages to show here. Other examples would be `[ 0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 ]` for every 10%, or you could set it to `false`, or `null`, to disable progress updates. Note that percentages are not shown when `post_debug` is enabled.
 ```
 		"ssid_block_list": [
 			"8100000000000000",
--- a/zefie_wtvp_minisrv/ServiceVault/wtv-home/home.js
+++ b/zefie_wtvp_minisrv/ServiceVault/wtv-home/home.js
@@ -23,27 +23,31 @@ function go() {
 }
 </script>
 <h2>Welcome to `+ z_title + `</h2>
-<h3>Encryption Status: `+cryptstatus+`</h3>
-Connection Speed: &rate;
+<b>Encryption Status</b>: ${cryptstatus}<br>
+<b>Connection Speed</b>: &rate;
+<p>
 <form name=access onsubmit="go()">
 <ul>
 <li><a href="client:relog">client:relog (direct)</a></li>
 <li><a href="wtv-tricks:/blastcache?return_to=wtv-home:/home">Clear Cache</a></li>
-<li><a href="wtv-flashrom:/willie">Ultra Willies</a></li>
+<li><a href="wtv-flashrom:/willie" selected>Ultra Willies</a></li>
 <li><a href="wtv-music:/demo/index">MIDI Music Demo</a></li>
-<li><a href="wtv-chat:/MakeChatPage?host=chat.irchat.tv&port=6667&channel=webtv">IRC Chat Test</a>
+<li><a href="client:diskhax">DiskHax</a> - <a href="client:vfathax">VFatHax</a></li>
 <li>Old MSNTV DealerDemo: <a href="wtv-update:/DealerDemo">Download</a> ~ <a href="file://Disk/Demo/index.html">Access (after Download)</a></li>
-<li><a href="http://duckduckgo.com/lite/">DuckDuckGo Lite</a></li>
-<li><input name=url `;
+<li><a href="http://duckduckgo.com/lite/">DuckDuckGo Lite</a></li>`
+if (ssid_sessions[socket.ssid].get('wtv-needs-upgrade') != 'true') {
+	data += `<li><input name=url `;

-if (request_headers.query.url) {
-	data += "value='" + unescape(request_headers.query.url)+"'";
+	if (request_headers.query.url) {
+		data += "value='" + unescape(request_headers.query.url) + "'";
+	}
+
+	data += `width=250  height=10 bgcolor=#444444 text=#ffdd33 cursor=#cc9933>
+<input type=submit value="Access URL">
+</form>`
 }

-data += `width=250  height=10 bgcolor=#444444 text=#ffdd33 cursor=#cc9933 selected>
-<input type=submit value="Access URL">
-</form></li>
-</ul>`
+data += "</li >\n</ul>";

 if (fs.existsSync(service_vaults[0] + "/" + service_name + "/home.zefie.html")) {
 	data += fs.readFileSync(service_vaults[0] + "/" + service_name + "/home.zefie.html", { 'encoding': 'utf8' });
--- a/zefie_wtvp_minisrv/ServiceVault/wtv-update/sync.js
+++ b/zefie_wtvp_minisrv/ServiceVault/wtv-update/sync.js
@@ -83,7 +83,7 @@ function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgrou
            if (!post_data_fileinfo[post_data_filecount]) post_data_fileinfo[post_data_filecount] = new Array();

            if (post_data_line_name == "Last-modified") {
-                post_data_fileinfo[post_data_filecount][post_data_line_name] = (Date.parse(post_data_line_data) / 1000);
+                post_data_fileinfo[post_data_filecount][post_data_line_name] = (new Date(new Date(Date.parse(post_data_line_data)).toUTCString()) / 1000);
            } else if (post_data_line_name == "Content-length") {
                post_data_fileinfo[post_data_filecount][post_data_line_name] = parseInt(post_data_line_data);
            }
@@ -107,14 +107,17 @@ function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgrou
            if (!fs.existsSync(post_match_file)) post_match_file = null;
        });

-        var post_match_file_lstat = fs.lstatSync(post_match_file);
-        var post_match_result = post_data_fileinfo.find(el => el.file === diskmap_group_data.files[k].file) || null;
+        var file_in_postdata = function (post_file) {
+            return post_file.file === diskmap_group_data.files[k].file
+        }

+        var post_match_file_lstat = fs.lstatSync(post_match_file);
+        var post_match_result = post_data_fileinfo.find(file_in_postdata) || null;
        var post_match_file_data = new Buffer.from(fs.readFileSync(post_match_file, {
            encoding: null,
            flags: 'r'
        }));
-        diskmap_group_data.files[k]["Last-modified"] = (post_match_file_lstat.mtime / 1000);
+        diskmap_group_data.files[k]["Last-modified"] = (new Date(new Date(post_match_file_lstat.mtime).toUTCString()) / 1000);
        diskmap_group_data.files[k]["Content-length"] = post_match_file_lstat.size;
        diskmap_group_data.files[k]["wtv-checksum"] = CryptoJS.MD5(CryptoJS.lib.WordArray.create(post_match_file_data)).toString(CryptoJS.enc.Hex).toLowerCase();
        if (!diskmap_group_data.files[k].display) diskmap_group_data.files[k].display = diskmap_group_data.display;
@@ -122,6 +125,9 @@ function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgrou
        if (post_match_result) {
            // md5s match, so client doesn't need file
            if (diskmap_group_data.files[k]['wtv-checksum'].toLowerCase() == post_match_result["wtv-checksum"]) return;
+            // last modified is equal to or newer than the last update, and file size match, so assume same file and client does not need it
+            else if ((post_match_result["Last-modified"] >= diskmap_group_data.files[k]["Last-modified"]) && (post_match_result["Content-length"] == diskmap_group_data.files[k]["Content-length"])) return;
+            // otherwise send to client
            else wtv_download_list.push(diskmap_group_data.files[k]);
        } else {
            wtv_download_list.push(diskmap_group_data.files[k]);
--- a/zefie_wtvp_minisrv/app.js
+++ b/zefie_wtvp_minisrv/app.js
@@ -410,7 +410,10 @@ async function sendToClient(socket, headers_obj, data) {
        // string to header object
        headers_obj = headerStringToObj(headers_obj, true);
    }
-
+    if (!socket_sessions[socket.id]) {
+        socket.destroy();
+        return;
+    }
    var wtv_connection_close = headers_obj["wtv-connection-close"];
    if (typeof(headers_obj["wtv-connection-close"]) != 'undefined') delete headers_obj["wtv-connection-close"];

@@ -548,27 +551,42 @@ function headersAreStandard(string, verbose = false) {
 }

 async function processRequest(socket, data_hex, returnHeadersBeforeSecure = false, encryptedRequest = false) {
-    var url = "";
-    var data = Buffer.from(data_hex,'hex').toString('ascii');

-    var headers = new Array();
+    // TODO: clean up this function (how much is even used anymore?)
+
+    var headers = null;
+    if (socket_sessions[socket.id]) {
+        if (socket_sessions[socket.id].headers) {
+            headers = socket_sessions[socket.id].headers;
+            delete socket_sessions[socket.id].headers;
+        }
+    }
+    var data = Buffer.from(data_hex, 'hex').toString('ascii');
    if (typeof data === "string") {
-        if (data.length > 1) {
+        if ((data.indexOf("\r\n\r\n") != -1 || data.indexOf("\n\n") != -1) && typeof socket_sessions[socket.id].post_data == "undefined") {
            if (data.indexOf("\r\n\r\n") != -1) {
                data = data.split("\r\n\r\n")[0];
            } else {
                data = data.split("\n\n")[0];
            }
            if (headersAreStandard(data)) {
-                headers = headerStringToObj(data);
+                if (headers != null) {
+                    var new_header_obj = headerStringToObj(data);
+                    Object.keys(new_header_obj).forEach(function (k, v) {
+                        headers[k] = new_header_obj[k];
+                    });
+                    new_header_obj = null;
+                } else {
+                    headers = headerStringToObj(data);
+                }
            } else if (!returnHeadersBeforeSecure) {
                // if its a POST request, assume its a binary blob and not encrypted (dangerous)
                if (!encryptedRequest) {
-                    // its not a POST and it 1failed the headersAreStandard test, so we think this is an encrypted blob
+                    // its not a POST and it failed the headersAreStandard test, so we think this is an encrypted blob
                    if (socket_sessions[socket.id].secure != true) {
                        // first time so reroll sessions
                        if (zdebug) console.log(" # [ UNEXPECTED BINARY BLOCK ] First sign of encryption, re-creating RC4 sessions for socket id", socket.id);
-                        socket_sessions[socket.id].wtvsec = new WTVSec(1,zdebug);
+                        socket_sessions[socket.id].wtvsec = new WTVSec(1, zdebug);
                        socket_sessions[socket.id].wtvsec.IssueChallenge();
                        socket_sessions[socket.id].wtvsec.SecureOn();
                        socket_sessions[socket.id].secure = true;
@@ -585,10 +603,13 @@ async function processRequest(socket, data_hex, returnHeadersBeforeSecure = fals
                        }
                        var dec_data = CryptoJS.lib.WordArray.create(socket_sessions[socket.id].wtvsec.Decrypt(0, enc_data));
                        var secure_headers = await processRequest(socket, dec_data.toString(CryptoJS.enc.Hex), true, true);
-                        headers.encrypted = true;
-                        Object.keys(secure_headers).forEach(function (k, v) {
-                            headers[k] = secure_headers[k];
-                        });
+                        if (secure_headers) {
+                            var headers = new Array();
+                            headers.encrypted = true;
+                            Object.keys(secure_headers).forEach(function (k, v) {
+                                headers[k] = secure_headers[k];
+                            });
+                        }
                    }
                }
            }
@@ -599,7 +620,7 @@ async function processRequest(socket, data_hex, returnHeadersBeforeSecure = fals
                    ssid_sessions[socket.ssid] = new ClientSessionData();
                }
                if (!ssid_sessions[socket.ssid].data_store.sockets) ssid_sessions[socket.ssid].data_store.sockets = new Array();
-                ssid_sessions[socket.ssid].data_store.sockets.push(socket.id);                
+                ssid_sessions[socket.ssid].data_store.sockets.push(socket.id);
            }

            var ip2long = function (ip) {
@@ -627,9 +648,9 @@ async function processRequest(socket, data_hex, returnHeadersBeforeSecure = fals
            };

            var rejectSSIDConnection = function (ssid, blacklist) {
-                if (blacklist) console.log(" * Request from SSID", filterSSID(ssid), "(" + socket.remoteAddr + "), but that SSID is in the blacklist, rejecting.");                    
+                if (blacklist) console.log(" * Request from SSID", filterSSID(ssid), "(" + socket.remoteAddr + "), but that SSID is in the blacklist, rejecting.");
                else console.log(" * Request from SSID", filterSSID(socket.ssid), "(" + socket.remoteAddress + "), but that SSID is not in the whitelist, rejecting.");
-                
+
                var errpage = doErrorPage(401, "Access to this service is denied.");
                headers = errpage[0];
                data = errpage[1];
@@ -714,7 +735,6 @@ async function processRequest(socket, data_hex, returnHeadersBeforeSecure = fals
            }

            if (returnHeadersBeforeSecure) {
-                headers = await checkForPostData(socket, headers, data, data_hex);
                return headers;
            }

@@ -736,13 +756,13 @@ async function processRequest(socket, data_hex, returnHeadersBeforeSecure = fals
                    socket_sessions[socket.id].secure = true;
                }
                if (!headers.request_url) {
-
+                    var header_length = 0;
                    if (data_hex.indexOf("0d0a0d0a")) {
                        // \r\n\r\n
-                        var header_length = data.length + 4;
+                        header_length = data.length + 4;
                    } else if (data_hex.indexOf("0a0a")) {
                        // \n\n
-                        var header_length = data.length + 2;
+                        header_length = data.length + 2;
                    }
                    var enc_data = CryptoJS.enc.Hex.parse(data_hex.substring(header_length * 2));
                    if (enc_data.sigBytes > 0) {
@@ -753,12 +773,13 @@ async function processRequest(socket, data_hex, returnHeadersBeforeSecure = fals
                            headers.psuedo_encryption = true;
                            ssid_sessions[socket.ssid].set("box-does-psuedo-encryption", true);
                            socket_sessions[socket.id].secure = false;
-                            var secure_headers = await processRequest(socket, enc_data.toString(CryptoJS.enc.Hex), true);
+                            var secure_headers = await processRequest(socket, enc_data.toString(CryptoJS.enc.Hex), true, true);
                        } else {
                            // SECURE ON and detected encrypted data
                            ssid_sessions[socket.ssid].set("box-does-psuedo-encryption", false);
                            var dec_data = CryptoJS.lib.WordArray.create(socket_sessions[socket.id].wtvsec.Decrypt(0, enc_data))
-                            var secure_headers = await processRequest(socket, dec_data.toString(CryptoJS.enc.Hex), true);
+                            var secure_headers = await processRequest(socket, dec_data.toString(CryptoJS.enc.Hex), true, true);
+                            if (!secure_headers) return;
                            if (zdebug) console.log(" # Encrypted Request (SECURE ON)", "on", socket.id);
                            if (zshowheaders) console.log(secure_headers);
                            if (!secure_headers.request) {
@@ -776,57 +797,162 @@ async function processRequest(socket, data_hex, returnHeadersBeforeSecure = fals
                        });
                    }
                }
-            } else {
-                headers = await checkForPostData(socket, headers, data, data_hex);
            }
-            if (!headers.request_url) {
-                // still no url, likely lost encryption stream, tell client to relog
-/*
-                socket_sessions[socket.id].secure = false;                
-                headers = `300 OK
-Connection: Keep-Alive
-Expires: Wed, 09 Oct 1991 22:00:00 GMT
-wtv-expire-all: wtv-head-waiter:
-wtv-expire-all: wtv-1800:
-Location: client:relog
-wtv-visit: client:relog
-Content-type: text/html`;
-                data = '';
-                */
-                socket_sessions[socket.id].secure = false
-                socket_sessions[socket.id].close_me = true;
-                delete socket_sessions[socket.id].wtvsec;
-                sendToClient(socket, headers, data);
+
+            // handle POST
+            if (headers['request']) {
+                if (headers['request'].substring(0, 4) == "POST") {
+                    if (typeof socket_sessions[socket.id].post_data == "undefined") {
+                        if (socket_sessions[socket.id].post_data_percents_shown) delete socket_sessions[socket.id].post_data_percents_shown;
+                        socket_sessions[socket.id].post_data_length = headers['Content-length'] || headers['Content-Length'] || 0;
+                        socket_sessions[socket.id].post_data_length = parseInt(socket_sessions[socket.id].post_data_length);
+                        socket_sessions[socket.id].post_data = "";
+                        socket_sessions[socket.id].headers = headers;
+                        var post_string = "POST";
+                        if (socket_sessions[socket.id].secure == true) {
+                            post_string = "Encrypted " + post_string;
+                        } else {
+                            // if the request is not encrypted, the client may have just sent the data with the primary headers, so lets look for that.
+                            if (data_hex.indexOf("0d0a0d0a") != -1) socket_sessions[socket.id].post_data = data_hex.substring(data_hex.indexOf("0d0a0d0a") + 8);
+                            if (data_hex.indexOf("0a0a") != -1) socket_sessions[socket.id].post_data = data_hex.substring(data_hex.indexOf("0a0a") + 4);
+                        }
+                        if (socket_sessions[socket.id].post_data.length == (socket_sessions[socket.id].post_data_length * 2)) {
+                            // got all expected data
+                            console.log(" * Incoming", post_string, "request on", socket.id, "from", filterSSID(socket.ssid), "to", headers['request_url'], "(got all expected", socket_sessions[socket.id].post_data_length, "bytes of data from client already)");
+                            headers.post_data = CryptoJS.enc.Hex.parse(socket_sessions[socket.id].post_data);
+                            processURL(socket, headers);
+                        } else {
+                            // expecting more data (see below)
+                            console.log(" * Incoming", post_string, "request on", socket.id, "from", filterSSID(socket.ssid), "to", headers['request_url'], "(expecting", socket_sessions[socket.id].post_data_length, "bytes of data from client...)");
+                        }
+                        if (socket_sessions[socket.id].post_data.length > (socket_sessions[socket.id].post_data_length * 2)) {
+                            // got too much data ? ... should not ever reach this code
+                            var errpage = doErrorPage(400, "Received too much data in POST request<br>Got " + (socket_sessions[socket.id].post_data.length / 2) + ", expected " + socket_sessions[socket.id].post_data_length);
+                            headers = errpage[0];
+                            data = errpage[1];
+                            sendToClient(socket, headers, data);
+                            return;
+                        }
+                        return;
+                    }
+                } else {
+                    delete socket_sessions[socket.id].headers;
+                    delete socket_sessions[socket.id].post_data;
+                    delete socket_sessions[socket.id].post_data_length;
+                    processURL(socket, headers);
+                    return;
+                }
            } else {
-                processURL(socket, headers);
+                socket_sessions[socket.id].headers = headers;
            }
        } else {
-            // socket error, terminate it.
-            socket.destroy();
-        }
-    }
-}
+            // handle streaming POST
+            if (typeof socket_sessions[socket.id].post_data != "undefined" && headers) {
+                socket_sessions[socket.id].headers = headers;
+                if (socket_sessions[socket.id].post_data.length < (socket_sessions[socket.id].post_data_length * 2)) {
+                    new_header_obj = null;
+                    var enc_data = CryptoJS.enc.Hex.parse(data_hex);
+                    if (socket_sessions[socket.id].secure) {
+                        // decrypt if encrypted
+                        var dec_data = CryptoJS.lib.WordArray.create(socket_sessions[socket.id].wtvsec.Decrypt(0, enc_data))
+                    } else {
+                        // just pass it over
+                        var dec_data = enc_data;
+                    }

-async function checkForPostData(socket, headers, data, data_hex) {
-    if (headers.request) {
-        if (headers.request.substring(0, 4) == "POST") {
-            if (data_hex.indexOf("0d0a0d0a") != -1) {
-                // \r\n\r\n
-                var header_length = data.length + 4;
-            } else if (data_hex.indexOf("0a0a") != -1) {
-                // \n\n
-                var header_length = data.length + 2;
-            }
-            var post_data = CryptoJS.enc.Hex.parse(data_hex.substring(header_length * 2));
-            if (socket_sessions[socket.id].secure == true) {
-                if (zdebug) console.log(" # Encrypted POST Content (SECURE ON)", "on", socket.id, "[", post_data.sigBytes, "bytes ]");
-            } else {
-                if (zdebug) console.log(" # Unencrypted POST Content", "on", socket.id);
-            }
-            headers.post_data = post_data;
+                    socket_sessions[socket.id].post_data += dec_data.toString(CryptoJS.enc.Hex);
+
+                    var post_string = "POST";
+                    if (socket_sessions[socket.id].secure == true) post_string = "Encrypted " + post_string;
+
+                    if (minisrv_config.config.post_debug) {
+                        // `post_debug` logging of every chunk
+                        console.log(" * ", Math.floor(new Date().getTime() / 1000), "Receiving", post_string, "data on", socket.id, "[", socket_sessions[socket.id].post_data.length / 2, "of", socket_sessions[socket.id].post_data_length, "bytes ]");
+                    } else {
+                        // calculate and display percentage of data received
+                        var getPercentage = function(partialValue, totalValue) {
+                            return Math.floor((100 * partialValue) / totalValue);
+                        }
+                        var postPercent = getPercentage(socket_sessions[socket.id].post_data.length, (socket_sessions[socket.id].post_data_length * 2));
+                        if (minisrv_config.config.post_percentages) {
+                            if (minisrv_config.config.post_percentages.includes(postPercent)) {
+                                if (!socket_sessions[socket.id].post_data_percents_shown) socket_sessions[socket.id].post_data_percents_shown = new Array();
+                                if (!socket_sessions[socket.id].post_data_percents_shown[postPercent]) {
+                                    console.log(" * Received", postPercent, "% of", socket_sessions[socket.id].post_data_length, "bytes on", socket.id, "from", filterSSID(socket.ssid));
+                                    socket_sessions[socket.id].post_data_percents_shown[postPercent] = true;
+                                }
+                                if (postPercent == 100) delete socket_sessions[socket.id].post_data_percents_shown;
+                            }
+                        }
+                    }
+                }
+                if (socket_sessions[socket.id].post_data.length == (socket_sessions[socket.id].post_data_length * 2)) {
+                    // got all expected data
+                    headers.post_data = CryptoJS.enc.Hex.parse(socket_sessions[socket.id].post_data);
+                    if (socket_sessions[socket.id].secure == true) {
+                        if (zdebug) console.log(" # Encrypted POST Content (SECURE ON)", "on", socket.id, "[", headers.post_data.sigBytes, "bytes ]");
+                    } else {
+                        if (zdebug) console.log(" # Unencrypted POST Content", "on", socket.id);
+                    }
+                    delete socket_sessions[socket.id].headers;
+                    delete socket_sessions[socket.id].post_data;
+                    delete socket_sessions[socket.id].post_data_length;
+                    processURL(socket, headers);
+                    return;
+                }
+                if (socket_sessions[socket.id].post_data.length > (socket_sessions[socket.id].post_data_length * 2)) {
+                    // got too much data ? ... should not ever reach this code
+                    var errpage = doErrorPage(400, "Received too much data in POST request<br>Got " + (socket_sessions[socket.id].post_data.length / 2) + ", expected " + socket_sessions[socket.id].post_data_length);
+                    headers = errpage[0];
+                    data = errpage[1];
+                    sendToClient(socket, headers, data);
+                    return;
+                }
+
+            } else if (!returnHeadersBeforeSecure) {
+                if (!encryptedRequest) {
+                    if (socket_sessions[socket.id].secure != true) {
+                        socket_sessions[socket.id].wtvsec = new WTVSec(1, zdebug);
+                        socket_sessions[socket.id].wtvsec.IssueChallenge();
+                        socket_sessions[socket.id].wtvsec.SecureOn();
+                        socket_sessions[socket.id].secure = true;
+                    }
+                    var enc_data = CryptoJS.enc.Hex.parse(data_hex);
+                    if (enc_data.sigBytes > 0) {
+                        if (!socket_sessions[socket.id].wtvsec) {
+                            var errpage = doErrorPage(400);
+                            var headers = errpage[0];
+                            headers += "wtv-visit: client:relog\n";
+                            data = errpage[1];
+                            sendToClient(socket, headers, data);
+                            return;
+                        }
+                        var str_test = enc_data.toString(CryptoJS.enc.Latin1);
+                        if (headersAreStandard(str_test)) {
+                            var dec_data = enc_data;
+                        } else {
+                            var dec_data = CryptoJS.lib.WordArray.create(socket_sessions[socket.id].wtvsec.Decrypt(0, enc_data));
+                        }
+                        var secure_headers = await processRequest(socket, dec_data.toString(CryptoJS.enc.Hex), false, true);
+                        if (secure_headers) {
+                            if (!headers) headers = new Array();
+                            headers.encrypted = true;
+                            Object.keys(secure_headers).forEach(function (k, v) {
+                                headers[k] = secure_headers[k];
+                            });
+                            if (headers['request']) {
+                                if (headers['request'].substring(0, 4) == "POST" && !socket_sessions[socket.id].post_data) {
+                                    socket_sessions[socket.id].post_data_length = headers['Content-length'] || headers['Content-Length'] || 0;
+                                    socket_sessions[socket.id].post_data = "";
+                                }
+                                processRequest(socket, dec_data.toString(CryptoJS.enc.Hex));
+                            }
+                        } 
+                    }
+                } 
+            } 
        }
-    }
-    return headers;
+    }    
 }

 async function cleanupSocket(socket) {
@@ -861,34 +987,28 @@ async function cleanupSocket(socket) {
 async function handleSocket(socket) {
    // create unique socket id with client address and port
    socket.id = parseInt(crc16('CCITT-FALSE', Buffer.from(String(socket.remoteAddress) + String(socket.remotePort), "utf8")).toString(16), 16);
-    socket_sessions[socket.id] = [];    
+    socket_sessions[socket.id] = [];
    socket.setEncoding('hex'); //set data encoding (Text: 'ascii', 'utf8' ~ Binary: 'hex', 'base64' (do not trust 'binary' encoding))

-    // NOTE: As it stands we use a 'timeout' to start processing data when we have not recieved any data
-    // from the client in X time (defined in config, in milliseconds). The problem with this is in the case of
-    // a modem retrain during a request. 
-
-    // TODO: Properly know when client is done sending data, by parsing headers.
-    // Caveat of this is that sometimes the Content-length header does not exist, or will be encrypted.
-    
    socket.on('data', function (data_hex) {
-        socket.setTimeout(minisrv_config.config.socket_timeout); // the timeout mentioned above
-
-        // Store all received data into a buffer. Kind of misleading as its not a true JS Buffer
-        // but instead a CryptoJS WordArray
-        if (socket_sessions[socket.id].buffer) {
-            socket_sessions[socket.id].buffer.concat(CryptoJS.enc.Hex.parse(data_hex));
+        if (!socket_sessions[socket.id].secure) {
+            // buffer unencrypted data until we see the classic double-newline, or get blank
+            if (!socket_sessions[socket.id].header_buffer) socket_sessions[socket.id].header_buffer = "";
+            socket_sessions[socket.id].header_buffer += data_hex;
+            var header_buffer_text = CryptoJS.enc.Hex.parse(socket_sessions[socket.id].header_buffer).toString(CryptoJS.enc.Latin1);
+            if (header_buffer_text.indexOf("\r\n\r\n") != -1 || header_buffer_text.indexOf("\n\n") != -1 || header_buffer_text == "") {
+                data_hex = socket_sessions[socket.id].header_buffer;
+                delete socket_sessions[socket.id].header_buffer;
+                processRequest(this, data_hex);
+            }
        } else {
-            socket_sessions[socket.id].buffer = CryptoJS.enc.Hex.parse(data_hex);
+            // stream encrypted requests through the processor
+            processRequest(this, data_hex);
        }
    });

-    socket.on('timeout', async function () {
-        // start the async chain
-        if (socket_sessions[socket.id].buffer) {
-            // process the request if the buffer exists
-            processRequest(this, socket_sessions[socket.id].buffer.toString(CryptoJS.enc.Hex));
-        }
+    socket.on('timeout', function () {
+        cleanupSocket(socket);
    });

    socket.on('error', (err) => {
@@ -947,12 +1067,20 @@ function getGitRevision() {
 }

 // SERVER START
-
-var z_title = "zefie's wtv minisrv v" + require('./package.json').version;
+var git_commit = getGitRevision()
+if (git_commit) {
+    var z_title = "zefie's wtv minisrv v" + require('./package.json').version + " (git " + git_commit.substring(0,8) + ")";
+} else {
+    var z_title = "zefie's wtv minisrv v" + require('./package.json').version;
+}
 console.log("**** Welcome to " + z_title + " ****");
 console.log(" *** Reading global configuration...");
 try {
    var minisrv_config = JSON.parse(fs.readFileSync(__dirname + "/config.json"));
+    if (git_commit) {
+        minisrv_config.config.git_commit = git_commit;
+        delete this.git_commit;
+    }
 } catch (e) {
    throw ("ERROR: Could not read config.json", e);
 }
--- a/zefie_wtvp_minisrv/config.json
+++ b/zefie_wtvp_minisrv/config.json
@@ -9,7 +9,7 @@
    "service_logo": "MSNLogo.gif",
    "service_splash_logo": "splash_logo_msn.gif",
    "hide_ssid_in_logs": true,
-    "socket_timeout": 350,
+    "post_percentages": [ 0, 25, 50, 100],
    "verbosity": 2
  },
  "services": {
--- a/zefie_wtvp_minisrv/package.json
+++ b/zefie_wtvp_minisrv/package.json
@@ -1,6 +1,6 @@
 {
  "name": "zefie_wtvp_minisrv",
-  "version": "0.9.4",
+  "version": "0.9.5",
  "description": "WebTV Service (WTVP) Emulation Server",
  "main": "app.js",
  "homepage": "https://github.com/zefie/zefie_wtvp_minisrv",