diff --git a/user_config_README.md b/user_config_README.md index 79d36bf4..3748eb9e 100644 --- a/user_config_README.md +++ b/user_config_README.md @@ -11,6 +11,14 @@ Some values are available that are not defined in `config.json` by default. I wi ``` You can set the image to be loaded in the top left in place of the WebTV or MSN logo, as well as the main Splash image shown on login. If an absolute path (`wtv-url:/`, `file://` url, or `http(s)://` url) is not passed, the server will search for the specified filename in `wtv-star/images` of any Service Vault. You'll want to keep the filesizes low. +``` + "post_debug": true +``` +If you would like to see debug information about realtime bytes received from a client POST request, set `post_debug` to true. +``` + "post_percentages": [ 0, 25, 50, 100] +``` +If you would like to see progress updates on client POST requests, you can define which percentages to show here. Other examples would be `[ 0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 ]` for every 10%, or you could set it to `false`, or `null`, to disable progress updates. Note that percentages are not shown when `post_debug` is enabled. ``` "ssid_block_list": [ "8100000000000000", diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-home/home.js b/zefie_wtvp_minisrv/ServiceVault/wtv-home/home.js index 5d3dcc16..3f8be789 100644 --- a/zefie_wtvp_minisrv/ServiceVault/wtv-home/home.js +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-home/home.js @@ -23,27 +23,31 @@ function go() { }
` } -data += `width=250 height=10 bgcolor=#444444 text=#ffdd33 cursor=#cc9933 selected> - - -` +data += "\n"; if (fs.existsSync(service_vaults[0] + "/" + service_name + "/home.zefie.html")) { data += fs.readFileSync(service_vaults[0] + "/" + service_name + "/home.zefie.html", { 'encoding': 'utf8' }); diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-update/sync.js b/zefie_wtvp_minisrv/ServiceVault/wtv-update/sync.js index 9c096a13..032a8dc5 100644 --- a/zefie_wtvp_minisrv/ServiceVault/wtv-update/sync.js +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-update/sync.js @@ -83,7 +83,7 @@ function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgrou if (!post_data_fileinfo[post_data_filecount]) post_data_fileinfo[post_data_filecount] = new Array(); if (post_data_line_name == "Last-modified") { - post_data_fileinfo[post_data_filecount][post_data_line_name] = (Date.parse(post_data_line_data) / 1000); + post_data_fileinfo[post_data_filecount][post_data_line_name] = (new Date(new Date(Date.parse(post_data_line_data)).toUTCString()) / 1000); } else if (post_data_line_name == "Content-length") { post_data_fileinfo[post_data_filecount][post_data_line_name] = parseInt(post_data_line_data); } @@ -107,14 +107,17 @@ function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgrou if (!fs.existsSync(post_match_file)) post_match_file = null; }); - var post_match_file_lstat = fs.lstatSync(post_match_file); - var post_match_result = post_data_fileinfo.find(el => el.file === diskmap_group_data.files[k].file) || null; + var file_in_postdata = function (post_file) { + return post_file.file === diskmap_group_data.files[k].file + } + var post_match_file_lstat = fs.lstatSync(post_match_file); + var post_match_result = post_data_fileinfo.find(file_in_postdata) || null; var post_match_file_data = new Buffer.from(fs.readFileSync(post_match_file, { encoding: null, flags: 'r' })); - diskmap_group_data.files[k]["Last-modified"] = (post_match_file_lstat.mtime / 1000); + diskmap_group_data.files[k]["Last-modified"] = (new Date(new Date(post_match_file_lstat.mtime).toUTCString()) / 1000); diskmap_group_data.files[k]["Content-length"] = post_match_file_lstat.size; diskmap_group_data.files[k]["wtv-checksum"] = CryptoJS.MD5(CryptoJS.lib.WordArray.create(post_match_file_data)).toString(CryptoJS.enc.Hex).toLowerCase(); if (!diskmap_group_data.files[k].display) diskmap_group_data.files[k].display = diskmap_group_data.display; @@ -122,6 +125,9 @@ function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgrou if (post_match_result) { // md5s match, so client doesn't need file if (diskmap_group_data.files[k]['wtv-checksum'].toLowerCase() == post_match_result["wtv-checksum"]) return; + // last modified is equal to or newer than the last update, and file size match, so assume same file and client does not need it + else if ((post_match_result["Last-modified"] >= diskmap_group_data.files[k]["Last-modified"]) && (post_match_result["Content-length"] == diskmap_group_data.files[k]["Content-length"])) return; + // otherwise send to client else wtv_download_list.push(diskmap_group_data.files[k]); } else { wtv_download_list.push(diskmap_group_data.files[k]); diff --git a/zefie_wtvp_minisrv/app.js b/zefie_wtvp_minisrv/app.js index c2558e76..366abe2d 100644 --- a/zefie_wtvp_minisrv/app.js +++ b/zefie_wtvp_minisrv/app.js @@ -410,7 +410,10 @@ async function sendToClient(socket, headers_obj, data) { // string to header object headers_obj = headerStringToObj(headers_obj, true); } - + if (!socket_sessions[socket.id]) { + socket.destroy(); + return; + } var wtv_connection_close = headers_obj["wtv-connection-close"]; if (typeof(headers_obj["wtv-connection-close"]) != 'undefined') delete headers_obj["wtv-connection-close"]; @@ -548,27 +551,42 @@ function headersAreStandard(string, verbose = false) { } async function processRequest(socket, data_hex, returnHeadersBeforeSecure = false, encryptedRequest = false) { - var url = ""; - var data = Buffer.from(data_hex,'hex').toString('ascii'); - var headers = new Array(); + // TODO: clean up this function (how much is even used anymore?) + + var headers = null; + if (socket_sessions[socket.id]) { + if (socket_sessions[socket.id].headers) { + headers = socket_sessions[socket.id].headers; + delete socket_sessions[socket.id].headers; + } + } + var data = Buffer.from(data_hex, 'hex').toString('ascii'); if (typeof data === "string") { - if (data.length > 1) { + if ((data.indexOf("\r\n\r\n") != -1 || data.indexOf("\n\n") != -1) && typeof socket_sessions[socket.id].post_data == "undefined") { if (data.indexOf("\r\n\r\n") != -1) { data = data.split("\r\n\r\n")[0]; } else { data = data.split("\n\n")[0]; } if (headersAreStandard(data)) { - headers = headerStringToObj(data); + if (headers != null) { + var new_header_obj = headerStringToObj(data); + Object.keys(new_header_obj).forEach(function (k, v) { + headers[k] = new_header_obj[k]; + }); + new_header_obj = null; + } else { + headers = headerStringToObj(data); + } } else if (!returnHeadersBeforeSecure) { // if its a POST request, assume its a binary blob and not encrypted (dangerous) if (!encryptedRequest) { - // its not a POST and it 1failed the headersAreStandard test, so we think this is an encrypted blob + // its not a POST and it failed the headersAreStandard test, so we think this is an encrypted blob if (socket_sessions[socket.id].secure != true) { // first time so reroll sessions if (zdebug) console.log(" # [ UNEXPECTED BINARY BLOCK ] First sign of encryption, re-creating RC4 sessions for socket id", socket.id); - socket_sessions[socket.id].wtvsec = new WTVSec(1,zdebug); + socket_sessions[socket.id].wtvsec = new WTVSec(1, zdebug); socket_sessions[socket.id].wtvsec.IssueChallenge(); socket_sessions[socket.id].wtvsec.SecureOn(); socket_sessions[socket.id].secure = true; @@ -585,10 +603,13 @@ async function processRequest(socket, data_hex, returnHeadersBeforeSecure = fals } var dec_data = CryptoJS.lib.WordArray.create(socket_sessions[socket.id].wtvsec.Decrypt(0, enc_data)); var secure_headers = await processRequest(socket, dec_data.toString(CryptoJS.enc.Hex), true, true); - headers.encrypted = true; - Object.keys(secure_headers).forEach(function (k, v) { - headers[k] = secure_headers[k]; - }); + if (secure_headers) { + var headers = new Array(); + headers.encrypted = true; + Object.keys(secure_headers).forEach(function (k, v) { + headers[k] = secure_headers[k]; + }); + } } } } @@ -599,7 +620,7 @@ async function processRequest(socket, data_hex, returnHeadersBeforeSecure = fals ssid_sessions[socket.ssid] = new ClientSessionData(); } if (!ssid_sessions[socket.ssid].data_store.sockets) ssid_sessions[socket.ssid].data_store.sockets = new Array(); - ssid_sessions[socket.ssid].data_store.sockets.push(socket.id); + ssid_sessions[socket.ssid].data_store.sockets.push(socket.id); } var ip2long = function (ip) { @@ -627,9 +648,9 @@ async function processRequest(socket, data_hex, returnHeadersBeforeSecure = fals }; var rejectSSIDConnection = function (ssid, blacklist) { - if (blacklist) console.log(" * Request from SSID", filterSSID(ssid), "(" + socket.remoteAddr + "), but that SSID is in the blacklist, rejecting."); + if (blacklist) console.log(" * Request from SSID", filterSSID(ssid), "(" + socket.remoteAddr + "), but that SSID is in the blacklist, rejecting."); else console.log(" * Request from SSID", filterSSID(socket.ssid), "(" + socket.remoteAddress + "), but that SSID is not in the whitelist, rejecting."); - + var errpage = doErrorPage(401, "Access to this service is denied."); headers = errpage[0]; data = errpage[1]; @@ -714,7 +735,6 @@ async function processRequest(socket, data_hex, returnHeadersBeforeSecure = fals } if (returnHeadersBeforeSecure) { - headers = await checkForPostData(socket, headers, data, data_hex); return headers; } @@ -736,13 +756,13 @@ async function processRequest(socket, data_hex, returnHeadersBeforeSecure = fals socket_sessions[socket.id].secure = true; } if (!headers.request_url) { - + var header_length = 0; if (data_hex.indexOf("0d0a0d0a")) { // \r\n\r\n - var header_length = data.length + 4; + header_length = data.length + 4; } else if (data_hex.indexOf("0a0a")) { // \n\n - var header_length = data.length + 2; + header_length = data.length + 2; } var enc_data = CryptoJS.enc.Hex.parse(data_hex.substring(header_length * 2)); if (enc_data.sigBytes > 0) { @@ -753,12 +773,13 @@ async function processRequest(socket, data_hex, returnHeadersBeforeSecure = fals headers.psuedo_encryption = true; ssid_sessions[socket.ssid].set("box-does-psuedo-encryption", true); socket_sessions[socket.id].secure = false; - var secure_headers = await processRequest(socket, enc_data.toString(CryptoJS.enc.Hex), true); + var secure_headers = await processRequest(socket, enc_data.toString(CryptoJS.enc.Hex), true, true); } else { // SECURE ON and detected encrypted data ssid_sessions[socket.ssid].set("box-does-psuedo-encryption", false); var dec_data = CryptoJS.lib.WordArray.create(socket_sessions[socket.id].wtvsec.Decrypt(0, enc_data)) - var secure_headers = await processRequest(socket, dec_data.toString(CryptoJS.enc.Hex), true); + var secure_headers = await processRequest(socket, dec_data.toString(CryptoJS.enc.Hex), true, true); + if (!secure_headers) return; if (zdebug) console.log(" # Encrypted Request (SECURE ON)", "on", socket.id); if (zshowheaders) console.log(secure_headers); if (!secure_headers.request) { @@ -776,57 +797,162 @@ async function processRequest(socket, data_hex, returnHeadersBeforeSecure = fals }); } } - } else { - headers = await checkForPostData(socket, headers, data, data_hex); } - if (!headers.request_url) { - // still no url, likely lost encryption stream, tell client to relog -/* - socket_sessions[socket.id].secure = false; - headers = `300 OK -Connection: Keep-Alive -Expires: Wed, 09 Oct 1991 22:00:00 GMT -wtv-expire-all: wtv-head-waiter: -wtv-expire-all: wtv-1800: -Location: client:relog -wtv-visit: client:relog -Content-type: text/html`; - data = ''; - */ - socket_sessions[socket.id].secure = false - socket_sessions[socket.id].close_me = true; - delete socket_sessions[socket.id].wtvsec; - sendToClient(socket, headers, data); + + // handle POST + if (headers['request']) { + if (headers['request'].substring(0, 4) == "POST") { + if (typeof socket_sessions[socket.id].post_data == "undefined") { + if (socket_sessions[socket.id].post_data_percents_shown) delete socket_sessions[socket.id].post_data_percents_shown; + socket_sessions[socket.id].post_data_length = headers['Content-length'] || headers['Content-Length'] || 0; + socket_sessions[socket.id].post_data_length = parseInt(socket_sessions[socket.id].post_data_length); + socket_sessions[socket.id].post_data = ""; + socket_sessions[socket.id].headers = headers; + var post_string = "POST"; + if (socket_sessions[socket.id].secure == true) { + post_string = "Encrypted " + post_string; + } else { + // if the request is not encrypted, the client may have just sent the data with the primary headers, so lets look for that. + if (data_hex.indexOf("0d0a0d0a") != -1) socket_sessions[socket.id].post_data = data_hex.substring(data_hex.indexOf("0d0a0d0a") + 8); + if (data_hex.indexOf("0a0a") != -1) socket_sessions[socket.id].post_data = data_hex.substring(data_hex.indexOf("0a0a") + 4); + } + if (socket_sessions[socket.id].post_data.length == (socket_sessions[socket.id].post_data_length * 2)) { + // got all expected data + console.log(" * Incoming", post_string, "request on", socket.id, "from", filterSSID(socket.ssid), "to", headers['request_url'], "(got all expected", socket_sessions[socket.id].post_data_length, "bytes of data from client already)"); + headers.post_data = CryptoJS.enc.Hex.parse(socket_sessions[socket.id].post_data); + processURL(socket, headers); + } else { + // expecting more data (see below) + console.log(" * Incoming", post_string, "request on", socket.id, "from", filterSSID(socket.ssid), "to", headers['request_url'], "(expecting", socket_sessions[socket.id].post_data_length, "bytes of data from client...)"); + } + if (socket_sessions[socket.id].post_data.length > (socket_sessions[socket.id].post_data_length * 2)) { + // got too much data ? ... should not ever reach this code + var errpage = doErrorPage(400, "Received too much data in POST request