zefie/minisrv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

pc_services now uses node Express

Browse Source 
- reduces chance of error with older webtv clients sending http request
- version bump to 0.9.32
This commit is contained in:
zefie
2022-10-10 15:40:09 -04:00
parent 300a8427cd
commit d4ceda7211
4 changed files with 1217 additions and 285 deletions
Download Patch File Download Diff File Expand all files Collapse all files

221
zefie_wtvp_minisrv/app.js
View File

@@ -19,6 +19,8 @@ const WTVMime = require("./WTVMime.js");
const { WTVShared, clientShowAlert } = require("./WTVShared.js");
const WTVFlashrom = require("./WTVFlashrom.js");
const vm = require('vm');
const express = require('express');
const { url } = require('inspector');
process
.on('SIGTERM', shutdown('SIGTERM'))
@@ -39,6 +41,7 @@ var ssid_sessions = new Array();
var socket_sessions = new Array();
var ports = [];
var pc_ports = [];
// add .reverse() feature to all JavaScript Strings in this application
// works for service vault scripts too.
@@ -185,7 +188,11 @@ var runScriptInVM = function (script_data, user_contextObj = {}, privileged = fa
return contextObj; // updated context object with whatever global varibles the script set
}
async function processPath(socket, service_vault_file_path, request_headers = new Array(), service_name, shared_romcache = null) {
function sendToPCClient(headers, data) {
}
async function processPath(socket, service_vault_file_path, request_headers = new Array(), service_name, shared_romcache = null, pc_services = false) {
var headers, data = null;
var request_is_async = false;
var service_vault_found = false;
@@ -208,11 +215,14 @@ async function processPath(socket, service_vault_file_path, request_headers = ne
// format: [ ourvarname, scriptsvarname ]
["headers", "headers"], // we need to be able to read the script's response headers
["data", "data"], // we need to be able to read the script's response data
["request_is_async", "request_is_async"], // we need to know if the script is async or not
["ssid_sessions", "ssid_sessions"], // global ssid_sessions object for privileged service scripts, such as wtv-setup, wtv-head-waiter, etc
["socket_sessions", "socket_sessions"], // global socket_sessions object for privileged service scripts, such as wtv-1800, etc
[`ssid_sessions[${socket.ssid}]`, "session_data"] // user-specific session data from unprivileged scripts
];
["request_is_async", "request_is_async"] // we need to know if the script is async or not
]
if (!pc_services) {
updateFromVM.push(["ssid_sessions", "ssid_sessions"]); // global ssid_sessions object for privileged service scripts, such as wtv-setup, wtv-head-waiter, etc
updateFromVM.push(["socket_sessions", "socket_sessions"]); // global socket_sessions object for privileged service scripts, such as wtv-1800, etc
updateFromVM.push([`ssid_sessions[${socket.ssid}]`, "session_data"]); // user-specific session data from unprivileged scripts
}
try {
service_vaults.forEach(function (service_vault_dir) {
@@ -238,6 +248,7 @@ async function processPath(socket, service_vault_file_path, request_headers = ne
} else {
service_vault_file_path = wtvshared.makeSafePath(service_vault_dir, service_path);
}
// deny access to catchall file name directly
var service_path_split = service_path.split("/");
var service_path_request_file = service_path_split[service_path_split.length - 1];
@@ -373,7 +384,7 @@ async function processPath(socket, service_vault_file_path, request_headers = ne
var script_data = fs.readFileSync(service_vault_file_path + ".js").toString();
var priv = false;
if (minisrv_config.services[service_name]) priv = (minisrv_config.services[service_name].privileged) ? true : false;
else if (socket.minisrv_pc_mode) priv = (minisrv_config.services['pc_services'].privileged) ? true : false;
else if (pc_services) priv = (minisrv_config.services['pc_services'].privileged) ? true : false;
var vmResults = runScriptInVM(script_data, contextObj, priv, service_vault_file_path + ".js");
// Here we read back certain data from the ServiceVault Script Context Object
@@ -417,7 +428,7 @@ async function processPath(socket, service_vault_file_path, request_headers = ne
var script_data = fs.readFileSync(catchall_file).toString();
var priv = false;
if (minisrv_config.services[service_name]) priv = (minisrv_config.services[service_name].privileged) ? true : false;
else if (socket.minisrv_pc_mode) priv = (minisrv_config.services['pc_services'].privileged) ? true : false;
else if (pc_services) priv = (minisrv_config.services['pc_services'].privileged) ? true : false;
runScriptInVM(script_data, contextObj, priv, catchall_file);
@@ -444,7 +455,7 @@ async function processPath(socket, service_vault_file_path, request_headers = ne
var errpage = wtvshared.doErrorPage(400);
headers = errpage[0];
data = errpage[1];
if (socket.minisrv_pc_mode) {
if (pc_services) {
if (minisrv_config.services.pc_services.show_verbose_errors)
data += "<br><br>The interpreter said:<br><pre>" + e.stack + "</pre>";
}
@@ -453,12 +464,12 @@ async function processPath(socket, service_vault_file_path, request_headers = ne
if (!request_is_async) {
if (!service_vault_found) {
console.error(" * Could not find a Service Vault for " + service_name + ":/" + service_path.replace(service_name + path.sep, "").replace(path.sep, '/'));
var errpage = wtvshared.doErrorPage(404, null, socket.minisrv_pc_mode);
var errpage = wtvshared.doErrorPage(404, null, pc_services);
headers = errpage[0];
data = errpage[1];
}
if (headers == null && !request_is_async) {
var errpage = wtvshared.doErrorPage(400, null, socket.minisrv_pc_mode);
var errpage = wtvshared.doErrorPage(400, null, pc_services);
headers = errpage[0];
data = errpage[1];
console.error(" * Scripting or Data error: Headers were not defined. (headers,data) as follows:")
@@ -486,29 +497,27 @@ function verifyServicePort(service_name, socket) {
return false;
}
async function processURL(socket, request_headers) {
async function processURL(socket, request_headers, pc_services = false) {
var shortURL, headers, data, service_name = "";
var enable_multi_query = false;
request_headers.query = new Array();
request_headers.query = {};
if (request_headers.request_url) {
if (pc_services) {
service_name = verifyServicePort(request_headers.service_name, socket);
delete request_headers.service_name;
}
if (request_headers.request_url.indexOf('?') >= 0) {
shortURL = request_headers.request_url.split('?')[0];
if (socket.minisrv_pc_mode)
service_name = verifyServicePort("pc_services", socket);
else
service_name = verifyServicePort(shortURL.split(':')[0], socket);
if (!service_name) service_name = verifyServicePort(shortURL.split(':')[0], socket);
if (!service_name) {
service_name = verifyServicePort("pc_services", socket);
socket.minisrv_pc_mode = true;
}
if (!service_name) {
var errpage = wtvshared.doErrorPage(500, null, socket.minisrv_pc_mode);
var errpage = wtvshared.doErrorPage(500, null, pc_services);
socket_sessions[socket.id].close_me = true;
sendToClient(socket, errpage[0], errpage[1]);
return
}
if (request_headers.request_url.indexOf('?') >= 0) {
shortURL = request_headers.request_url.split('?')[0];
if (minisrv_config.services[service_name]) enable_multi_query = minisrv_config.services[service_name].enable_multi_query || false;
var qraw = request_headers.request_url.split('?')[1];
if (qraw.length > 0) {
@@ -534,6 +543,9 @@ async function processURL(socket, request_headers) {
} else {
shortURL = unescape(request_headers.request_url);
}
} else {
shortURL = unescape(request_headers.request_url);
}
if (request_headers['wtv-request-type']) socket_sessions[socket.id].wtv_request_type = request_headers['wtv-request-type'];
@@ -598,49 +610,9 @@ async function processURL(socket, request_headers) {
shortURL_split.shift();
var shortURL_service_path = shortURL_split.join(":");
shortURL = shortURL_service_name + ":/" + shortURL_service_path;
} else if (shortURL.indexOf(":") == -1 && request_headers.request.indexOf("HTTP/1") > 0 && request_headers.Host && socket.ssid == null) {
// PC browsers typically send a Host header
if (minisrv_config.services.pc_services) {
if (!minisrv_config.services.pc_services.disabled) {
socket.minisrv_pc_mode = true;
service_name = verifyServicePort("pc_services", socket);
if (!service_name) {
if (minisrv_config.services.pc_services.drop_connection_on_wrong_port) {
// just close the connection, no fancy error
socket.end();
return;
}
var errpage = wtvshared.doErrorPage(500, null, socket.minisrv_pc_mode);
socket_sessions[socket.id].close_me = true;
sendToClient(socket, errpage[0], errpage[1]);
return;
}
// if a directory, request index
if (shortURL.substring(shortURL.length - 1) == "/") shortURL += "index";
shortURL = service_name + ":" + shortURL;
} else {
// minimal pc mode to send error
socket.minisrv_pc_mode = true;
var errpage = wtvshared.doErrorPage(401, "PC services are disabled on this server", socket.minisrv_pc_mode);
headers = errpage[0];
data = errpage[1]
socket_sessions[socket.id].close_me = true;
sendToClient(socket, headers, data);
return;
}
} else {
// minimal pc mode to send error
socket.minisrv_pc_mode = true;
var errpage = wtvshared.doErrorPage(401, "PC services are disabled on this server", socket.minisrv_pc_mode);
headers = errpage[0];
data = errpage[1]
socket_sessions[socket.id].close_me = true;
sendToClient(socket, headers, data);
return;
}
}
if (!socket.minisrv_pc_mode && !socket.ssid) {
if (socket.ssid) {
// skip box auth tests for pc mode
// check security
@@ -704,10 +676,10 @@ minisrv-no-mail-count: true`;
} else {
console.log(" * " + reqverb + " for " + request_headers.request_url, 'on', socket.id);
}
if (!socket.minisrv_pc_mode) {
if (!pc_services) {
var service_name = verifyServicePort(shortURL.split(':/')[0], socket);
if (!service_name) {
var errpage = wtvshared.doErrorPage(500, null, socket.minisrv_pc_mode);
var errpage = wtvshared.doErrorPage(500, null, pc_services);
socket_sessions[socket.id].close_me = true;
sendToClient(socket, errpage[0], errpage[1]);
return
@@ -720,9 +692,14 @@ minisrv-no-mail-count: true`;
}
if (minisrv_config.config.debug_flags.show_headers) console.log(" * Incoming headers on socket ID", socket.id, (await wtvshared.decodePostData(wtvshared.filterRequestLog(wtvshared.filterSSID(request_headers)))));
socket_sessions[socket.id].request_headers = request_headers;
processPath(socket, urlToPath, request_headers, service_name, shared_romcache);
} else if (shortURL.indexOf('http://') >= 0 || shortURL.indexOf('https://') >= 0) {
processPath(socket, urlToPath, request_headers, service_name, shared_romcache, pc_services);
} else if ((shortURL.indexOf('http://') >= 0 || shortURL.indexOf('https://') >= 0) && !pc_services) {
doHTTPProxy(socket, request_headers);
} else if (pc_services) {
// if a directory, request index
if (shortURL.substring(shortURL.length - 1) == "/") shortURL += "index";
var urlToPath = wtvshared.fixPathSlashes(service_name + path.sep + shortURL);
processPath(socket, urlToPath, request_headers, service_name, shared_romcache, pc_services);
} else {
// error reading headers (no request_url provided)
var errpage = wtvshared.doErrorPage(400);
@@ -922,7 +899,7 @@ function stripHeaders(headers_obj, whitelist) {
function headerStringToObj(headers, response = false) {
var inc_headers = 0;
var headers_obj = new Array();
var headers_obj = {};
var headers_obj_pre = headers.split("\n");
headers_obj_pre.forEach(function (d) {
if (/^SECURE ON/.test(d) && !response) {
@@ -973,6 +950,7 @@ async function sendToClient(socket, headers_obj, data) {
socket.destroy();
return;
}
if (!socket.res) {
var wtv_connection_close = (headers_obj["wtv-connection-close"]) ? true : false;
if (typeof (headers_obj["wtv-connection-close"]) != 'undefined') delete headers_obj["wtv-connection-close"];
@@ -988,6 +966,7 @@ async function sendToClient(socket, headers_obj, data) {
if (headers_obj['wtv-mail-count']) delete headers_obj['wtv-mail-count'];
delete headers_obj['minisrv-no-mail-count'];
}
}
// add Connection header if missing, default to Keep-Alive
if (!headers_obj.Connection) {
@@ -1041,6 +1020,10 @@ async function sendToClient(socket, headers_obj, data) {
delete headers_obj["minisrv-force-compression"];
}
if (socket.res) { // pc mode with response object available
if (compression_type == 1) compression_type = 2; // just in case
}
// compress if needed
if (compression_type > 0 && content_length > 0 && headers_obj['Response'].substring(0, 3) == "200") {
var uncompressed_content_length = content_length;
@@ -1076,7 +1059,7 @@ async function sendToClient(socket, headers_obj, data) {
if (uncompressed_content_length != compressed_content_length) if (minisrv_config.config.debug_flags.debug) console.log(" # Compression stats: Orig Size:", uncompressed_content_length, "~ Comp Size:", compressed_content_length, "~ Ratio:", compression_ratio, "~ Saved:", compression_percentage.toString() + "%");
}
if (!socket.res) {
// encrypt if needed
if (socket_sessions[socket.id].secure == true && !socket_sessions[socket.id].do_not_encrypt) {
headers_obj["wtv-encrypted"] = 'true';
@@ -1092,6 +1075,7 @@ async function sendToClient(socket, headers_obj, data) {
if (headers_obj["wtv-encrypted"]) delete headers_obj["wtv-encrypted"];
if (headers_obj["secure"]) delete headers_obj["secure"];
}
}
// calculate content length
@@ -1109,6 +1093,7 @@ async function sendToClient(socket, headers_obj, data) {
}
if (!socket.res) {
// Send wtv-ticket if it has been flagged as updated
if (ssid_sessions[socket.ssid]) {
if (ssid_sessions[socket.ssid].data_store.wtvsec_login) {
@@ -1121,28 +1106,11 @@ async function sendToClient(socket, headers_obj, data) {
}
}
}
}
if (!socket.res) {
var end_of_line = "\n";
if (socket.minisrv_pc_mode) {
end_of_line = "\r\n";
headers_obj['Response'] = "HTTP/1.0 " + headers_obj['Response'];
}
/* // wtv-request-type download wants minimal headers?
if (data.byteLength > 0) {
if (socket_sessions[socket.id].wtv_request_type == "download") {
if (headers_obj['Content-Type'] != "wtv/download-list") {
// minimalize headers
var new_headers = { "Response": headers_obj['Response'].split(" ")[0] + " " }
if (headers_obj['wtv-encrypted']) new_headers['wtv-encrypted'] = headers_obj['wtv-encrypted'];
new_headers["content-type"] = headers_obj['Content-Type'];
new_headers["content-length"] = headers_obj['Content-length'];
headers_obj = new_headers;
}
}
}
*/
// header object to string
if (minisrv_config.config.debug_flags.show_headers) console.log(" * Outgoing headers on socket ID", socket.id, (await wtvshared.filterSSID(headers_obj)));
Object.keys(headers_obj).forEach(function (k) {
@@ -1163,8 +1131,18 @@ async function sendToClient(socket, headers_obj, data) {
socket_sessions[socket.id].destroy_me = true;
}
}
}
// send to client
if (socket.res) {
var resCode = parseInt(headers_obj.Response.substr(0, 3));
headers_obj['x-powered-by'] = "Express via " + z_title;
socket.res.writeHead(resCode, headers_obj);
socket.res.end(data);
var log_obj = Object.assign({}, socket.res.getHeaders());
if (minisrv_config.config.debug_flags.show_headers) console.log(" * Outgoing PC headers on " + socket.service_name + " socket ID", socket.id, log_obj);
} else {
var toClient = null;
if (typeof data == 'string') {
toClient = headers + end_of_line + data;
@@ -1182,6 +1160,7 @@ async function sendToClient(socket, headers_obj, data) {
if (minisrv_config.config.debug_flags.quiet) console.log(" * Sent" + verbosity_mod + " " + headers_obj.Response + " to client (Content-Type:", headers_obj['Content-Type'], "~", headers_obj['Content-length'], "bytes)");
}
}
}
async function sendToSocket(socket, data) {
var chunk_size = 16384;
@@ -1542,13 +1521,7 @@ async function processRequest(socket, data_hex, skipSecure = false, encryptedReq
socket_sessions[socket.id].headers = headers;
}
} else {
if (!socket.ssid) {
// incomplete, dont use https on pc services yet
socket = new tls.TLSSocket(socket);
socket.minisrv_pc_mode = true;
console.log(socket)
console.log("pc https?");
} else {
// handle streaming POST
if (socket_sessions[socket.id].expecting_post_data && headers) {
socket_sessions[socket.id].headers = headers;
@@ -1676,7 +1649,6 @@ async function processRequest(socket, data_hex, skipSecure = false, encryptedReq
}
}
}
}
async function cleanupSocket(socket) {
try {
@@ -1784,8 +1756,8 @@ function reloadConfig() {
// SERVER START
var git_commit = getGitRevision()
var z_title = "zefie's wtv minisrv v" + require('./package.json').version;
if (git_commit) console.log("**** Welcome to " + z_title + " (git " + git_commit + ") ****");
else console.log("**** Welcome to " + z_title + " ****");
if (git_commit) z_title += " (git " + git_commit + ")";
console.log("**** Welcome to " + z_title + " ****");
const wtvshared = new WTVShared(); // creates minisrv_config
minisrv_config = wtvshared.getMiniSrvConfig(); // snatches minisrv_config
@@ -1834,7 +1806,8 @@ Object.keys(minisrv_config.services).forEach(function (k) {
minisrv_config.services[k].host = service_ip;
}
if (minisrv_config.services[k].port && !minisrv_config.services[k].nobind) {
ports.push(minisrv_config.services[k].port);
if (minisrv_config.services[k].pc_services) pc_ports.push(minisrv_config.services[k].port);
else ports.push(minisrv_config.services[k].port);
}
// minisrv_config service toString
minisrv_config.services[k].toString = function (overrides) {
@@ -1892,8 +1865,22 @@ process.on('uncaughtException', function (err) {
});
function findServiceByPort(port) {
var service_name = null;
Object.keys(minisrv_config.services).forEach(function (k) {
if (service_name) return;
if (minisrv_config.services[k].port) {
if (port == parseInt(minisrv_config.services[k].port))
service_name = k;
}
})
return service_name;
}
var initstring = '';
ports.sort();
pc_ports.sort();
// de-duplicate ports in case user configured multiple services on same port
const bind_ports = [...new Set(ports)]
@@ -1909,6 +1896,40 @@ bind_ports.every(function (v) {
}
return false;
});
// PC Services via express
// de-duplicate ports in case user configured multiple services on same port
const pc_bind_ports = [...new Set(pc_ports)]
if (!minisrv_config.config.bind_ip) minisrv_config.config.bind_ip = "0.0.0.0";
pc_bind_ports.every(function (v) {
try {
var server = express();
server.listen(v, minisrv_config.config.bind_ip);
initstring += v + ", ";
server.get('*', (req, res) => {
var request_headers = {};
req.socket.id = parseInt(crc16('CCITT-FALSE', Buffer.from(String(req.socket.remoteAddress) + String(req.socket.remotePort), "utf8")).toString(16), 16);
socket_sessions[req.socket.id] = [];
var service_name = findServiceByPort(v);
request_headers['request'] = "GET " + req.originalUrl + " HTTP/1.1";
request_headers.request_url = req.originalUrl;
Object.keys(req.headers).forEach(function (k) {
request_headers[k] = req.headers[k];
});
request_headers.query = req.query;
if (minisrv_config.config.debug_flags.show_headers) console.log(" * Incoming PC Headers on", service_name, "socket ID", req.socket.id, wtvshared.filterRequestLog(request_headers));
request_headers.service_name = service_name;
req.socket.minisrv_pc_mode = true;
req.socket.res = res;
req.socket.service_name = service_name;
processURL(req.socket, request_headers, true)
})
return true;
} catch (e) {
throw ("Could not bind to port", v, "on", minisrv_config.config.bind_ip, e.toString());
}
return false;
});
initstring = initstring.substring(0, initstring.length - 2);

2
zefie_wtvp_minisrv/config.json
View File

@@ -155,6 +155,8 @@
},
"pc_services": {
"port": 1699,
"pc_services": true,
"hide_minisrv_version": true,
"disabled": true,
"servicevault_dir": "http_pc",
"drop_connection_on_wrong_port": false,

912
zefie_wtvp_minisrv/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

3
zefie_wtvp_minisrv/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "zefie_wtvp_minisrv",
"version": "0.9.31",
"version": "0.9.32",
"description": "WebTV Service (WTVP) Emulation Server",
"main": "app.js",
"homepage": "https://github.com/zefie/zefie_wtvp_minisrv",
@@ -31,6 +31,7 @@
"crypto-js": "^4.1.1",
"easy-crc": "0.0.2",
"endianness": "^8.0.2",
"express": "^4.18.2",
"html-entities": "^2.3.3",
"iconv-lite": "^0.6.3",
"mime-types": "^2.1.35",