zefie/minisrv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modulize HTTP(s) proxy, add Gopher support, change how protocol-specific modules are loaded

Browse Source
This commit is contained in:
zefie
2026-04-23 14:46:43 -04:00
parent 1b108c64f3
commit 614b4fe5f7
6 changed files with 595 additions and 265 deletions
Download Patch File Download Diff File Expand all files Collapse all files

295
zefie_wtvp_minisrv/app.js
View File

@@ -11,8 +11,7 @@ const nunjucks = require('nunjucks');
const zlib = require('zlib');
const {serialize, unserialize} = require('php-serialize');
const {spawn} = require('child_process');
const http = require('follow-redirects').http
const https = require('follow-redirects').https
const http = require('follow-redirects').http;
const httpx = require(classPath + "/HTTPX.js");
const { URL } = require('url');
const net = require('net');
@@ -32,6 +31,7 @@ const WTVPNM = require(classPath + "/WTVPNM.js");
const vm = require('vm');
const debug = require('debug')('app');
const express = require('express');
let handlerModules = [];
let wtvnewsserver = null;
const protocolServers = [];
@@ -304,8 +304,8 @@ const runScriptInVM = function (script_data, user_contextObj = {}, privileged =
// Our modules
"wtvmime": wtvmime,
"http": http,
"https": https,
"http": require('follow-redirects').http,
"https": require('follow-redirects').https,
"util": util,
"sharp": sharp,
"nunjucks": nunjucks,
@@ -984,7 +984,11 @@ async function processURL(socket, request_headers, pc_services = false) {
let shortURL, headers, data, service_name;
let original_service_name = "";
let shared_romcache = null;
let allow_double_slash = false, enable_multi_query = false, use_external_proxy = false;
let allow_double_slash = false;
let enable_multi_query = false;
let use_external_proxy = false;
let disallow_no_slash = false;
let uses_service_vault = true;
request_headers.query = {};
if (request_headers.request_url) {
@@ -993,6 +997,8 @@ async function processURL(socket, request_headers, pc_services = false) {
allow_double_slash = minisrv_config.services[service_name].allow_double_slash || false;
enable_multi_query = minisrv_config.services[service_name].enable_multi_query || false;
use_external_proxy = minisrv_config.services[service_name].use_external_proxy || false;
disallow_no_slash = minisrv_config.services[service_name].disallow_no_slash || false;
uses_service_vault = (minisrv_config.services[service_name].uses_service_vault === false) ? false : true;
}
if (pc_services) {
original_service_name = socket.service_name; // store service name
@@ -1078,7 +1084,7 @@ async function processURL(socket, request_headers, pc_services = false) {
}
}
if ((!shortURL.startsWith("http") && !shortURL.startsWith("ftp") && shortURL.includes(":") && !shortURL.includes(":/"))) {
if ((!disallow_no_slash && shortURL.includes(":") && !shortURL.includes(":/"))) {
// Apparently it is within WTVP spec to accept urls without a slash (eg wtv-home:home)
// Here, we just reassemble the request URL as if it was a proper URL (eg wtv-home:/home)
// we will allow this on any service except http(s) and ftp
@@ -1143,7 +1149,7 @@ minisrv-no-mail-count: true`;
else console.log(" * " + ((ssl) ? "SSL " : "") + "PC request on service " + original_service_name + " (Service Vault " + service_name + ") for " + request_headers.request_url, 'on', socket.id);
}
if ((shortURL.includes(':/')) && (!shortURL.includes('://') || (shortURL.includes('://') && allow_double_slash))) {
if ((shortURL.includes(':/')) && (!shortURL.includes('://') || (shortURL.includes('://') && allow_double_slash) && uses_service_vault)) {
let ssid = socket.ssid;
if (ssid === null) {
// prevent possible injection attacks via malformed SSID and filesystem SessionStore
@@ -1185,8 +1191,10 @@ minisrv-no-mail-count: true`;
socket_sessions[socket.id].request_headers = request_headers;
processPath(socket, urlToPath, request_headers, service_name, shared_romcache, pc_services);
} else if (shortURL.includes('http://') || shortURL.includes('https://') || (use_external_proxy === true && shortURL.includes(service_name + "://")) && !pc_services) {
doHTTPProxy(socket, request_headers);
} else if (handlerModules["wtvhttp"] && ((shortURL.includes('http://') || shortURL.includes('https://')) || (use_external_proxy === true && shortURL.includes(service_name + "://")) && !pc_services)) {
handlerModules["wtvhttp"].doHTTPProxy(socket, request_headers);
} else if (handlerModules["wtvgopher"] && shortURL.startsWith("gopher://")) {
handlerModules["wtvgopher"].handleGopherRequest(socket, request_headers, wtvshared, sendToClient);
} else if (shortURL.startsWith('ftp://')) {
if (minisrv_config.config.debug_flags.show_headers) console.debug(" * Incoming FTP request on WTVP socket ID", socket.id, await wtvshared.decodePostData(await wtvshared.filterRequestLog(await wtvshared.filterSSID(request_headers))));
const wtvftp = new WTVFTP(wtvshared, sendToClient);
@@ -1230,250 +1238,6 @@ minisrv-no-mail-count: true`;
}
}
function handleProxy(socket, request_type, request_headers, res, data) {
console.log(` * Proxy Request ${request_type.toUpperCase()} ${res.statusCode} for ${request_headers.request}`)
// an http response error is not a request error, and will come here under the 'end' event rather than an 'error' event.
switch (res.statusCode) {
case 404:
res.headers.Status = res.statusCode + " The publisher can’t find the page requested.";
break;
case 401:
case 403:
res.headers.Status = res.statusCode + " The publisher of that page has not authorized you to use it.";
break;
case 500:
res.headers.Status = res.statusCode + " The publisher of that page can’t be reached.";
break;
default:
res.headers.Status = res.statusCode + " " + res.statusMessage;
break;
}
if (res.headers['Content-type']) {
res.headers['Content-Type'] = res.headers['Content-type'];
delete (res.headers['Content-type'])
}
if (res.headers['content-type']) {
res.headers['Content-Type'] = res.headers['content-type'];
delete (res.headers['content-type'])
}
// header pass-through whitelist, case insensitive comparsion to server, however, you should
// specify the header case as you intend for the client
const headers = wtvshared.stripHeaders(res.headers, [
'Connection',
'Server',
'Date',
'Content-Type',
'Cookie',
'Location',
'Accept-Ranges',
'Last-Modified'
]);
headers["wtv-http-proxy"] = true;
headers["wtv-trusted"] = false;
if (typeof res.headers['Content-Type'] === 'string' && res.headers['Content-Type'].startsWith("text")) {
// Get the original URL for relative link fixing
const originalUrl = request_headers.request.split(' ')[1];
// Transform HTML content for WebTV compatibility
if (res.headers['Content-Type'].includes('html') &&
minisrv_config.services[request_type]?.use_minifying_proxy === true) {
try {
const WTVMinifyingProxy = require('./includes/classes/WTVMinifyingProxy.js');
const proxy = new WTVMinifyingProxy(minisrv_config);
let htmlContent = Buffer.concat(data).toString();
// Apply WebTV-specific transformations
const transformOptions = {
removeImages: minisrv_config.services[request_type]?.remove_images || false,
maxImageWidth: minisrv_config.services[request_type]?.max_image_width || 400,
simplifyTables: minisrv_config.services[request_type]?.simplify_tables !== false,
maxWidth: minisrv_config.services[request_type]?.max_width || 544,
preserveJellyScript: minisrv_config.services[request_type]?.preserve_jellyscript !== false,
jellyScriptMaxSize: minisrv_config.services[request_type]?.jellyscript_max_size || 8192
};
htmlContent = proxy.transformForWebTV(htmlContent, originalUrl, transformOptions);
data = [Buffer.from(htmlContent)];
if (minisrv_config.config.verbosity >= 3) {
console.log(` * HTML transformed for WebTV compatibility (${originalUrl})`);
}
} catch (err) {
console.warn(` * HTML transformation failed: ${err.message}`);
}
}
if (request_type !== "http" && request_type !== "https") {
// replace http and https links on non http/https protocol (for proto:// for example)
const data_t = Buffer.concat(data).toString().replaceAll("http://", request_type + "://").replaceAll("https://", request_type + "://");
data = [Buffer.from(data_t)]
}
}
// if Connection: close header, set our internal variable to close the socket
if (headers['Connection']) {
if (headers['Connection'].toLowerCase().includes('close')) {
headers["wtv-connection-close"] = true;
}
}
// if a wtv-explaination is defined for an error code (except 200), define the header here to
// show the 'Explain' button on the client error ShowAlert
if (minisrv_config.services['http']['wtv-explanation']) {
if (minisrv_config.services['http']['wtv-explanation'][res.statusCode]) {
headers['wtv-explanation-url'] = minisrv_config.services['http']['wtv-explanation'][res.statusCode];
}
}
let data_hex = Buffer.concat(data).toString('hex');
if (data_hex.startsWith("0d0a0d0a")) data_hex = data_hex.slice(8);
if (data_hex.startsWith("0a0d0a")) data_hex = data_hex.slice(6);
if (data_hex.startsWith("0a0a")) data_hex = data_hex.slice(4);
sendToClient(socket, headers, Buffer.from(data_hex, 'hex'));
}
async function doHTTPProxy(socket, request_headers) {
// detect protocol name
const idx = request_headers.request_url.indexOf('/') - 1;
const request_type = request_headers.request_url.slice(0, idx);
if (minisrv_config.config.debug_flags.show_headers) console.debug(request_type.toUpperCase() + " Proxy: Client Request Headers on socket ID", socket.id, (await wtvshared.decodePostData(await wtvshared.filterRequestLog(await wtvshared.filterSSID(request_headers)))));
else debug(request_type.toUpperCase() + " Proxy: Client Request Headers on socket ID", socket.id, (await wtvshared.decodePostData(await wtvshared.filterRequestLog(await wtvshared.filterSSID(request_headers)))));
let proxy_agent;
switch (request_type) {
case "https":
proxy_agent = https;
break;
case "http":
case "proto":
proxy_agent = http;
break;
}
const request_data = [];
const data = [];
request_data.method = request_headers.request.split(' ')[0];
const request_url_split = request_headers.request.split(' ')[1].split('/');
request_data.host = request_url_split[2];
if (request_data.host.indexOf(':') > 0) {
request_data.port = request_data.host.split(':')[1];
request_data.host = request_data.host.split(':')[0];
} else {
if (request_type === "https") request_data.port = 443;
else request_data.port = 80;
}
for (let i = 0; i < 3; i++) request_url_split.shift();
request_data.path = "/" + request_url_split.join('/');
if (request_data.method && request_data.host && request_data.path) {
const options = {
host: request_data.host,
port: request_data.port,
path: request_data.path,
method: request_data.method,
followAllRedirects: true,
headers: {
"User-Agent": request_headers["User-Agent"] || "WebTV",
"Connection": "Keep-Alive"
}
}
// RFC7239
if (socket.remoteAddress !== "127.0.0.1") {
options.headers["X-Forwarded-For"] = socket.remoteAddress;
}
if (request_headers.post_data) {
if (request_headers["Content-type"]) options.headers["Content-type"] = request_headers["Content-type"];
if (request_headers["Content-length"]) options.headers["Content-length"] = request_headers["Content-length"];
}
if (minisrv_config.services[request_type].use_external_proxy && minisrv_config.services[request_type].external_proxy_port) {
// configure connection to an external proxy
if (minisrv_config.services[request_type].external_proxy_is_socks) {
// configure connection to remote socks proxy
const { SocksProxyAgent }= require('socks-proxy-agent');
options.agent = new SocksProxyAgent("socks://" + (minisrv_config.services[request_type].external_proxy_host || "127.0.0.1") + ":" + minisrv_config.services[request_type].external_proxy_port);
options.agents = {
"http": options.agent,
"https": options.agent
}
} else {
// configure connection to remote http proxy
proxy_agent = http;
options.host = minisrv_config.services[request_type].external_proxy_host;
options.port = minisrv_config.services[request_type].external_proxy_port;
options.path = request_headers.request.split(' ')[1];
options.headers.Host = request_data.host + ":" + request_data.port;
if (minisrv_config.services[request_type].replace_protocol) {
options.path = options.path.replace(request_type, minisrv_config.services[request_type].replace_protocol);
}
}
if (minisrv_config.services[request_type].external_proxy_is_http1) {
options.insecureHTTPParser = true;
options.headers.Connection = 'close'
}
}
const req = proxy_agent.request(options, function (res) {
let total_data = 0;
res.on('data', d => {
data.push(d);
total_data += d.length;
if (total_data > 1024 * 1024 * parseFloat(minisrv_config.services[request_type].max_response_size || 16)) {
console.warn(` * Response data exceeded ${minisrv_config.services[request_type].max_response_size || 16}MB limit, destroying...`);
res.destroy();
const errpage = wtvshared.doErrorPage(400, "The item chosen is too large to be used.");
sendToClient(socket, errpage[0], errpage[1]);
}
})
res.on('error', function (err) {
// hack for Protoweb ECONNRESET
if (minisrv_config.services[request_type].external_proxy_is_http1 && data.length > 0) {
handleProxy(socket, request_type, request_headers, res, data);
} else {
console.error(" * Unhandled Proxy Request Error:", err);
}
});
res.on('end', function () {
// For when http proxies behave correctly
if (!minisrv_config.services[request_type].external_proxy_is_http1 || data.length > 0) {
handleProxy(socket, request_type, request_headers, res, data);
}
});
}).on('error', function (err) {
// severe errors, such as unable to connect.
if (err.code === "ENOTFOUND" || err.message.indexOf("HostUnreachable") > 0) {
const errpage = wtvshared.doErrorPage(400, `The publisher <b>${request_data.host}</b> is unknown.`);
sendToClient(socket, errpage[0], errpage[1]);
} else {
console.error(" * Unhandled Proxy Request Error:", err);
const errpage = wtvshared.doErrorPage(400);
sendToClient(socket, errpage[0], errpage[1]);
}
});
if (request_headers.post_data) {
req.write(Buffer.from(request_headers.post_data.toString(CryptoJS.enc.Hex), 'hex'), function () {
req.end();
});
} else {
req.end();
}
}
}
async function sendToClient(socket, headers_obj, data = null) {
let headers = "";
let content_length = 0;
@@ -2463,9 +2227,27 @@ const service_ip = minisrv_config.config.service_ip;
Object.keys(minisrv_config.services).forEach(function (k) {
if (typeof(minisrv_config.services[k]) === 'function') return;
if (configureService(k, minisrv_config.services[k], true)) {
let loadedModule = false;
if (minisrv_config.services[k].handler_module) {
try {
if (!handlerModules[minisrv_config.services[k].handler_module + "_main"]) {
handlerModules[minisrv_config.services[k].handler_module + "_main"] = require(classPath + "/" + minisrv_config.services[k].handler_module + ".js");
var args = [];
for (let i = 0; i < (minisrv_config.services[k].handler_extra_vars || []).length; i++) {
let extraVar = eval(minisrv_config.services[k].handler_extra_vars[i]);
args.push(extraVar);
}
const constructorArgs = [minisrv_config, k, ...args];
handlerModules[minisrv_config.services[k].handler_module.toLowerCase()] = new handlerModules[minisrv_config.services[k].handler_module + "_main"](...constructorArgs);
}
loadedModule = true;
} catch (e) {
console.error(" # Failed to load handler module for service", k, "module:", minisrv_config.services[k].handler_module, e);
}
}
const using_tls = (minisrv_config.services[k].pc_services && minisrv_config.services[k].https_cert && minisrv_config.services[k].use_https) ? true : false;
const protocol = (minisrv_config.services[k].protocol_handler) ? minisrv_config.services[k].protocol_handler.toUpperCase() : (minisrv_config.services[k].pc_services) ? "HTTP" : "WTVP";
console.log(" * Configured Service:", k, "on Port", minisrv_config.services[k].port, "- Service Host:", minisrv_config.services[k].host + ((using_tls) ? " (TLS)" : ""), "- Protocol:", protocol);
console.log(" * Configured Service:", k, "on Port", minisrv_config.services[k].port, "- Service Host:", minisrv_config.services[k].host + ((using_tls) ? " (TLS)" : ""), "- Protocol:", protocol, (loadedModule) ? "- Handler Module: " + minisrv_config.services[k].handler_module : "");
if (minisrv_config.services[k].local_nntp_enabled && minisrv_config.services[k].local_nntp_port) {
if (!wtvnewsserver) {
@@ -2548,9 +2330,8 @@ Object.keys(minisrv_config.services).forEach((service_name) => {
if (!service || service.disabled || !service.port) return;
if (service.protocol_handler === 'pnm') {
try {
const pnmServer = new WTVPNM(minisrv_config, service_name);
pnmServer.listen(service.port, minisrv_config.config.bind_ip);
protocolServers.push(pnmServer);
handlerModules['wtvpnm'].listen(service.port, minisrv_config.config.bind_ip);
protocolServers.push(handlerModules['wtvpnm']);
} catch (e) {
throw ("Could not bind PNM protocol handler to port " + service.port + " on " + minisrv_config.config.bind_ip + ": " + e.toString());
}

259
zefie_wtvp_minisrv/includes/classes/WTVGopher.js Normal file
View File

@@ -0,0 +1,259 @@
const WTVMime = require('./WTVMime.js');
const net = require('net');
class WTVGopher {
// Adapted from WebTV Redialed's Gopher support
constructor(...[minisrv_config, service_name]) {
this.minisrv_config = minisrv_config;
this.wtvmime = new WTVMime(minisrv_config);
this.logGopher = minisrv_config.services[service_name].log_raw_gopher || false;
}
looksLikeMenu(gopherData) {
const lines = gopherData.split(/\r?\n/);
let checked = 0;
let menuLines = 0;
for (const line of lines) {
if (!line || line === ".") continue;
checked++;
let typeOffset = 0;
let type = " ";
while ((type === " " || type == "\t") && typeOffset <= 10) {
type = line[typeOffset];
typeOffset++;
}
const rest = line.slice(1);
if (
rest.includes("\t") &&
rest.split("\t").length >= 3 &&
/^[0-9A-Za-z+gIihs]$/.test(type)
) {
menuLines++;
}
if (checked >= 5) break;
}
return menuLines >= 2 || (lines.length <= 2 && menuLines == 1);
}
processGopherData(gopherData) {
// currently looking at textfile, don't process into HTML
if (!this.looksLikeMenu(gopherData)) {
return `<pre>${gopherData}</pre>`;
}
// okay, we're not a textfile, now do the menu shit
let pageTitle = "Gopher Menu"
const lines = gopherData.split("\r\n");
let html = "";
for (const line of lines) {
if (!line || line === ".") continue;
const type = line[0];
const parts = line.slice(1).split("\t");
const text = parts[0] || "";
const selector = parts[1];
const host = parts[2];
const port = parts[3] || 70;
var url = `gopher://${host}:${port}${selector}`;
// determine page title from first line
const firstline = line[0].slice(1).trim();
if (line[0] === "i" && firstline.length > 0) {
pageTitle = line.slice(1).trim();
html = `<title>${pageTitle}</title><pre>\n`;
} else if (pageTitle === "Gopher Menu") {
for (const line of lines) {
if (!line || line === ".") continue;
let typeOffset = 0;
let type = " ";
while ((type === " " || type == "\t") && typeOffset <= 10) {
type = line[typeOffset];
typeOffset++;
}
const parts = line.slice(1).split("\t");
const text = parts[0]?.trim();
if (text && text.length > 0) {
pageTitle = text;
html = `<title>${pageTitle}</title><pre>\n`;
break;
}
}
}
switch (type) {
case "i": // informational / "just text"
html += `${text}\n`;
break;
case "0": // text file
case "1": // directory
html += `<a href="${url}">${text}</a>\n`;
break;
case "3": // error, otherwise just plain text
html += `${text}<br>\n`;
case "h": // HTML link
if (selector?.startsWith("URL:")) {
const httpUrl = selector.slice(4);
html += `<a href="${httpUrl}">${text}</a>\n`;
}
break;
case "7": // search
html += `<form action="${url}" method="get">
<label for="search">${text}</label>
<input type="search" name="q" required>
</form>`;
break;
case "g":
case "I":
case "p":
url = `gopher://${host}:${port}${selector}?type=${type}`;
html += `<a href="${url}">${text}</a>\n`;
break;
default:
html += `${text} (unsupported type ${type})\n`;
}
}
html += "</pre>";
return html;
}
async handleGopherRequest(socket, request_headers, wtvshared, sendToClient) {
if (this.minisrv_config.config.debug_flags.show_headers) {
console.log("Gopher: Client Request on socket ID",
socket.id,
await wtvshared.decodePostData(
wtvshared.filterRequestLog(wtvshared.filterSSID(request_headers))
));
}
// crlf for sending at the end of a request
const crlf = "0D0A"
const crlf_bytes = Buffer.from(crlf, 'hex');
// chunk stuff for gopher-to-html conversion
let chunks = [];
var request_data = new Array();
request_data.method = request_headers.request.split(' ')[0];
const rawUrl = decodeURIComponent(request_headers.request.split(' ')[1]).replaceAll('\\', '/');
const [pathPart, queryPart] = rawUrl.split('?');
var request_url_split = pathPart.split('/');
let queryParams = {};
if (queryPart) {
for (const kv of queryPart.split('&')) {
const [k, v] = kv.split('=');
queryParams[k] = decodeURIComponent(v || "");
}
}
request_data.host = request_url_split[2];
if (request_data.host.indexOf(':') > 0) {
request_data.port = request_data.host.split(':')[1];
request_data.host = request_data.host.split(':')[0];
} else {
request_data.port = 70;
}
for (var i = 0; i < 3; i++) request_url_split.shift();
request_data.path = "/" + request_url_split.join('/');
// vars for determining if a link is an image
const imageTypes = ["g", "I", "p"];
let requestType = null;
if (queryParams.type && imageTypes.includes(queryParams.type)) {
requestType = queryParams.type;
}
const isImageDownload = !!requestType;
const client = new net.Socket();
client.setTimeout(3000);
// make the initial request to the server
client.connect(request_data.port, request_data.host, () => {
let gopherRequest = "";
// if user requested path
if (request_data.path.length >= 2) {
gopherRequest = request_data.path;
}
// if user requested type 7 (search)
if (queryParams.q) {
const query = queryParams.q.replace(/\+/g, ' ');
gopherRequest += "\t" + query;
}
client.write(gopherRequest + crlf_bytes);
});
// "holy shit we got data guys"
client.on("data", chunk => {
chunks.push(chunk);
});
// datastream end, time to process it
client.on("end", () => {
const gopherData = Buffer.concat(chunks).toString("utf-8");
if (this.logGopher) {
console.log("Gopher: Data received from server for socket ID", socket.id);
console.log("Gopher: Data length:", Buffer.concat(chunks).length);
console.log("isImageDownload:", isImageDownload);
console.log("Gopher Data:\n", gopherData);
}
// are we downloading an image?
if (isImageDownload) {
const imageData = Buffer.concat(chunks);
const mimetype = this.wtvmime.detectMimeTypeFromBuffer(imageData);
const headers = {
"Status": "200 OK",
"Content-Type": mimetype
}
sendToClient(socket, headers, imageData);
return;
} else {
// convert gophermap to html
const htmlData = this.processGopherData(gopherData);
// since gopher doesn't exactly have "headers" and by this point we're probably already fine to just say it's okay, we're just sending back the bare minimum to prevent screaming
const headers = {
"Status": "200 OK",
"Content-Type": "text/html"
}
sendToClient(socket, headers, htmlData);
}
});
// blew up?
// todo: figure out what error actually looks like and if appropriate to send to client (or just "Connection failed" or smth)
client.on('error', (err) => {
console.error('Gopher error: ' + err);
let friendlyErr = err.toString();
if (friendlyErr.includes('ETIMEDOUT')) {
friendlyErr = "Connection timed out";
} else if (friendlyErr.includes('ECONNREFUSED')) {
friendlyErr = "Connection refused";
} else if (friendlyErr.includes('ENOTFOUND')) {
friendlyErr = "Host not found";
}
sendToClient(socket, {"Status": "400 Gopher Error: " + friendlyErr}, friendlyErr);
});
}
}
module.exports = WTVGopher;

266
zefie_wtvp_minisrv/includes/classes/WTVHTTP.js Normal file
View File

@@ -0,0 +1,266 @@
const {WTVShared, clientShowAlert} = require('./WTVShared.js');
class WTVHTTP {
constructor(...[minisrv_config, service_name, http, sendToClient]) {
this.minisrv_config = minisrv_config;
this.service_name = service_name;
this.wtvshared = new WTVShared(minisrv_config);
this.sendToClient = sendToClient;
this.http = http;
this.https = require('follow-redirects').https
this.proxy_agent = null;
}
async doHTTPProxy(socket, request_headers) {
// detect protocol name
const idx = request_headers.request_url.indexOf('/') - 1;
const request_type = request_headers.request_url.slice(0, idx);
if (this.minisrv_config.config.debug_flags.show_headers) console.debug(request_type.toUpperCase() + " Proxy: Client Request Headers on socket ID", socket.id, (await this.wtvshared.decodePostData(await this.wtvshared.filterRequestLog(await this.wtvshared.filterSSID(request_headers)))));
else debug(request_type.toUpperCase() + " Proxy: Client Request Headers on socket ID", socket.id, (await this.wtvshared.decodePostData(await this.wtvshared.filterRequestLog(await this.wtvshared.filterSSID(request_headers)))));
switch (request_type) {
case "https":
this.proxy_agent = this.https;
break;
default:
this.proxy_agent = this.http;
break;
}
const request_data = [];
const data = [];
request_data.method = request_headers.request.split(' ')[0];
const request_url_split = request_headers.request.split(' ')[1].split('/');
request_data.host = request_url_split[2];
if (request_data.host.indexOf(':') > 0) {
request_data.port = request_data.host.split(':')[1];
request_data.host = request_data.host.split(':')[0];
} else {
if (request_type === "https") request_data.port = 443;
else request_data.port = 80;
}
for (let i = 0; i < 3; i++) request_url_split.shift();
request_data.path = "/" + request_url_split.join('/');
if (request_data.method && request_data.host && request_data.path) {
const options = {
host: request_data.host,
port: request_data.port,
path: request_data.path,
method: request_data.method,
followAllRedirects: true,
headers: {
"User-Agent": request_headers["User-Agent"] || "WebTV",
"Connection": "Keep-Alive"
}
}
// RFC7239
if (socket.remoteAddress !== "127.0.0.1") {
options.headers["X-Forwarded-For"] = socket.remoteAddress;
}
if (request_headers.post_data) {
if (request_headers["Content-type"]) options.headers["Content-type"] = request_headers["Content-type"];
if (request_headers["Content-length"]) options.headers["Content-length"] = request_headers["Content-length"];
}
if (request_type == "https" && this.minisrv_config.services[request_type].allow_self_signed_ssl) {
options.rejectUnauthorized = false;
}
if (this.minisrv_config.services[request_type].use_external_proxy && minisrv_config.services[request_type].external_proxy_port) {
// configure connection to an external proxy
if (this.minisrv_config.services[request_type].external_proxy_is_socks) {
// configure connection to remote socks proxy
const { SocksProxyAgent }= require('socks-proxy-agent');
options.agent = new SocksProxyAgent("socks://" + (minisrv_config.services[request_type].external_proxy_host || "127.0.0.1") + ":" + minisrv_config.services[request_type].external_proxy_port);
options.agents = {
"http": options.agent,
"https": options.agent
}
} else {
// configure connection to remote http proxy
this.proxy_agent = this.http;
options.host = this.minisrv_config.services[request_type].external_proxy_host;
options.port = this.minisrv_config.services[request_type].external_proxy_port;
options.path = request_headers.request.split(' ')[1];
options.headers.Host = request_data.host + ":" + request_data.port;
if (this.minisrv_config.services[request_type].replace_protocol) {
options.path = options.path.replace(request_type, this.minisrv_config.services[request_type].replace_protocol);
}
}
if (this.minisrv_config.services[request_type].external_proxy_is_http1) {
options.insecureHTTPParser = true;
options.headers.Connection = 'close'
}
}
const req = this.proxy_agent.request(options, (res) => {
let total_data = 0;
res.on('data', d => {
data.push(d);
total_data += d.length;
if (total_data > 1024 * 1024 * parseFloat(this.minisrv_config.services[request_type].max_response_size || 16)) {
console.warn(` * Response data exceeded ${this.minisrv_config.services[request_type].max_response_size || 16}MB limit, destroying...`);
res.destroy();
const errpage = this.wtvshared.doErrorPage(400, "The item chosen is too large to be used.");
this.sendToClient(socket, errpage[0], errpage[1]);
}
})
res.on('error', (err) => {
// hack for Protoweb ECONNRESET
if (this.minisrv_config.services[request_type].external_proxy_is_http1 && data.length > 0) {
this.handleProxy(socket, request_type, request_headers, res, data);
} else {
console.error(" * Unhandled Proxy Request Error:", err);
}
});
res.on('end', () => {
// For when http proxies behave correctly
if (!this.minisrv_config.services[request_type].external_proxy_is_http1 || data.length > 0) {
this.handleProxy(socket, request_type, request_headers, res, data);
}
});
}).on('error', (err) => {
// severe errors, such as unable to connect.
if (err.code === "ENOTFOUND" || err.message.indexOf("HostUnreachable") > 0) {
const errpage = this.wtvshared.doErrorPage(400, `The publisher <b>${request_data.host}</b> is unknown.`);
this.sendToClient(socket, errpage[0], errpage[1]);
} else {
console.error(" * Unhandled Proxy Request Error:", err);
const errpage = this.wtvshared.doErrorPage(400);
this.sendToClient(socket, errpage[0], errpage[1]);
}
});
if (request_headers.post_data) {
req.write(Buffer.from(request_headers.post_data.toString(CryptoJS.enc.Hex), 'hex'), () => {
req.end();
});
} else {
req.end();
}
}
}
handleProxy(socket, request_type, request_headers, res, data) {
console.log(` * Proxy Request ${request_type.toUpperCase()} ${res.statusCode} for ${request_headers.request}`)
// an http response error is not a request error, and will come here under the 'end' event rather than an 'error' event.
switch (res.statusCode) {
case 404:
res.headers.Status = res.statusCode + " The publisher can&#146;t find the page requested.";
break;
case 401:
case 403:
res.headers.Status = res.statusCode + " The publisher of that page has not authorized you to use it.";
break;
case 500:
res.headers.Status = res.statusCode + " The publisher of that page can&#146;t be reached.";
break;
default:
res.headers.Status = res.statusCode + " " + res.statusMessage;
break;
}
if (res.headers['Content-type']) {
res.headers['Content-Type'] = res.headers['Content-type'];
delete (res.headers['Content-type'])
}
if (res.headers['content-type']) {
res.headers['Content-Type'] = res.headers['content-type'];
delete (res.headers['content-type'])
}
// header pass-through whitelist, case insensitive comparsion to server, however, you should
// specify the header case as you intend for the client
const headers = this.wtvshared.stripHeaders(res.headers, [
'Connection',
'Server',
'Date',
'Content-Type',
'Cookie',
'Location',
'Accept-Ranges',
'Last-Modified'
]);
headers["wtv-http-proxy"] = true;
headers["wtv-trusted"] = false;
if (typeof res.headers['Content-Type'] === 'string' && res.headers['Content-Type'].startsWith("text")) {
// Get the original URL for relative link fixing
const originalUrl = request_headers.request.split(' ')[1];
// Transform HTML content for WebTV compatibility
if (res.headers['Content-Type'].includes('html') &&
this.minisrv_config.services[request_type]?.use_minifying_proxy === true) {
try {
const WTVMinifyingProxy = require('./WTVMinifyingProxy.js');
const proxy = new WTVMinifyingProxy(this.minisrv_config);
let htmlContent = Buffer.concat(data).toString();
// Apply WebTV-specific transformations
const transformOptions = {
removeImages: this.minisrv_config.services[request_type]?.remove_images || false,
maxImageWidth: this.minisrv_config.services[request_type]?.max_image_width || 400,
simplifyTables: this.minisrv_config.services[request_type]?.simplify_tables !== false,
maxWidth: this.minisrv_config.services[request_type]?.max_width || 544,
preserveJellyScript: this.minisrv_config.services[request_type]?.preserve_jellyscript !== false,
jellyScriptMaxSize: this.minisrv_config.services[request_type]?.jellyscript_max_size || 8192
};
htmlContent = proxy.transformForWebTV(htmlContent, originalUrl, transformOptions);
data = [Buffer.from(htmlContent)];
if (this.minisrv_config.config.verbosity >= 3) {
console.log(` * HTML transformed for WebTV compatibility (${originalUrl})`);
}
} catch (err) {
console.warn(` * HTML transformation failed: ${err.message}`);
}
}
if (request_type !== "http" && request_type !== "https") {
// replace http and https links on non http/https protocol (for proto:// for example)
const data_t = Buffer.concat(data).toString().replaceAll("http://", request_type + "://").replaceAll("https://", request_type + "://");
data = [Buffer.from(data_t)]
}
}
// if Connection: close header, set our internal variable to close the socket
if (headers['Connection']) {
if (headers['Connection'].toLowerCase().includes('close')) {
headers["wtv-connection-close"] = true;
}
}
// if a wtv-explaination is defined for an error code (except 200), define the header here to
// show the 'Explain' button on the client error ShowAlert
if (this.minisrv_config.services[request_type]['wtv-explanation']) {
if (this.minisrv_config.services[request_type]['wtv-explanation'][res.statusCode]) {
headers['wtv-explanation-url'] = this.minisrv_config.services[request_type]['wtv-explanation'][res.statusCode];
}
} else if (this.minisrv_config.services['http']['wtv-explanation']) {
if (this.minisrv_config.services['http']['wtv-explanation'][res.statusCode]) {
headers['wtv-explanation-url'] = this.minisrv_config.services['http']['wtv-explanation'][res.statusCode];
}
}
let data_hex = Buffer.concat(data).toString('hex');
if (data_hex.startsWith("0d0a0d0a")) data_hex = data_hex.slice(8);
if (data_hex.startsWith("0a0d0a")) data_hex = data_hex.slice(6);
if (data_hex.startsWith("0a0a")) data_hex = data_hex.slice(4);
this.sendToClient(socket, headers, Buffer.from(data_hex, 'hex'));
}
}
module.exports = WTVHTTP;

2
zefie_wtvp_minisrv/includes/classes/WTVPNM.js
View File

@@ -30,7 +30,7 @@ class WTVPNM {
wtvshared = null;
sessions = new Map();
constructor(minisrv_config, service_name = 'pnm') {
constructor(...[minisrv_config, service_name]) {
this.minisrv_config = minisrv_config;
this.service_name = service_name;
this.service_config = minisrv_config.services[service_name] || {};

32
zefie_wtvp_minisrv/includes/config.json
View File

@@ -312,9 +312,13 @@
"external_proxy_host": "127.0.0.1", // IP address or hostname of proxy
"external_proxy_port": 1080, // Port of proxy
"flags": "0x00000001",
"max_response_size": 16 // Megabytes
"max_response_size": 16, // Megabytes
"disallow_no_slash": true,
"handler_module": "WTVHTTP",
"handler_extra_vars": ["http", "sendToClient"]
},
"https": {
// https upstream
"port": 1650,
"connections": 3,
"use_external_proxy": false, // use an external proxy (WebONE or some other minifying proxy is recommended)
@@ -322,9 +326,14 @@
"external_proxy_host": "127.0.0.1", // IP address or hostname of proxy
"external_proxy_port": 1080, // Port of proxy
"flags": "0x00000001",
"max_response_size": 16 // Megabytes
"max_response_size": 16, // Megabytes
"disallow_no_slash": true,
"handler_module": "WTVHTTP",
"handler_extra_vars": ["http", "sendToClient"],
"allow_self_signed_ssl": true // If true, will allow self-signed SSL certificates via the proxy.
},
"proto": {
// ProtoWeb Proxy
"port": 1650,
"connections": 3,
"use_external_proxy": true,
@@ -334,7 +343,21 @@
"external_proxy_port": 7851,
"external_proxy_is_http1": true,
"flags": "0x00000001",
"max_response_size": 16 // Megabytes
"max_response_size": 16, // Megabytes
"disallow_no_slash": true,
"handler_module": "WTVHTTP",
"handler_extra_vars": ["http", "sendToClient"]
},
"gopher": {
// Gopher processor
"port": 1651,
"connections": 3,
"allow_double_slash": true,
"flags": "0x00000001",
"uses_service_vault": false, // For custom services that use modules instead of service vaults
"disallow_no_slash": true,
"log_raw_gopher": false, // set to true to log raw gopher responses to the console for debugging
"handler_module": "WTVGopher"
},
"wtv-passport": {
// wtv-passport (for messenger)
@@ -404,7 +427,8 @@
"descriptor_after_hello_ms": 85,
"burst_prestart_ms": 5000,
"debug": false,
"allow_indexing": true
"allow_indexing": true,
"handler_module": "WTVPNM"
}
},
"favorites": {

2
zefie_wtvp_minisrv/ra_win/build.cmd
View File

@@ -1,4 +1,4 @@
@echo OFF
cl rpcli.cpp /O2 /MT /Fe:rpcli.exe
del rpcli.obj prct3260.thl
del rpcli.obj prct3260.tlh