diff --git a/zefie_wtvp_minisrv/app.js b/zefie_wtvp_minisrv/app.js
index 6510aaa4..60e1999d 100644
--- a/zefie_wtvp_minisrv/app.js
+++ b/zefie_wtvp_minisrv/app.js
@@ -11,8 +11,7 @@ const nunjucks = require('nunjucks');
const zlib = require('zlib');
const {serialize, unserialize} = require('php-serialize');
const {spawn} = require('child_process');
-const http = require('follow-redirects').http
-const https = require('follow-redirects').https
+const http = require('follow-redirects').http;
const httpx = require(classPath + "/HTTPX.js");
const { URL } = require('url');
const net = require('net');
@@ -32,6 +31,7 @@ const WTVPNM = require(classPath + "/WTVPNM.js");
const vm = require('vm');
const debug = require('debug')('app');
const express = require('express');
+let handlerModules = [];
let wtvnewsserver = null;
const protocolServers = [];
@@ -304,8 +304,8 @@ const runScriptInVM = function (script_data, user_contextObj = {}, privileged =
// Our modules
"wtvmime": wtvmime,
- "http": http,
- "https": https,
+ "http": require('follow-redirects').http,
+ "https": require('follow-redirects').https,
"util": util,
"sharp": sharp,
"nunjucks": nunjucks,
@@ -984,7 +984,11 @@ async function processURL(socket, request_headers, pc_services = false) {
let shortURL, headers, data, service_name;
let original_service_name = "";
let shared_romcache = null;
- let allow_double_slash = false, enable_multi_query = false, use_external_proxy = false;
+ let allow_double_slash = false;
+ let enable_multi_query = false;
+ let use_external_proxy = false;
+ let disallow_no_slash = false;
+ let uses_service_vault = true;
request_headers.query = {};
if (request_headers.request_url) {
@@ -993,6 +997,8 @@ async function processURL(socket, request_headers, pc_services = false) {
allow_double_slash = minisrv_config.services[service_name].allow_double_slash || false;
enable_multi_query = minisrv_config.services[service_name].enable_multi_query || false;
use_external_proxy = minisrv_config.services[service_name].use_external_proxy || false;
+ disallow_no_slash = minisrv_config.services[service_name].disallow_no_slash || false;
+ uses_service_vault = (minisrv_config.services[service_name].uses_service_vault === false) ? false : true;
}
if (pc_services) {
original_service_name = socket.service_name; // store service name
@@ -1078,7 +1084,7 @@ async function processURL(socket, request_headers, pc_services = false) {
}
}
- if ((!shortURL.startsWith("http") && !shortURL.startsWith("ftp") && shortURL.includes(":") && !shortURL.includes(":/"))) {
+ if ((!disallow_no_slash && shortURL.includes(":") && !shortURL.includes(":/"))) {
// Apparently it is within WTVP spec to accept urls without a slash (eg wtv-home:home)
// Here, we just reassemble the request URL as if it was a proper URL (eg wtv-home:/home)
// we will allow this on any service except http(s) and ftp
@@ -1143,7 +1149,7 @@ minisrv-no-mail-count: true`;
else console.log(" * " + ((ssl) ? "SSL " : "") + "PC request on service " + original_service_name + " (Service Vault " + service_name + ") for " + request_headers.request_url, 'on', socket.id);
}
- if ((shortURL.includes(':/')) && (!shortURL.includes('://') || (shortURL.includes('://') && allow_double_slash))) {
+ if ((shortURL.includes(':/')) && (!shortURL.includes('://') || (shortURL.includes('://') && allow_double_slash) && uses_service_vault)) {
let ssid = socket.ssid;
if (ssid === null) {
// prevent possible injection attacks via malformed SSID and filesystem SessionStore
@@ -1185,8 +1191,10 @@ minisrv-no-mail-count: true`;
socket_sessions[socket.id].request_headers = request_headers;
processPath(socket, urlToPath, request_headers, service_name, shared_romcache, pc_services);
- } else if (shortURL.includes('http://') || shortURL.includes('https://') || (use_external_proxy === true && shortURL.includes(service_name + "://")) && !pc_services) {
- doHTTPProxy(socket, request_headers);
+ } else if (handlerModules["wtvhttp"] && ((shortURL.includes('http://') || shortURL.includes('https://')) || (use_external_proxy === true && shortURL.includes(service_name + "://")) && !pc_services)) {
+ handlerModules["wtvhttp"].doHTTPProxy(socket, request_headers);
+ } else if (handlerModules["wtvgopher"] && shortURL.startsWith("gopher://")) {
+ handlerModules["wtvgopher"].handleGopherRequest(socket, request_headers, wtvshared, sendToClient);
} else if (shortURL.startsWith('ftp://')) {
if (minisrv_config.config.debug_flags.show_headers) console.debug(" * Incoming FTP request on WTVP socket ID", socket.id, await wtvshared.decodePostData(await wtvshared.filterRequestLog(await wtvshared.filterSSID(request_headers))));
const wtvftp = new WTVFTP(wtvshared, sendToClient);
@@ -1230,250 +1238,6 @@ minisrv-no-mail-count: true`;
}
}
-function handleProxy(socket, request_type, request_headers, res, data) {
- console.log(` * Proxy Request ${request_type.toUpperCase()} ${res.statusCode} for ${request_headers.request}`)
- // an http response error is not a request error, and will come here under the 'end' event rather than an 'error' event.
- switch (res.statusCode) {
- case 404:
- res.headers.Status = res.statusCode + " The publisher can’t find the page requested.";
- break;
-
- case 401:
- case 403:
- res.headers.Status = res.statusCode + " The publisher of that page has not authorized you to use it.";
- break;
-
- case 500:
- res.headers.Status = res.statusCode + " The publisher of that page can’t be reached.";
- break;
-
- default:
- res.headers.Status = res.statusCode + " " + res.statusMessage;
- break;
- }
-
- if (res.headers['Content-type']) {
- res.headers['Content-Type'] = res.headers['Content-type'];
- delete (res.headers['Content-type'])
- }
-
- if (res.headers['content-type']) {
- res.headers['Content-Type'] = res.headers['content-type'];
- delete (res.headers['content-type'])
- }
-
- // header pass-through whitelist, case insensitive comparsion to server, however, you should
- // specify the header case as you intend for the client
- const headers = wtvshared.stripHeaders(res.headers, [
- 'Connection',
- 'Server',
- 'Date',
- 'Content-Type',
- 'Cookie',
- 'Location',
- 'Accept-Ranges',
- 'Last-Modified'
- ]);
- headers["wtv-http-proxy"] = true;
- headers["wtv-trusted"] = false;
-
- if (typeof res.headers['Content-Type'] === 'string' && res.headers['Content-Type'].startsWith("text")) {
- // Get the original URL for relative link fixing
- const originalUrl = request_headers.request.split(' ')[1];
-
- // Transform HTML content for WebTV compatibility
- if (res.headers['Content-Type'].includes('html') &&
- minisrv_config.services[request_type]?.use_minifying_proxy === true) {
- try {
- const WTVMinifyingProxy = require('./includes/classes/WTVMinifyingProxy.js');
- const proxy = new WTVMinifyingProxy(minisrv_config);
-
- let htmlContent = Buffer.concat(data).toString();
-
- // Apply WebTV-specific transformations
- const transformOptions = {
- removeImages: minisrv_config.services[request_type]?.remove_images || false,
- maxImageWidth: minisrv_config.services[request_type]?.max_image_width || 400,
- simplifyTables: minisrv_config.services[request_type]?.simplify_tables !== false,
- maxWidth: minisrv_config.services[request_type]?.max_width || 544,
- preserveJellyScript: minisrv_config.services[request_type]?.preserve_jellyscript !== false,
- jellyScriptMaxSize: minisrv_config.services[request_type]?.jellyscript_max_size || 8192
- };
-
- htmlContent = proxy.transformForWebTV(htmlContent, originalUrl, transformOptions);
- data = [Buffer.from(htmlContent)];
-
- if (minisrv_config.config.verbosity >= 3) {
- console.log(` * HTML transformed for WebTV compatibility (${originalUrl})`);
- }
- } catch (err) {
- console.warn(` * HTML transformation failed: ${err.message}`);
- }
- }
-
- if (request_type !== "http" && request_type !== "https") {
- // replace http and https links on non http/https protocol (for proto:// for example)
- const data_t = Buffer.concat(data).toString().replaceAll("http://", request_type + "://").replaceAll("https://", request_type + "://");
- data = [Buffer.from(data_t)]
- }
- }
-
- // if Connection: close header, set our internal variable to close the socket
- if (headers['Connection']) {
- if (headers['Connection'].toLowerCase().includes('close')) {
- headers["wtv-connection-close"] = true;
- }
- }
-
- // if a wtv-explaination is defined for an error code (except 200), define the header here to
- // show the 'Explain' button on the client error ShowAlert
- if (minisrv_config.services['http']['wtv-explanation']) {
- if (minisrv_config.services['http']['wtv-explanation'][res.statusCode]) {
- headers['wtv-explanation-url'] = minisrv_config.services['http']['wtv-explanation'][res.statusCode];
- }
- }
- let data_hex = Buffer.concat(data).toString('hex');
- if (data_hex.startsWith("0d0a0d0a")) data_hex = data_hex.slice(8);
- if (data_hex.startsWith("0a0d0a")) data_hex = data_hex.slice(6);
- if (data_hex.startsWith("0a0a")) data_hex = data_hex.slice(4);
- sendToClient(socket, headers, Buffer.from(data_hex, 'hex'));
-}
-
-async function doHTTPProxy(socket, request_headers) {
- // detect protocol name
- const idx = request_headers.request_url.indexOf('/') - 1;
-
- const request_type = request_headers.request_url.slice(0, idx);
- if (minisrv_config.config.debug_flags.show_headers) console.debug(request_type.toUpperCase() + " Proxy: Client Request Headers on socket ID", socket.id, (await wtvshared.decodePostData(await wtvshared.filterRequestLog(await wtvshared.filterSSID(request_headers)))));
- else debug(request_type.toUpperCase() + " Proxy: Client Request Headers on socket ID", socket.id, (await wtvshared.decodePostData(await wtvshared.filterRequestLog(await wtvshared.filterSSID(request_headers)))));
- let proxy_agent;
-
- switch (request_type) {
- case "https":
- proxy_agent = https;
- break;
- case "http":
- case "proto":
- proxy_agent = http;
- break;
- }
-
- const request_data = [];
- const data = [];
-
- request_data.method = request_headers.request.split(' ')[0];
- const request_url_split = request_headers.request.split(' ')[1].split('/');
- request_data.host = request_url_split[2];
- if (request_data.host.indexOf(':') > 0) {
- request_data.port = request_data.host.split(':')[1];
- request_data.host = request_data.host.split(':')[0];
- } else {
- if (request_type === "https") request_data.port = 443;
- else request_data.port = 80;
- }
- for (let i = 0; i < 3; i++) request_url_split.shift();
- request_data.path = "/" + request_url_split.join('/');
- if (request_data.method && request_data.host && request_data.path) {
-
- const options = {
- host: request_data.host,
- port: request_data.port,
- path: request_data.path,
- method: request_data.method,
- followAllRedirects: true,
- headers: {
- "User-Agent": request_headers["User-Agent"] || "WebTV",
- "Connection": "Keep-Alive"
- }
- }
-
- // RFC7239
- if (socket.remoteAddress !== "127.0.0.1") {
- options.headers["X-Forwarded-For"] = socket.remoteAddress;
- }
-
- if (request_headers.post_data) {
- if (request_headers["Content-type"]) options.headers["Content-type"] = request_headers["Content-type"];
- if (request_headers["Content-length"]) options.headers["Content-length"] = request_headers["Content-length"];
- }
-
- if (minisrv_config.services[request_type].use_external_proxy && minisrv_config.services[request_type].external_proxy_port) {
- // configure connection to an external proxy
- if (minisrv_config.services[request_type].external_proxy_is_socks) {
- // configure connection to remote socks proxy
- const { SocksProxyAgent }= require('socks-proxy-agent');
- options.agent = new SocksProxyAgent("socks://" + (minisrv_config.services[request_type].external_proxy_host || "127.0.0.1") + ":" + minisrv_config.services[request_type].external_proxy_port);
- options.agents = {
- "http": options.agent,
- "https": options.agent
- }
- } else {
- // configure connection to remote http proxy
- proxy_agent = http;
- options.host = minisrv_config.services[request_type].external_proxy_host;
- options.port = minisrv_config.services[request_type].external_proxy_port;
- options.path = request_headers.request.split(' ')[1];
- options.headers.Host = request_data.host + ":" + request_data.port;
- if (minisrv_config.services[request_type].replace_protocol) {
- options.path = options.path.replace(request_type, minisrv_config.services[request_type].replace_protocol);
- }
- }
- if (minisrv_config.services[request_type].external_proxy_is_http1) {
- options.insecureHTTPParser = true;
- options.headers.Connection = 'close'
- }
- }
- const req = proxy_agent.request(options, function (res) {
- let total_data = 0;
-
- res.on('data', d => {
- data.push(d);
- total_data += d.length;
- if (total_data > 1024 * 1024 * parseFloat(minisrv_config.services[request_type].max_response_size || 16)) {
- console.warn(` * Response data exceeded ${minisrv_config.services[request_type].max_response_size || 16}MB limit, destroying...`);
- res.destroy();
- const errpage = wtvshared.doErrorPage(400, "The item chosen is too large to be used.");
- sendToClient(socket, errpage[0], errpage[1]);
- }
- })
-
- res.on('error', function (err) {
- // hack for Protoweb ECONNRESET
- if (minisrv_config.services[request_type].external_proxy_is_http1 && data.length > 0) {
- handleProxy(socket, request_type, request_headers, res, data);
- } else {
- console.error(" * Unhandled Proxy Request Error:", err);
- }
- });
-
- res.on('end', function () {
- // For when http proxies behave correctly
- if (!minisrv_config.services[request_type].external_proxy_is_http1 || data.length > 0) {
- handleProxy(socket, request_type, request_headers, res, data);
- }
- });
- }).on('error', function (err) {
- // severe errors, such as unable to connect.
- if (err.code === "ENOTFOUND" || err.message.indexOf("HostUnreachable") > 0) {
- const errpage = wtvshared.doErrorPage(400, `The publisher ${request_data.host} is unknown.`);
- sendToClient(socket, errpage[0], errpage[1]);
- } else {
- console.error(" * Unhandled Proxy Request Error:", err);
- const errpage = wtvshared.doErrorPage(400);
- sendToClient(socket, errpage[0], errpage[1]);
- }
-
- });
- if (request_headers.post_data) {
- req.write(Buffer.from(request_headers.post_data.toString(CryptoJS.enc.Hex), 'hex'), function () {
- req.end();
- });
- } else {
- req.end();
- }
- }
-}
-
async function sendToClient(socket, headers_obj, data = null) {
let headers = "";
let content_length = 0;
@@ -2463,9 +2227,27 @@ const service_ip = minisrv_config.config.service_ip;
Object.keys(minisrv_config.services).forEach(function (k) {
if (typeof(minisrv_config.services[k]) === 'function') return;
if (configureService(k, minisrv_config.services[k], true)) {
+ let loadedModule = false;
+ if (minisrv_config.services[k].handler_module) {
+ try {
+ if (!handlerModules[minisrv_config.services[k].handler_module + "_main"]) {
+ handlerModules[minisrv_config.services[k].handler_module + "_main"] = require(classPath + "/" + minisrv_config.services[k].handler_module + ".js");
+ var args = [];
+ for (let i = 0; i < (minisrv_config.services[k].handler_extra_vars || []).length; i++) {
+ let extraVar = eval(minisrv_config.services[k].handler_extra_vars[i]);
+ args.push(extraVar);
+ }
+ const constructorArgs = [minisrv_config, k, ...args];
+ handlerModules[minisrv_config.services[k].handler_module.toLowerCase()] = new handlerModules[minisrv_config.services[k].handler_module + "_main"](...constructorArgs);
+ }
+ loadedModule = true;
+ } catch (e) {
+ console.error(" # Failed to load handler module for service", k, "module:", minisrv_config.services[k].handler_module, e);
+ }
+ }
const using_tls = (minisrv_config.services[k].pc_services && minisrv_config.services[k].https_cert && minisrv_config.services[k].use_https) ? true : false;
const protocol = (minisrv_config.services[k].protocol_handler) ? minisrv_config.services[k].protocol_handler.toUpperCase() : (minisrv_config.services[k].pc_services) ? "HTTP" : "WTVP";
- console.log(" * Configured Service:", k, "on Port", minisrv_config.services[k].port, "- Service Host:", minisrv_config.services[k].host + ((using_tls) ? " (TLS)" : ""), "- Protocol:", protocol);
+ console.log(" * Configured Service:", k, "on Port", minisrv_config.services[k].port, "- Service Host:", minisrv_config.services[k].host + ((using_tls) ? " (TLS)" : ""), "- Protocol:", protocol, (loadedModule) ? "- Handler Module: " + minisrv_config.services[k].handler_module : "");
if (minisrv_config.services[k].local_nntp_enabled && minisrv_config.services[k].local_nntp_port) {
if (!wtvnewsserver) {
@@ -2548,9 +2330,8 @@ Object.keys(minisrv_config.services).forEach((service_name) => {
if (!service || service.disabled || !service.port) return;
if (service.protocol_handler === 'pnm') {
try {
- const pnmServer = new WTVPNM(minisrv_config, service_name);
- pnmServer.listen(service.port, minisrv_config.config.bind_ip);
- protocolServers.push(pnmServer);
+ handlerModules['wtvpnm'].listen(service.port, minisrv_config.config.bind_ip);
+ protocolServers.push(handlerModules['wtvpnm']);
} catch (e) {
throw ("Could not bind PNM protocol handler to port " + service.port + " on " + minisrv_config.config.bind_ip + ": " + e.toString());
}
diff --git a/zefie_wtvp_minisrv/includes/classes/WTVGopher.js b/zefie_wtvp_minisrv/includes/classes/WTVGopher.js
new file mode 100644
index 00000000..f3c3f06f
--- /dev/null
+++ b/zefie_wtvp_minisrv/includes/classes/WTVGopher.js
@@ -0,0 +1,259 @@
+const WTVMime = require('./WTVMime.js');
+const net = require('net');
+
+class WTVGopher {
+ // Adapted from WebTV Redialed's Gopher support
+ constructor(...[minisrv_config, service_name]) {
+ this.minisrv_config = minisrv_config;
+ this.wtvmime = new WTVMime(minisrv_config);
+ this.logGopher = minisrv_config.services[service_name].log_raw_gopher || false;
+ }
+
+ looksLikeMenu(gopherData) {
+ const lines = gopherData.split(/\r?\n/);
+
+ let checked = 0;
+ let menuLines = 0;
+
+ for (const line of lines) {
+ if (!line || line === ".") continue;
+
+ checked++;
+ let typeOffset = 0;
+ let type = " ";
+ while ((type === " " || type == "\t") && typeOffset <= 10) {
+ type = line[typeOffset];
+ typeOffset++;
+ }
+ const rest = line.slice(1);
+
+ if (
+ rest.includes("\t") &&
+ rest.split("\t").length >= 3 &&
+ /^[0-9A-Za-z+gIihs]$/.test(type)
+ ) {
+ menuLines++;
+ }
+
+ if (checked >= 5) break;
+ }
+
+ return menuLines >= 2 || (lines.length <= 2 && menuLines == 1);
+ }
+
+ processGopherData(gopherData) {
+ // currently looking at textfile, don't process into HTML
+ if (!this.looksLikeMenu(gopherData)) {
+ return `${gopherData}`;
+ }
+
+ // okay, we're not a textfile, now do the menu shit
+ let pageTitle = "Gopher Menu"
+ const lines = gopherData.split("\r\n");
+
+ let html = "";
+
+ for (const line of lines) {
+ if (!line || line === ".") continue;
+
+ const type = line[0];
+ const parts = line.slice(1).split("\t");
+
+ const text = parts[0] || "";
+ const selector = parts[1];
+ const host = parts[2];
+ const port = parts[3] || 70;
+ var url = `gopher://${host}:${port}${selector}`;
+
+ // determine page title from first line
+ const firstline = line[0].slice(1).trim();
+
+ if (line[0] === "i" && firstline.length > 0) {
+ pageTitle = line.slice(1).trim();
+ html = `${pageTitle}\n`;
+ } else if (pageTitle === "Gopher Menu") {
+ for (const line of lines) {
+ if (!line || line === ".") continue;
+
+ let typeOffset = 0;
+ let type = " ";
+ while ((type === " " || type == "\t") && typeOffset <= 10) {
+ type = line[typeOffset];
+ typeOffset++;
+ }
+ const parts = line.slice(1).split("\t");
+ const text = parts[0]?.trim();
+
+ if (text && text.length > 0) {
+ pageTitle = text;
+ html = `${pageTitle}\n`;
+ break;
+ }
+ }
+ }
+ switch (type) {
+ case "i": // informational / "just text"
+ html += `${text}\n`;
+ break;
+
+ case "0": // text file
+ case "1": // directory
+ html += `${text}\n`;
+ break;
+ case "3": // error, otherwise just plain text
+ html += `${text}
\n`;
+ case "h": // HTML link
+ if (selector?.startsWith("URL:")) {
+ const httpUrl = selector.slice(4);
+ html += `${text}\n`;
+ }
+ break;
+
+ case "7": // search
+ html += ``;
+ break;
+ case "g":
+ case "I":
+ case "p":
+ url = `gopher://${host}:${port}${selector}?type=${type}`;
+ html += `${text}\n`;
+ break;
+
+ default:
+ html += `${text} (unsupported type ${type})\n`;
+ }
+ }
+
+ html += "";
+ return html;
+ }
+
+ async handleGopherRequest(socket, request_headers, wtvshared, sendToClient) {
+ if (this.minisrv_config.config.debug_flags.show_headers) {
+ console.log("Gopher: Client Request on socket ID",
+ socket.id,
+ await wtvshared.decodePostData(
+ wtvshared.filterRequestLog(wtvshared.filterSSID(request_headers))
+ ));
+ }
+
+ // crlf for sending at the end of a request
+ const crlf = "0D0A"
+ const crlf_bytes = Buffer.from(crlf, 'hex');
+ // chunk stuff for gopher-to-html conversion
+ let chunks = [];
+
+ var request_data = new Array();
+ request_data.method = request_headers.request.split(' ')[0];
+
+ const rawUrl = decodeURIComponent(request_headers.request.split(' ')[1]).replaceAll('\\', '/');
+ const [pathPart, queryPart] = rawUrl.split('?');
+ var request_url_split = pathPart.split('/');
+
+ let queryParams = {};
+ if (queryPart) {
+ for (const kv of queryPart.split('&')) {
+ const [k, v] = kv.split('=');
+ queryParams[k] = decodeURIComponent(v || "");
+ }
+ }
+
+ request_data.host = request_url_split[2];
+ if (request_data.host.indexOf(':') > 0) {
+ request_data.port = request_data.host.split(':')[1];
+ request_data.host = request_data.host.split(':')[0];
+ } else {
+ request_data.port = 70;
+ }
+
+ for (var i = 0; i < 3; i++) request_url_split.shift();
+ request_data.path = "/" + request_url_split.join('/');
+ // vars for determining if a link is an image
+ const imageTypes = ["g", "I", "p"];
+ let requestType = null;
+ if (queryParams.type && imageTypes.includes(queryParams.type)) {
+ requestType = queryParams.type;
+ }
+ const isImageDownload = !!requestType;
+
+ const client = new net.Socket();
+ client.setTimeout(3000);
+
+ // make the initial request to the server
+ client.connect(request_data.port, request_data.host, () => {
+ let gopherRequest = "";
+
+ // if user requested path
+ if (request_data.path.length >= 2) {
+ gopherRequest = request_data.path;
+ }
+
+ // if user requested type 7 (search)
+ if (queryParams.q) {
+ const query = queryParams.q.replace(/\+/g, ' ');
+ gopherRequest += "\t" + query;
+ }
+
+ client.write(gopherRequest + crlf_bytes);
+ });
+
+ // "holy shit we got data guys"
+ client.on("data", chunk => {
+ chunks.push(chunk);
+ });
+
+ // datastream end, time to process it
+ client.on("end", () => {
+ const gopherData = Buffer.concat(chunks).toString("utf-8");
+ if (this.logGopher) {
+ console.log("Gopher: Data received from server for socket ID", socket.id);
+ console.log("Gopher: Data length:", Buffer.concat(chunks).length);
+ console.log("isImageDownload:", isImageDownload);
+ console.log("Gopher Data:\n", gopherData);
+ }
+
+ // are we downloading an image?
+ if (isImageDownload) {
+ const imageData = Buffer.concat(chunks);
+ const mimetype = this.wtvmime.detectMimeTypeFromBuffer(imageData);
+
+ const headers = {
+ "Status": "200 OK",
+ "Content-Type": mimetype
+ }
+
+ sendToClient(socket, headers, imageData);
+ return;
+ } else {
+ // convert gophermap to html
+ const htmlData = this.processGopherData(gopherData);
+ // since gopher doesn't exactly have "headers" and by this point we're probably already fine to just say it's okay, we're just sending back the bare minimum to prevent screaming
+ const headers = {
+ "Status": "200 OK",
+ "Content-Type": "text/html"
+ }
+ sendToClient(socket, headers, htmlData);
+ }
+ });
+
+ // blew up?
+ // todo: figure out what error actually looks like and if appropriate to send to client (or just "Connection failed" or smth)
+ client.on('error', (err) => {
+ console.error('Gopher error: ' + err);
+ let friendlyErr = err.toString();
+ if (friendlyErr.includes('ETIMEDOUT')) {
+ friendlyErr = "Connection timed out";
+ } else if (friendlyErr.includes('ECONNREFUSED')) {
+ friendlyErr = "Connection refused";
+ } else if (friendlyErr.includes('ENOTFOUND')) {
+ friendlyErr = "Host not found";
+ }
+ sendToClient(socket, {"Status": "400 Gopher Error: " + friendlyErr}, friendlyErr);
+ });
+ }
+}
+
+module.exports = WTVGopher;
\ No newline at end of file
diff --git a/zefie_wtvp_minisrv/includes/classes/WTVHTTP.js b/zefie_wtvp_minisrv/includes/classes/WTVHTTP.js
new file mode 100644
index 00000000..b20250e7
--- /dev/null
+++ b/zefie_wtvp_minisrv/includes/classes/WTVHTTP.js
@@ -0,0 +1,266 @@
+const {WTVShared, clientShowAlert} = require('./WTVShared.js');
+
+class WTVHTTP {
+ constructor(...[minisrv_config, service_name, http, sendToClient]) {
+ this.minisrv_config = minisrv_config;
+ this.service_name = service_name;
+ this.wtvshared = new WTVShared(minisrv_config);
+ this.sendToClient = sendToClient;
+ this.http = http;
+ this.https = require('follow-redirects').https
+ this.proxy_agent = null;
+ }
+
+ async doHTTPProxy(socket, request_headers) {
+ // detect protocol name
+ const idx = request_headers.request_url.indexOf('/') - 1;
+
+ const request_type = request_headers.request_url.slice(0, idx);
+ if (this.minisrv_config.config.debug_flags.show_headers) console.debug(request_type.toUpperCase() + " Proxy: Client Request Headers on socket ID", socket.id, (await this.wtvshared.decodePostData(await this.wtvshared.filterRequestLog(await this.wtvshared.filterSSID(request_headers)))));
+ else debug(request_type.toUpperCase() + " Proxy: Client Request Headers on socket ID", socket.id, (await this.wtvshared.decodePostData(await this.wtvshared.filterRequestLog(await this.wtvshared.filterSSID(request_headers)))));
+
+ switch (request_type) {
+ case "https":
+ this.proxy_agent = this.https;
+ break;
+ default:
+ this.proxy_agent = this.http;
+ break;
+ }
+
+ const request_data = [];
+ const data = [];
+
+ request_data.method = request_headers.request.split(' ')[0];
+ const request_url_split = request_headers.request.split(' ')[1].split('/');
+ request_data.host = request_url_split[2];
+ if (request_data.host.indexOf(':') > 0) {
+ request_data.port = request_data.host.split(':')[1];
+ request_data.host = request_data.host.split(':')[0];
+ } else {
+ if (request_type === "https") request_data.port = 443;
+ else request_data.port = 80;
+ }
+ for (let i = 0; i < 3; i++) request_url_split.shift();
+ request_data.path = "/" + request_url_split.join('/');
+ if (request_data.method && request_data.host && request_data.path) {
+
+ const options = {
+ host: request_data.host,
+ port: request_data.port,
+ path: request_data.path,
+ method: request_data.method,
+ followAllRedirects: true,
+ headers: {
+ "User-Agent": request_headers["User-Agent"] || "WebTV",
+ "Connection": "Keep-Alive"
+ }
+ }
+
+ // RFC7239
+ if (socket.remoteAddress !== "127.0.0.1") {
+ options.headers["X-Forwarded-For"] = socket.remoteAddress;
+ }
+
+ if (request_headers.post_data) {
+ if (request_headers["Content-type"]) options.headers["Content-type"] = request_headers["Content-type"];
+ if (request_headers["Content-length"]) options.headers["Content-length"] = request_headers["Content-length"];
+ }
+
+ if (request_type == "https" && this.minisrv_config.services[request_type].allow_self_signed_ssl) {
+ options.rejectUnauthorized = false;
+ }
+
+ if (this.minisrv_config.services[request_type].use_external_proxy && minisrv_config.services[request_type].external_proxy_port) {
+ // configure connection to an external proxy
+ if (this.minisrv_config.services[request_type].external_proxy_is_socks) {
+ // configure connection to remote socks proxy
+ const { SocksProxyAgent }= require('socks-proxy-agent');
+ options.agent = new SocksProxyAgent("socks://" + (minisrv_config.services[request_type].external_proxy_host || "127.0.0.1") + ":" + minisrv_config.services[request_type].external_proxy_port);
+ options.agents = {
+ "http": options.agent,
+ "https": options.agent
+ }
+ } else {
+ // configure connection to remote http proxy
+ this.proxy_agent = this.http;
+ options.host = this.minisrv_config.services[request_type].external_proxy_host;
+ options.port = this.minisrv_config.services[request_type].external_proxy_port;
+ options.path = request_headers.request.split(' ')[1];
+ options.headers.Host = request_data.host + ":" + request_data.port;
+ if (this.minisrv_config.services[request_type].replace_protocol) {
+ options.path = options.path.replace(request_type, this.minisrv_config.services[request_type].replace_protocol);
+ }
+ }
+ if (this.minisrv_config.services[request_type].external_proxy_is_http1) {
+ options.insecureHTTPParser = true;
+ options.headers.Connection = 'close'
+ }
+ }
+ const req = this.proxy_agent.request(options, (res) => {
+ let total_data = 0;
+
+ res.on('data', d => {
+ data.push(d);
+ total_data += d.length;
+ if (total_data > 1024 * 1024 * parseFloat(this.minisrv_config.services[request_type].max_response_size || 16)) {
+ console.warn(` * Response data exceeded ${this.minisrv_config.services[request_type].max_response_size || 16}MB limit, destroying...`);
+ res.destroy();
+ const errpage = this.wtvshared.doErrorPage(400, "The item chosen is too large to be used.");
+ this.sendToClient(socket, errpage[0], errpage[1]);
+ }
+ })
+
+ res.on('error', (err) => {
+ // hack for Protoweb ECONNRESET
+ if (this.minisrv_config.services[request_type].external_proxy_is_http1 && data.length > 0) {
+ this.handleProxy(socket, request_type, request_headers, res, data);
+ } else {
+ console.error(" * Unhandled Proxy Request Error:", err);
+ }
+ });
+
+ res.on('end', () => {
+ // For when http proxies behave correctly
+ if (!this.minisrv_config.services[request_type].external_proxy_is_http1 || data.length > 0) {
+ this.handleProxy(socket, request_type, request_headers, res, data);
+ }
+ });
+ }).on('error', (err) => {
+ // severe errors, such as unable to connect.
+ if (err.code === "ENOTFOUND" || err.message.indexOf("HostUnreachable") > 0) {
+ const errpage = this.wtvshared.doErrorPage(400, `The publisher ${request_data.host} is unknown.`);
+ this.sendToClient(socket, errpage[0], errpage[1]);
+ } else {
+ console.error(" * Unhandled Proxy Request Error:", err);
+ const errpage = this.wtvshared.doErrorPage(400);
+ this.sendToClient(socket, errpage[0], errpage[1]);
+ }
+
+ });
+ if (request_headers.post_data) {
+ req.write(Buffer.from(request_headers.post_data.toString(CryptoJS.enc.Hex), 'hex'), () => {
+ req.end();
+ });
+ } else {
+ req.end();
+ }
+ }
+ }
+
+ handleProxy(socket, request_type, request_headers, res, data) {
+ console.log(` * Proxy Request ${request_type.toUpperCase()} ${res.statusCode} for ${request_headers.request}`)
+ // an http response error is not a request error, and will come here under the 'end' event rather than an 'error' event.
+ switch (res.statusCode) {
+ case 404:
+ res.headers.Status = res.statusCode + " The publisher can’t find the page requested.";
+ break;
+
+ case 401:
+ case 403:
+ res.headers.Status = res.statusCode + " The publisher of that page has not authorized you to use it.";
+ break;
+
+ case 500:
+ res.headers.Status = res.statusCode + " The publisher of that page can’t be reached.";
+ break;
+
+ default:
+ res.headers.Status = res.statusCode + " " + res.statusMessage;
+ break;
+ }
+
+ if (res.headers['Content-type']) {
+ res.headers['Content-Type'] = res.headers['Content-type'];
+ delete (res.headers['Content-type'])
+ }
+
+ if (res.headers['content-type']) {
+ res.headers['Content-Type'] = res.headers['content-type'];
+ delete (res.headers['content-type'])
+ }
+
+ // header pass-through whitelist, case insensitive comparsion to server, however, you should
+ // specify the header case as you intend for the client
+ const headers = this.wtvshared.stripHeaders(res.headers, [
+ 'Connection',
+ 'Server',
+ 'Date',
+ 'Content-Type',
+ 'Cookie',
+ 'Location',
+ 'Accept-Ranges',
+ 'Last-Modified'
+ ]);
+ headers["wtv-http-proxy"] = true;
+ headers["wtv-trusted"] = false;
+
+ if (typeof res.headers['Content-Type'] === 'string' && res.headers['Content-Type'].startsWith("text")) {
+ // Get the original URL for relative link fixing
+ const originalUrl = request_headers.request.split(' ')[1];
+
+ // Transform HTML content for WebTV compatibility
+ if (res.headers['Content-Type'].includes('html') &&
+ this.minisrv_config.services[request_type]?.use_minifying_proxy === true) {
+ try {
+ const WTVMinifyingProxy = require('./WTVMinifyingProxy.js');
+ const proxy = new WTVMinifyingProxy(this.minisrv_config);
+
+ let htmlContent = Buffer.concat(data).toString();
+
+ // Apply WebTV-specific transformations
+ const transformOptions = {
+ removeImages: this.minisrv_config.services[request_type]?.remove_images || false,
+ maxImageWidth: this.minisrv_config.services[request_type]?.max_image_width || 400,
+ simplifyTables: this.minisrv_config.services[request_type]?.simplify_tables !== false,
+ maxWidth: this.minisrv_config.services[request_type]?.max_width || 544,
+ preserveJellyScript: this.minisrv_config.services[request_type]?.preserve_jellyscript !== false,
+ jellyScriptMaxSize: this.minisrv_config.services[request_type]?.jellyscript_max_size || 8192
+ };
+
+ htmlContent = proxy.transformForWebTV(htmlContent, originalUrl, transformOptions);
+ data = [Buffer.from(htmlContent)];
+
+ if (this.minisrv_config.config.verbosity >= 3) {
+ console.log(` * HTML transformed for WebTV compatibility (${originalUrl})`);
+ }
+ } catch (err) {
+ console.warn(` * HTML transformation failed: ${err.message}`);
+ }
+ }
+
+ if (request_type !== "http" && request_type !== "https") {
+ // replace http and https links on non http/https protocol (for proto:// for example)
+ const data_t = Buffer.concat(data).toString().replaceAll("http://", request_type + "://").replaceAll("https://", request_type + "://");
+ data = [Buffer.from(data_t)]
+ }
+ }
+
+ // if Connection: close header, set our internal variable to close the socket
+ if (headers['Connection']) {
+ if (headers['Connection'].toLowerCase().includes('close')) {
+ headers["wtv-connection-close"] = true;
+ }
+ }
+
+ // if a wtv-explaination is defined for an error code (except 200), define the header here to
+ // show the 'Explain' button on the client error ShowAlert
+ if (this.minisrv_config.services[request_type]['wtv-explanation']) {
+ if (this.minisrv_config.services[request_type]['wtv-explanation'][res.statusCode]) {
+ headers['wtv-explanation-url'] = this.minisrv_config.services[request_type]['wtv-explanation'][res.statusCode];
+ }
+ } else if (this.minisrv_config.services['http']['wtv-explanation']) {
+ if (this.minisrv_config.services['http']['wtv-explanation'][res.statusCode]) {
+ headers['wtv-explanation-url'] = this.minisrv_config.services['http']['wtv-explanation'][res.statusCode];
+ }
+ }
+ let data_hex = Buffer.concat(data).toString('hex');
+ if (data_hex.startsWith("0d0a0d0a")) data_hex = data_hex.slice(8);
+ if (data_hex.startsWith("0a0d0a")) data_hex = data_hex.slice(6);
+ if (data_hex.startsWith("0a0a")) data_hex = data_hex.slice(4);
+ this.sendToClient(socket, headers, Buffer.from(data_hex, 'hex'));
+ }
+
+}
+
+module.exports = WTVHTTP;
\ No newline at end of file
diff --git a/zefie_wtvp_minisrv/includes/classes/WTVPNM.js b/zefie_wtvp_minisrv/includes/classes/WTVPNM.js
index 92b77474..508541a4 100644
--- a/zefie_wtvp_minisrv/includes/classes/WTVPNM.js
+++ b/zefie_wtvp_minisrv/includes/classes/WTVPNM.js
@@ -30,7 +30,7 @@ class WTVPNM {
wtvshared = null;
sessions = new Map();
- constructor(minisrv_config, service_name = 'pnm') {
+ constructor(...[minisrv_config, service_name]) {
this.minisrv_config = minisrv_config;
this.service_name = service_name;
this.service_config = minisrv_config.services[service_name] || {};
diff --git a/zefie_wtvp_minisrv/includes/config.json b/zefie_wtvp_minisrv/includes/config.json
index 86547762..b97b49b2 100644
--- a/zefie_wtvp_minisrv/includes/config.json
+++ b/zefie_wtvp_minisrv/includes/config.json
@@ -301,7 +301,7 @@
},
"ftp": {
"port": 1650,
- "connections": 3
+ "connections": 3
},
"http": {
// http upstream
@@ -312,9 +312,13 @@
"external_proxy_host": "127.0.0.1", // IP address or hostname of proxy
"external_proxy_port": 1080, // Port of proxy
"flags": "0x00000001",
- "max_response_size": 16 // Megabytes
+ "max_response_size": 16, // Megabytes
+ "disallow_no_slash": true,
+ "handler_module": "WTVHTTP",
+ "handler_extra_vars": ["http", "sendToClient"]
},
"https": {
+ // https upstream
"port": 1650,
"connections": 3,
"use_external_proxy": false, // use an external proxy (WebONE or some other minifying proxy is recommended)
@@ -322,9 +326,14 @@
"external_proxy_host": "127.0.0.1", // IP address or hostname of proxy
"external_proxy_port": 1080, // Port of proxy
"flags": "0x00000001",
- "max_response_size": 16 // Megabytes
+ "max_response_size": 16, // Megabytes
+ "disallow_no_slash": true,
+ "handler_module": "WTVHTTP",
+ "handler_extra_vars": ["http", "sendToClient"],
+ "allow_self_signed_ssl": true // If true, will allow self-signed SSL certificates via the proxy.
},
"proto": {
+ // ProtoWeb Proxy
"port": 1650,
"connections": 3,
"use_external_proxy": true,
@@ -334,8 +343,22 @@
"external_proxy_port": 7851,
"external_proxy_is_http1": true,
"flags": "0x00000001",
- "max_response_size": 16 // Megabytes
- },
+ "max_response_size": 16, // Megabytes
+ "disallow_no_slash": true,
+ "handler_module": "WTVHTTP",
+ "handler_extra_vars": ["http", "sendToClient"]
+ },
+ "gopher": {
+ // Gopher processor
+ "port": 1651,
+ "connections": 3,
+ "allow_double_slash": true,
+ "flags": "0x00000001",
+ "uses_service_vault": false, // For custom services that use modules instead of service vaults
+ "disallow_no_slash": true,
+ "log_raw_gopher": false, // set to true to log raw gopher responses to the console for debugging
+ "handler_module": "WTVGopher"
+ },
"wtv-passport": {
// wtv-passport (for messenger)
"port": 1654
@@ -404,7 +427,8 @@
"descriptor_after_hello_ms": 85,
"burst_prestart_ms": 5000,
"debug": false,
- "allow_indexing": true
+ "allow_indexing": true,
+ "handler_module": "WTVPNM"
}
},
"favorites": {
diff --git a/zefie_wtvp_minisrv/ra_win/build.cmd b/zefie_wtvp_minisrv/ra_win/build.cmd
index b2ad25ea..21cd2b8b 100644
--- a/zefie_wtvp_minisrv/ra_win/build.cmd
+++ b/zefie_wtvp_minisrv/ra_win/build.cmd
@@ -1,4 +1,4 @@
@echo OFF
cl rpcli.cpp /O2 /MT /Fe:rpcli.exe
-del rpcli.obj prct3260.thl
+del rpcli.obj prct3260.tlh