modulize HTTP(s) proxy, add Gopher support, change how protocol-specific modules are loaded

zefie_wtvp_minisrv/app.js

@@ -11,8 +11,7 @@ const nunjucks = require('nunjucks');
 const zlib = require('zlib');
 const {serialize, unserialize} = require('php-serialize');
 const {spawn} = require('child_process');
-const http = require('follow-redirects').http
+const http = require('follow-redirects').http;
-const https = require('follow-redirects').https
 const httpx = require(classPath + "/HTTPX.js");
 const { URL } = require('url');
 const net = require('net');
@@ -32,6 +31,7 @@ const WTVPNM = require(classPath + "/WTVPNM.js");
 const vm = require('vm');
 const debug = require('debug')('app');
 const express = require('express');
+let handlerModules = [];
 
 let wtvnewsserver = null;
 const protocolServers = [];
@@ -304,8 +304,8 @@ const runScriptInVM = function (script_data, user_contextObj = {}, privileged =
 
         // Our modules
         "wtvmime": wtvmime,
-        "http": http,
+        "http": require('follow-redirects').http,
-        "https": https,
+        "https": require('follow-redirects').https,
         "util": util,
         "sharp": sharp,
         "nunjucks": nunjucks,
@@ -984,7 +984,11 @@ async function processURL(socket, request_headers, pc_services = false) {
     let shortURL, headers, data, service_name;
     let original_service_name = "";
     let shared_romcache = null;
-    let allow_double_slash = false, enable_multi_query = false, use_external_proxy = false;
+    let allow_double_slash = false;
+    let enable_multi_query = false;
+    let use_external_proxy = false;
+    let disallow_no_slash = false;
+    let uses_service_vault = true;
     request_headers.query = {};
 
     if (request_headers.request_url) {
@@ -993,6 +997,8 @@ async function processURL(socket, request_headers, pc_services = false) {
             allow_double_slash = minisrv_config.services[service_name].allow_double_slash || false;
             enable_multi_query = minisrv_config.services[service_name].enable_multi_query || false;
             use_external_proxy = minisrv_config.services[service_name].use_external_proxy || false;
+            disallow_no_slash = minisrv_config.services[service_name].disallow_no_slash || false;
+            uses_service_vault = (minisrv_config.services[service_name].uses_service_vault === false) ? false : true;
         }
         if (pc_services) {           
             original_service_name = socket.service_name; // store service name
@@ -1078,7 +1084,7 @@ async function processURL(socket, request_headers, pc_services = false) {
                
             }
         }
-        if ((!shortURL.startsWith("http") && !shortURL.startsWith("ftp") && shortURL.includes(":") && !shortURL.includes(":/"))) {
+        if ((!disallow_no_slash && shortURL.includes(":") && !shortURL.includes(":/"))) {
             // Apparently it is within WTVP spec to accept urls without a slash (eg wtv-home:home)
             // Here, we just reassemble the request URL as if it was a proper URL (eg wtv-home:/home)
             // we will allow this on any service except http(s) and ftp
@@ -1143,7 +1149,7 @@ minisrv-no-mail-count: true`;
             else console.log(" * " + ((ssl) ? "SSL " : "") + "PC request on service " + original_service_name + " (Service Vault " + service_name + ") for " + request_headers.request_url, 'on', socket.id);
         } 
 
-        if ((shortURL.includes(':/')) && (!shortURL.includes('://') || (shortURL.includes('://') && allow_double_slash))) {
+        if ((shortURL.includes(':/')) && (!shortURL.includes('://') || (shortURL.includes('://') && allow_double_slash) && uses_service_vault)) {
             let ssid = socket.ssid;
             if (ssid === null) {
                 // prevent possible injection attacks via malformed SSID and filesystem SessionStore
@@ -1185,8 +1191,10 @@ minisrv-no-mail-count: true`;
 
             socket_sessions[socket.id].request_headers = request_headers;
             processPath(socket, urlToPath, request_headers, service_name, shared_romcache, pc_services);
-        } else if (shortURL.includes('http://') || shortURL.includes('https://') || (use_external_proxy === true && shortURL.includes(service_name + "://")) && !pc_services) {
+        } else if (handlerModules["wtvhttp"] && ((shortURL.includes('http://') || shortURL.includes('https://')) || (use_external_proxy === true && shortURL.includes(service_name + "://")) && !pc_services)) {
-            doHTTPProxy(socket, request_headers);
+            handlerModules["wtvhttp"].doHTTPProxy(socket, request_headers);
+        } else if (handlerModules["wtvgopher"] && shortURL.startsWith("gopher://")) {
+            handlerModules["wtvgopher"].handleGopherRequest(socket, request_headers, wtvshared, sendToClient);
         } else if (shortURL.startsWith('ftp://')) {
             if (minisrv_config.config.debug_flags.show_headers) console.debug(" * Incoming FTP request on WTVP socket ID", socket.id, await wtvshared.decodePostData(await wtvshared.filterRequestLog(await wtvshared.filterSSID(request_headers))));
             const wtvftp = new WTVFTP(wtvshared, sendToClient);
@@ -1230,250 +1238,6 @@ minisrv-no-mail-count: true`;
     }
 }
 
-function handleProxy(socket, request_type, request_headers, res, data) {
-    console.log(` * Proxy Request ${request_type.toUpperCase()} ${res.statusCode} for ${request_headers.request}`)
-    // an http response error is not a request error, and will come here under the 'end' event rather than an 'error' event.
-    switch (res.statusCode) {
-        case 404:
-            res.headers.Status = res.statusCode + " The publisher can’t find the page requested.";
-            break;
-
-        case 401:
-        case 403:
-            res.headers.Status = res.statusCode + " The publisher of that page has not authorized you to use it.";
-            break;
-
-        case 500:
-            res.headers.Status = res.statusCode + " The publisher of that page can’t be reached.";
-            break;
-
-        default:
-            res.headers.Status = res.statusCode + " " + res.statusMessage;
-            break;
-    }
-
-    if (res.headers['Content-type']) {
-        res.headers['Content-Type'] = res.headers['Content-type'];
-        delete (res.headers['Content-type'])
-    }
-
-    if (res.headers['content-type']) {
-        res.headers['Content-Type'] = res.headers['content-type'];
-        delete (res.headers['content-type'])
-    }
-  
-    // header pass-through whitelist, case insensitive comparsion to server, however, you should
-    // specify the header case as you intend for the client
-    const headers = wtvshared.stripHeaders(res.headers, [
-        'Connection',
-        'Server',
-        'Date',
-        'Content-Type',
-        'Cookie',
-        'Location',
-        'Accept-Ranges',
-        'Last-Modified'
-    ]);
-    headers["wtv-http-proxy"] = true;
-    headers["wtv-trusted"] = false;
-
-    if (typeof res.headers['Content-Type'] === 'string' && res.headers['Content-Type'].startsWith("text")) {
-        // Get the original URL for relative link fixing
-        const originalUrl = request_headers.request.split(' ')[1];
-        
-        // Transform HTML content for WebTV compatibility
-        if (res.headers['Content-Type'].includes('html') && 
-            minisrv_config.services[request_type]?.use_minifying_proxy === true) {
-            try {
-                const WTVMinifyingProxy = require('./includes/classes/WTVMinifyingProxy.js');
-                const proxy = new WTVMinifyingProxy(minisrv_config);
-                
-                let htmlContent = Buffer.concat(data).toString();
-                
-                // Apply WebTV-specific transformations
-                const transformOptions = {
-                    removeImages: minisrv_config.services[request_type]?.remove_images || false,
-                    maxImageWidth: minisrv_config.services[request_type]?.max_image_width || 400,
-                    simplifyTables: minisrv_config.services[request_type]?.simplify_tables !== false,
-                    maxWidth: minisrv_config.services[request_type]?.max_width || 544,
-                    preserveJellyScript: minisrv_config.services[request_type]?.preserve_jellyscript !== false,
-                    jellyScriptMaxSize: minisrv_config.services[request_type]?.jellyscript_max_size || 8192
-                };
-                
-                htmlContent = proxy.transformForWebTV(htmlContent, originalUrl, transformOptions);
-                data = [Buffer.from(htmlContent)];
-                
-                if (minisrv_config.config.verbosity >= 3) {
-                    console.log(` * HTML transformed for WebTV compatibility (${originalUrl})`);
-                }
-            } catch (err) {
-                console.warn(` * HTML transformation failed: ${err.message}`);
-            }
-        }
-
-        if (request_type !== "http" && request_type !== "https") {
-            // replace http and https links on non http/https protocol (for proto:// for example)
-            const data_t = Buffer.concat(data).toString().replaceAll("http://", request_type + "://").replaceAll("https://", request_type + "://");
-            data = [Buffer.from(data_t)]
-        }
-    }
-
-    // if Connection: close header, set our internal variable to close the socket
-    if (headers['Connection']) {
-        if (headers['Connection'].toLowerCase().includes('close')) {
-            headers["wtv-connection-close"] = true;
-        }
-    }
-
-    // if a wtv-explaination is defined for an error code (except 200), define the header here to
-    // show the 'Explain' button on the client error ShowAlert
-    if (minisrv_config.services['http']['wtv-explanation']) {
-        if (minisrv_config.services['http']['wtv-explanation'][res.statusCode]) {
-            headers['wtv-explanation-url'] = minisrv_config.services['http']['wtv-explanation'][res.statusCode];
-        }
-    }
-    let data_hex = Buffer.concat(data).toString('hex');
-    if (data_hex.startsWith("0d0a0d0a")) data_hex = data_hex.slice(8);
-    if (data_hex.startsWith("0a0d0a")) data_hex = data_hex.slice(6);
-    if (data_hex.startsWith("0a0a")) data_hex = data_hex.slice(4);
-    sendToClient(socket, headers, Buffer.from(data_hex, 'hex'));
-}
-
-async function doHTTPProxy(socket, request_headers) {
-    // detect protocol name
-    const idx = request_headers.request_url.indexOf('/') - 1;
-
-    const request_type = request_headers.request_url.slice(0, idx);
-    if (minisrv_config.config.debug_flags.show_headers) console.debug(request_type.toUpperCase() + " Proxy: Client Request Headers on socket ID", socket.id, (await wtvshared.decodePostData(await wtvshared.filterRequestLog(await wtvshared.filterSSID(request_headers)))));
-    else debug(request_type.toUpperCase() + " Proxy: Client Request Headers on socket ID", socket.id, (await wtvshared.decodePostData(await wtvshared.filterRequestLog(await wtvshared.filterSSID(request_headers)))));
-    let proxy_agent;
-
-    switch (request_type) {
-        case "https":
-            proxy_agent = https;
-            break;
-        case "http":
-        case "proto":
-            proxy_agent = http;
-            break;
-    }
-
-    const request_data = [];
-    const data = [];
-
-    request_data.method = request_headers.request.split(' ')[0];
-    const request_url_split = request_headers.request.split(' ')[1].split('/');
-    request_data.host = request_url_split[2];
-    if (request_data.host.indexOf(':') > 0) {
-        request_data.port = request_data.host.split(':')[1];
-        request_data.host = request_data.host.split(':')[0];
-    } else {
-        if (request_type === "https") request_data.port = 443;
-        else request_data.port = 80;
-    }
-    for (let i = 0; i < 3; i++) request_url_split.shift();
-    request_data.path = "/" + request_url_split.join('/');
-    if (request_data.method && request_data.host && request_data.path) {
-
-        const options = {
-            host: request_data.host,
-            port: request_data.port,
-            path: request_data.path,
-            method: request_data.method,
-            followAllRedirects: true,
-            headers: {
-                "User-Agent": request_headers["User-Agent"] || "WebTV",
-                "Connection": "Keep-Alive"
-            }
-        }
-
-        // RFC7239
-        if (socket.remoteAddress !== "127.0.0.1") {
-            options.headers["X-Forwarded-For"] = socket.remoteAddress;
-        }
-
-        if (request_headers.post_data) {
-            if (request_headers["Content-type"]) options.headers["Content-type"] = request_headers["Content-type"];
-            if (request_headers["Content-length"]) options.headers["Content-length"] = request_headers["Content-length"];
-        }
-
-        if (minisrv_config.services[request_type].use_external_proxy && minisrv_config.services[request_type].external_proxy_port) {
-            // configure connection to an external proxy
-            if (minisrv_config.services[request_type].external_proxy_is_socks) {
-                // configure connection to remote socks proxy
-                const { SocksProxyAgent }= require('socks-proxy-agent');
-                options.agent = new SocksProxyAgent("socks://" + (minisrv_config.services[request_type].external_proxy_host || "127.0.0.1") + ":" + minisrv_config.services[request_type].external_proxy_port);
-                options.agents = {
-                    "http": options.agent,
-                    "https": options.agent
-                }
-            } else {
-                // configure connection to remote http proxy
-                proxy_agent = http;
-                options.host = minisrv_config.services[request_type].external_proxy_host;
-                options.port = minisrv_config.services[request_type].external_proxy_port;
-                options.path = request_headers.request.split(' ')[1];
-                options.headers.Host = request_data.host + ":" + request_data.port;
-                if (minisrv_config.services[request_type].replace_protocol) {
-                    options.path = options.path.replace(request_type, minisrv_config.services[request_type].replace_protocol);
-                }
-            }
-            if (minisrv_config.services[request_type].external_proxy_is_http1) {
-                options.insecureHTTPParser = true;
-                options.headers.Connection = 'close'
-            }
-        }
-        const req = proxy_agent.request(options, function (res) {
-            let total_data = 0;
-
-            res.on('data', d => {
-                data.push(d);
-                total_data += d.length;
-                if (total_data > 1024 * 1024 * parseFloat(minisrv_config.services[request_type].max_response_size || 16)) {
-                    console.warn(` * Response data exceeded ${minisrv_config.services[request_type].max_response_size || 16}MB limit, destroying...`);
-                    res.destroy();
-                    const errpage = wtvshared.doErrorPage(400, "The item chosen is too large to be used.");
-                    sendToClient(socket, errpage[0], errpage[1]);
-                }
-            })
-
-            res.on('error', function (err) {
-                // hack for Protoweb ECONNRESET
-                if (minisrv_config.services[request_type].external_proxy_is_http1 && data.length > 0) {
-                    handleProxy(socket, request_type, request_headers, res, data);
-                } else {
-                    console.error(" * Unhandled Proxy Request Error:", err);
-                }
-            });
-
-            res.on('end', function () {
-                // For when http proxies behave correctly
-                if (!minisrv_config.services[request_type].external_proxy_is_http1 || data.length > 0) {
-                    handleProxy(socket, request_type, request_headers, res, data);
-                }
-            });
-        }).on('error', function (err) {
-                // severe errors, such as unable to connect.
-            if (err.code === "ENOTFOUND" || err.message.indexOf("HostUnreachable") > 0) {
-                const errpage = wtvshared.doErrorPage(400, `The publisher <b>${request_data.host}</b> is unknown.`);
-                sendToClient(socket, errpage[0], errpage[1]);
-            } else {
-                console.error(" * Unhandled Proxy Request Error:", err);
-                const errpage = wtvshared.doErrorPage(400);
-                sendToClient(socket, errpage[0], errpage[1]);
-            }
-           
-        });
-        if (request_headers.post_data) {
-            req.write(Buffer.from(request_headers.post_data.toString(CryptoJS.enc.Hex), 'hex'), function () {
-                req.end();
-            });
-        } else {
-            req.end();
-        }
-    }
-}
-
 async function sendToClient(socket, headers_obj, data = null) {
     let headers = "";
     let content_length = 0;
@@ -2463,9 +2227,27 @@ const service_ip = minisrv_config.config.service_ip;
 Object.keys(minisrv_config.services).forEach(function (k) {
     if (typeof(minisrv_config.services[k]) === 'function') return;
     if (configureService(k, minisrv_config.services[k], true)) {
+        let loadedModule = false;
+        if (minisrv_config.services[k].handler_module) {
+            try {
+                if (!handlerModules[minisrv_config.services[k].handler_module + "_main"]) {
+                    handlerModules[minisrv_config.services[k].handler_module + "_main"] = require(classPath + "/" + minisrv_config.services[k].handler_module + ".js");
+                    var args = [];
+                    for (let i = 0; i < (minisrv_config.services[k].handler_extra_vars || []).length; i++) {
+                        let extraVar = eval(minisrv_config.services[k].handler_extra_vars[i]);
+                        args.push(extraVar);
+                    }
+                    const constructorArgs = [minisrv_config, k, ...args];
+                    handlerModules[minisrv_config.services[k].handler_module.toLowerCase()] = new handlerModules[minisrv_config.services[k].handler_module + "_main"](...constructorArgs);
+                }
+                loadedModule = true;
+            } catch (e) {
+                console.error(" # Failed to load handler module for service", k, "module:", minisrv_config.services[k].handler_module, e);
+            }
+        }
         const using_tls = (minisrv_config.services[k].pc_services && minisrv_config.services[k].https_cert && minisrv_config.services[k].use_https) ? true : false;
         const protocol = (minisrv_config.services[k].protocol_handler) ? minisrv_config.services[k].protocol_handler.toUpperCase() : (minisrv_config.services[k].pc_services) ? "HTTP" : "WTVP";
-        console.log(" * Configured Service:", k, "on Port", minisrv_config.services[k].port, "- Service Host:", minisrv_config.services[k].host + ((using_tls) ? " (TLS)" : ""), "- Protocol:", protocol);
+        console.log(" * Configured Service:", k, "on Port", minisrv_config.services[k].port, "- Service Host:", minisrv_config.services[k].host + ((using_tls) ? " (TLS)" : ""), "- Protocol:", protocol, (loadedModule) ? "- Handler Module: " + minisrv_config.services[k].handler_module : "");
 
         if (minisrv_config.services[k].local_nntp_enabled && minisrv_config.services[k].local_nntp_port) {
             if (!wtvnewsserver) {
@@ -2548,9 +2330,8 @@ Object.keys(minisrv_config.services).forEach((service_name) => {
     if (!service || service.disabled || !service.port) return;
     if (service.protocol_handler === 'pnm') {
         try {
-            const pnmServer = new WTVPNM(minisrv_config, service_name);
+            handlerModules['wtvpnm'].listen(service.port, minisrv_config.config.bind_ip);
-            pnmServer.listen(service.port, minisrv_config.config.bind_ip);
+            protocolServers.push(handlerModules['wtvpnm']);
-            protocolServers.push(pnmServer);
         } catch (e) {
             throw ("Could not bind PNM protocol handler to port " + service.port + " on " + minisrv_config.config.bind_ip + ": " + e.toString());
         }

zefie_wtvp_minisrv/includes/classes/WTVGopher.js

@@ -0,0 +1,259 @@
+const WTVMime = require('./WTVMime.js');
+const net = require('net');
+
+class WTVGopher {
+    // Adapted from WebTV Redialed's Gopher support
+    constructor(...[minisrv_config, service_name]) {
+        this.minisrv_config = minisrv_config;
+        this.wtvmime = new WTVMime(minisrv_config);
+        this.logGopher = minisrv_config.services[service_name].log_raw_gopher || false;
+    }
+
+    looksLikeMenu(gopherData) {
+        const lines = gopherData.split(/\r?\n/);
+
+        let checked = 0;
+        let menuLines = 0;
+
+        for (const line of lines) {
+            if (!line || line === ".") continue;
+
+            checked++;
+            let typeOffset = 0;
+            let type = " ";
+            while ((type === " " || type == "\t") && typeOffset <= 10) {
+                type = line[typeOffset];
+                typeOffset++;
+            }         
+            const rest = line.slice(1);
+
+            if (
+                rest.includes("\t") &&
+                rest.split("\t").length >= 3 &&
+                /^[0-9A-Za-z+gIihs]$/.test(type)
+            ) {
+                menuLines++;
+            }
+
+            if (checked >= 5) break;
+        }
+
+        return menuLines >= 2 || (lines.length <= 2 && menuLines == 1);
+    }
+
+    processGopherData(gopherData) {
+        // currently looking at textfile, don't process into HTML
+        if (!this.looksLikeMenu(gopherData)) {
+            return `<pre>${gopherData}</pre>`;
+        }
+
+        // okay, we're not a textfile, now do the menu shit
+        let pageTitle = "Gopher Menu"
+        const lines = gopherData.split("\r\n");
+
+        let html = "";
+
+        for (const line of lines) {
+            if (!line || line === ".") continue;
+
+            const type = line[0];
+            const parts = line.slice(1).split("\t");
+
+            const text = parts[0] || "";
+            const selector = parts[1];
+            const host = parts[2];
+            const port = parts[3] || 70;
+            var url = `gopher://${host}:${port}${selector}`;
+
+            // determine page title from first line
+            const firstline = line[0].slice(1).trim();
+
+            if (line[0] === "i" && firstline.length > 0) {
+                pageTitle = line.slice(1).trim();
+                html = `<title>${pageTitle}</title><pre>\n`;
+            } else if (pageTitle === "Gopher Menu") {
+                for (const line of lines) {
+                    if (!line || line === ".") continue;
+
+                    let typeOffset = 0;
+                    let type = " ";
+                    while ((type === " " || type == "\t") && typeOffset <= 10) {
+                        type = line[typeOffset];
+                        typeOffset++;
+                    }
+                    const parts = line.slice(1).split("\t");
+                    const text = parts[0]?.trim();
+
+                    if (text && text.length > 0) {
+                        pageTitle = text;
+                        html = `<title>${pageTitle}</title><pre>\n`;
+                        break;
+                    }
+                }
+            }
+            switch (type) {
+                case "i": // informational / "just text"
+                    html += `${text}\n`;
+                break;
+
+                case "0": // text file
+                case "1": // directory
+                    html += `<a href="${url}">${text}</a>\n`;
+                    break;
+                case "3": // error, otherwise just plain text
+                    html += `${text}<br>\n`;
+                case "h": // HTML link
+                    if (selector?.startsWith("URL:")) {
+                    const httpUrl = selector.slice(4);
+                    html += `<a href="${httpUrl}">${text}</a>\n`;
+                    }
+                    break;
+                
+                case "7": // search
+                    html += `<form action="${url}" method="get">
+<label for="search">${text}</label>
+<input type="search" name="q" required>
+</form>`;
+                    break;
+                case "g":
+                case "I":
+                case "p":
+                    url = `gopher://${host}:${port}${selector}?type=${type}`;
+                    html += `<a href="${url}">${text}</a>\n`;
+                    break;
+
+                default:
+                    html += `${text} (unsupported type ${type})\n`;
+            }
+        }
+
+        html += "</pre>";
+        return html;
+    }
+
+    async handleGopherRequest(socket, request_headers, wtvshared, sendToClient) {
+        if (this.minisrv_config.config.debug_flags.show_headers) {
+            console.log("Gopher: Client Request on socket ID",
+                socket.id,
+                await wtvshared.decodePostData(
+                    wtvshared.filterRequestLog(wtvshared.filterSSID(request_headers))
+                ));
+        }
+
+        // crlf for sending at the end of a request
+        const crlf = "0D0A"
+        const crlf_bytes = Buffer.from(crlf, 'hex');
+        // chunk stuff for gopher-to-html conversion
+        let chunks = [];
+
+        var request_data = new Array();
+        request_data.method = request_headers.request.split(' ')[0];
+
+        const rawUrl = decodeURIComponent(request_headers.request.split(' ')[1]).replaceAll('\\', '/');
+        const [pathPart, queryPart] = rawUrl.split('?');
+        var request_url_split = pathPart.split('/');
+
+        let queryParams = {};
+        if (queryPart) {
+            for (const kv of queryPart.split('&')) {
+                const [k, v] = kv.split('=');
+                queryParams[k] = decodeURIComponent(v || "");
+            }
+        }
+
+        request_data.host = request_url_split[2];
+        if (request_data.host.indexOf(':') > 0) {
+            request_data.port = request_data.host.split(':')[1];
+            request_data.host = request_data.host.split(':')[0];
+        } else {
+            request_data.port = 70;
+        }
+
+        for (var i = 0; i < 3; i++) request_url_split.shift();
+        request_data.path = "/" + request_url_split.join('/');
+        // vars for determining if a link is an image
+        const imageTypes = ["g", "I", "p"];
+        let requestType = null;
+        if (queryParams.type && imageTypes.includes(queryParams.type)) {
+            requestType = queryParams.type;
+        }
+        const isImageDownload = !!requestType;
+
+        const client = new net.Socket();
+        client.setTimeout(3000);
+
+        // make the initial request to the server
+        client.connect(request_data.port, request_data.host, () => {
+            let gopherRequest = "";
+
+            // if user requested path
+            if (request_data.path.length >= 2) {
+                gopherRequest = request_data.path;
+            }
+
+            // if user requested type 7 (search)
+            if (queryParams.q) {
+                const query = queryParams.q.replace(/\+/g, ' ');
+                gopherRequest += "\t" + query;
+            }
+
+            client.write(gopherRequest + crlf_bytes);
+        });
+
+        // "holy shit we got data guys"
+        client.on("data", chunk => {
+            chunks.push(chunk);
+        });
+
+        // datastream end, time to process it
+        client.on("end", () => {
+            const gopherData = Buffer.concat(chunks).toString("utf-8");
+            if (this.logGopher) {
+                console.log("Gopher: Data received from server for socket ID", socket.id);
+                console.log("Gopher: Data length:", Buffer.concat(chunks).length);
+                console.log("isImageDownload:", isImageDownload);
+                console.log("Gopher Data:\n", gopherData);
+            }
+
+            // are we downloading an image?
+            if (isImageDownload) {
+                const imageData = Buffer.concat(chunks);
+                const mimetype = this.wtvmime.detectMimeTypeFromBuffer(imageData);
+
+                const headers = {
+                    "Status": "200 OK",
+                    "Content-Type": mimetype
+                }
+
+                sendToClient(socket, headers, imageData);
+                return;
+            } else {
+                // convert gophermap to html
+                const htmlData = this.processGopherData(gopherData);
+                // since gopher doesn't exactly have "headers" and by this point we're probably already fine to just say it's okay, we're just sending back the bare minimum to prevent screaming
+                const headers = {
+                    "Status": "200 OK",
+                    "Content-Type": "text/html"
+                }
+                sendToClient(socket, headers, htmlData);
+            }
+        });
+
+        // blew up?
+        // todo: figure out what error actually looks like and if appropriate to send to client (or just "Connection failed" or smth)
+        client.on('error', (err) => {
+            console.error('Gopher error: ' + err);
+            let friendlyErr = err.toString();
+            if (friendlyErr.includes('ETIMEDOUT')) {
+                friendlyErr = "Connection timed out";
+            } else if (friendlyErr.includes('ECONNREFUSED')) {
+                friendlyErr = "Connection refused";
+            } else if (friendlyErr.includes('ENOTFOUND')) {
+                friendlyErr = "Host not found";
+            }
+            sendToClient(socket, {"Status": "400 Gopher Error: " + friendlyErr}, friendlyErr);
+        });
+    }
+}
+
+module.exports = WTVGopher;

zefie_wtvp_minisrv/includes/classes/WTVHTTP.js

@@ -0,0 +1,266 @@
+const {WTVShared, clientShowAlert} = require('./WTVShared.js');
+
+class WTVHTTP {
+    constructor(...[minisrv_config, service_name, http, sendToClient]) {
+        this.minisrv_config = minisrv_config;
+        this.service_name = service_name;
+        this.wtvshared = new WTVShared(minisrv_config);
+        this.sendToClient = sendToClient;
+        this.http = http;
+        this.https = require('follow-redirects').https
+        this.proxy_agent = null;
+    }
+
+    async doHTTPProxy(socket, request_headers) {
+        // detect protocol name
+        const idx = request_headers.request_url.indexOf('/') - 1;
+
+        const request_type = request_headers.request_url.slice(0, idx);
+        if (this.minisrv_config.config.debug_flags.show_headers) console.debug(request_type.toUpperCase() + " Proxy: Client Request Headers on socket ID", socket.id, (await this.wtvshared.decodePostData(await this.wtvshared.filterRequestLog(await this.wtvshared.filterSSID(request_headers)))));
+        else debug(request_type.toUpperCase() + " Proxy: Client Request Headers on socket ID", socket.id, (await this.wtvshared.decodePostData(await this.wtvshared.filterRequestLog(await this.wtvshared.filterSSID(request_headers)))));
+
+        switch (request_type) {
+            case "https":
+                this.proxy_agent = this.https;
+                break;
+            default:
+                this.proxy_agent = this.http;
+                break;
+        }
+
+        const request_data = [];
+        const data = [];
+
+        request_data.method = request_headers.request.split(' ')[0];
+        const request_url_split = request_headers.request.split(' ')[1].split('/');
+        request_data.host = request_url_split[2];
+        if (request_data.host.indexOf(':') > 0) {
+            request_data.port = request_data.host.split(':')[1];
+            request_data.host = request_data.host.split(':')[0];
+        } else {
+            if (request_type === "https") request_data.port = 443;
+            else request_data.port = 80;
+        }
+        for (let i = 0; i < 3; i++) request_url_split.shift();
+        request_data.path = "/" + request_url_split.join('/');
+        if (request_data.method && request_data.host && request_data.path) {
+
+            const options = {
+                host: request_data.host,
+                port: request_data.port,
+                path: request_data.path,
+                method: request_data.method,
+                followAllRedirects: true,
+                headers: {
+                    "User-Agent": request_headers["User-Agent"] || "WebTV",
+                    "Connection": "Keep-Alive"
+                }
+            }
+
+            // RFC7239
+            if (socket.remoteAddress !== "127.0.0.1") {
+                options.headers["X-Forwarded-For"] = socket.remoteAddress;
+            }
+
+            if (request_headers.post_data) {
+                if (request_headers["Content-type"]) options.headers["Content-type"] = request_headers["Content-type"];
+                if (request_headers["Content-length"]) options.headers["Content-length"] = request_headers["Content-length"];
+            }
+
+            if (request_type == "https" && this.minisrv_config.services[request_type].allow_self_signed_ssl) {
+                options.rejectUnauthorized = false;
+            }
+
+            if (this.minisrv_config.services[request_type].use_external_proxy && minisrv_config.services[request_type].external_proxy_port) {
+                // configure connection to an external proxy
+                if (this.minisrv_config.services[request_type].external_proxy_is_socks) {
+                    // configure connection to remote socks proxy
+                    const { SocksProxyAgent }= require('socks-proxy-agent');
+                    options.agent = new SocksProxyAgent("socks://" + (minisrv_config.services[request_type].external_proxy_host || "127.0.0.1") + ":" + minisrv_config.services[request_type].external_proxy_port);
+                    options.agents = {
+                        "http": options.agent,
+                        "https": options.agent
+                    }
+                } else {
+                    // configure connection to remote http proxy
+                    this.proxy_agent = this.http;
+                    options.host = this.minisrv_config.services[request_type].external_proxy_host;
+                    options.port = this.minisrv_config.services[request_type].external_proxy_port;
+                    options.path = request_headers.request.split(' ')[1];
+                    options.headers.Host = request_data.host + ":" + request_data.port;
+                    if (this.minisrv_config.services[request_type].replace_protocol) {
+                        options.path = options.path.replace(request_type, this.minisrv_config.services[request_type].replace_protocol);
+                    }
+                }
+                if (this.minisrv_config.services[request_type].external_proxy_is_http1) {
+                    options.insecureHTTPParser = true;
+                    options.headers.Connection = 'close'
+                }
+            }
+            const req = this.proxy_agent.request(options, (res) => {
+                let total_data = 0;
+
+                res.on('data', d => {
+                    data.push(d);
+                    total_data += d.length;
+                    if (total_data > 1024 * 1024 * parseFloat(this.minisrv_config.services[request_type].max_response_size || 16)) {
+                        console.warn(` * Response data exceeded ${this.minisrv_config.services[request_type].max_response_size || 16}MB limit, destroying...`);
+                        res.destroy();
+                        const errpage = this.wtvshared.doErrorPage(400, "The item chosen is too large to be used.");
+                        this.sendToClient(socket, errpage[0], errpage[1]);
+                    }
+                })
+
+                res.on('error', (err) => {
+                    // hack for Protoweb ECONNRESET
+                    if (this.minisrv_config.services[request_type].external_proxy_is_http1 && data.length > 0) {
+                        this.handleProxy(socket, request_type, request_headers, res, data);
+                    } else {
+                        console.error(" * Unhandled Proxy Request Error:", err);
+                    }
+                });
+
+                res.on('end', () => {
+                    // For when http proxies behave correctly
+                    if (!this.minisrv_config.services[request_type].external_proxy_is_http1 || data.length > 0) {
+                        this.handleProxy(socket, request_type, request_headers, res, data);
+                    }
+                });
+            }).on('error', (err) => {
+                    // severe errors, such as unable to connect.
+                if (err.code === "ENOTFOUND" || err.message.indexOf("HostUnreachable") > 0) {
+                    const errpage = this.wtvshared.doErrorPage(400, `The publisher <b>${request_data.host}</b> is unknown.`);
+                    this.sendToClient(socket, errpage[0], errpage[1]);
+                } else {
+                    console.error(" * Unhandled Proxy Request Error:", err);
+                    const errpage = this.wtvshared.doErrorPage(400);
+                    this.sendToClient(socket, errpage[0], errpage[1]);
+                }
+                
+            });
+            if (request_headers.post_data) {
+                req.write(Buffer.from(request_headers.post_data.toString(CryptoJS.enc.Hex), 'hex'), () => {
+                    req.end();
+                });
+            } else {
+                req.end();
+            }
+        }    
+    }
+
+    handleProxy(socket, request_type, request_headers, res, data) {
+        console.log(` * Proxy Request ${request_type.toUpperCase()} ${res.statusCode} for ${request_headers.request}`)
+        // an http response error is not a request error, and will come here under the 'end' event rather than an 'error' event.
+        switch (res.statusCode) {
+            case 404:
+                res.headers.Status = res.statusCode + " The publisher can&#146;t find the page requested.";
+                break;
+
+            case 401:
+            case 403:
+                res.headers.Status = res.statusCode + " The publisher of that page has not authorized you to use it.";
+                break;
+
+            case 500:
+                res.headers.Status = res.statusCode + " The publisher of that page can&#146;t be reached.";
+                break;
+
+            default:
+                res.headers.Status = res.statusCode + " " + res.statusMessage;
+                break;
+        }
+
+        if (res.headers['Content-type']) {
+            res.headers['Content-Type'] = res.headers['Content-type'];
+            delete (res.headers['Content-type'])
+        }
+
+        if (res.headers['content-type']) {
+            res.headers['Content-Type'] = res.headers['content-type'];
+            delete (res.headers['content-type'])
+        }
+    
+        // header pass-through whitelist, case insensitive comparsion to server, however, you should
+        // specify the header case as you intend for the client
+        const headers = this.wtvshared.stripHeaders(res.headers, [
+            'Connection',
+            'Server',
+            'Date',
+            'Content-Type',
+            'Cookie',
+            'Location',
+            'Accept-Ranges',
+            'Last-Modified'
+        ]);
+        headers["wtv-http-proxy"] = true;
+        headers["wtv-trusted"] = false;
+
+        if (typeof res.headers['Content-Type'] === 'string' && res.headers['Content-Type'].startsWith("text")) {
+            // Get the original URL for relative link fixing
+            const originalUrl = request_headers.request.split(' ')[1];
+            
+            // Transform HTML content for WebTV compatibility
+            if (res.headers['Content-Type'].includes('html') && 
+                this.minisrv_config.services[request_type]?.use_minifying_proxy === true) {
+                try {
+                    const WTVMinifyingProxy = require('./WTVMinifyingProxy.js');
+                    const proxy = new WTVMinifyingProxy(this.minisrv_config);
+                    
+                    let htmlContent = Buffer.concat(data).toString();
+                    
+                    // Apply WebTV-specific transformations
+                    const transformOptions = {
+                        removeImages: this.minisrv_config.services[request_type]?.remove_images || false,
+                        maxImageWidth: this.minisrv_config.services[request_type]?.max_image_width || 400,
+                        simplifyTables: this.minisrv_config.services[request_type]?.simplify_tables !== false,
+                        maxWidth: this.minisrv_config.services[request_type]?.max_width || 544,
+                        preserveJellyScript: this.minisrv_config.services[request_type]?.preserve_jellyscript !== false,
+                        jellyScriptMaxSize: this.minisrv_config.services[request_type]?.jellyscript_max_size || 8192
+                    };
+                    
+                    htmlContent = proxy.transformForWebTV(htmlContent, originalUrl, transformOptions);
+                    data = [Buffer.from(htmlContent)];
+                    
+                    if (this.minisrv_config.config.verbosity >= 3) {
+                        console.log(` * HTML transformed for WebTV compatibility (${originalUrl})`);
+                    }
+                } catch (err) {
+                    console.warn(` * HTML transformation failed: ${err.message}`);
+                }
+            }
+
+            if (request_type !== "http" && request_type !== "https") {
+                // replace http and https links on non http/https protocol (for proto:// for example)
+                const data_t = Buffer.concat(data).toString().replaceAll("http://", request_type + "://").replaceAll("https://", request_type + "://");
+                data = [Buffer.from(data_t)]
+            }
+        }
+
+        // if Connection: close header, set our internal variable to close the socket
+        if (headers['Connection']) {
+            if (headers['Connection'].toLowerCase().includes('close')) {
+                headers["wtv-connection-close"] = true;
+            }
+        }
+
+        // if a wtv-explaination is defined for an error code (except 200), define the header here to
+        // show the 'Explain' button on the client error ShowAlert
+        if (this.minisrv_config.services[request_type]['wtv-explanation']) {
+            if (this.minisrv_config.services[request_type]['wtv-explanation'][res.statusCode]) {
+                headers['wtv-explanation-url'] = this.minisrv_config.services[request_type]['wtv-explanation'][res.statusCode];
+            }
+        } else if (this.minisrv_config.services['http']['wtv-explanation']) {
+            if (this.minisrv_config.services['http']['wtv-explanation'][res.statusCode]) {
+                headers['wtv-explanation-url'] = this.minisrv_config.services['http']['wtv-explanation'][res.statusCode];
+            }            
+        }
+        let data_hex = Buffer.concat(data).toString('hex');
+        if (data_hex.startsWith("0d0a0d0a")) data_hex = data_hex.slice(8);
+        if (data_hex.startsWith("0a0d0a")) data_hex = data_hex.slice(6);
+        if (data_hex.startsWith("0a0a")) data_hex = data_hex.slice(4);
+        this.sendToClient(socket, headers, Buffer.from(data_hex, 'hex'));
+    }
+
+}
+
+module.exports = WTVHTTP;

zefie_wtvp_minisrv/includes/classes/WTVPNM.js

@@ -30,7 +30,7 @@ class WTVPNM {
     wtvshared = null;
     sessions = new Map();
 
-    constructor(minisrv_config, service_name = 'pnm') {
+    constructor(...[minisrv_config, service_name]) {
         this.minisrv_config = minisrv_config;
         this.service_name = service_name;
         this.service_config = minisrv_config.services[service_name] || {};

zefie_wtvp_minisrv/includes/config.json

@@ -301,7 +301,7 @@
     },
     "ftp": {
       "port": 1650,
-      "connections": 3      
+      "connections": 3   
     },
     "http": {
       // http upstream
@@ -312,9 +312,13 @@
       "external_proxy_host": "127.0.0.1", // IP address or hostname of proxy
       "external_proxy_port": 1080, // Port of proxy
       "flags": "0x00000001",
-      "max_response_size": 16 // Megabytes
+      "max_response_size": 16, // Megabytes
+      "disallow_no_slash": true,
+      "handler_module": "WTVHTTP",
+      "handler_extra_vars": ["http", "sendToClient"]
     },
     "https": {
+      // https upstream
       "port": 1650,
       "connections": 3,
       "use_external_proxy": false, // use an external proxy (WebONE or some other minifying proxy is recommended)
@@ -322,9 +326,14 @@
       "external_proxy_host": "127.0.0.1", // IP address or hostname of proxy
       "external_proxy_port": 1080, // Port of proxy
       "flags": "0x00000001",
-      "max_response_size": 16 // Megabytes
+      "max_response_size": 16, // Megabytes
+      "disallow_no_slash": true,
+      "handler_module": "WTVHTTP",
+      "handler_extra_vars": ["http", "sendToClient"],
+      "allow_self_signed_ssl": true // If true, will allow self-signed SSL certificates via the proxy.
     },
     "proto": {
+      // ProtoWeb Proxy
       "port": 1650,
       "connections": 3,
       "use_external_proxy": true,
@@ -334,8 +343,22 @@
       "external_proxy_port": 7851,
       "external_proxy_is_http1": true,
       "flags": "0x00000001",
-      "max_response_size": 16 // Megabytes
+      "max_response_size": 16, // Megabytes
-    },        
+      "disallow_no_slash": true,
+      "handler_module": "WTVHTTP",
+      "handler_extra_vars": ["http", "sendToClient"]
+    },
+    "gopher": {
+      // Gopher processor
+      "port": 1651,
+      "connections": 3,
+      "allow_double_slash": true,
+      "flags": "0x00000001",
+      "uses_service_vault": false, // For custom services that use modules instead of service vaults
+      "disallow_no_slash": true,
+      "log_raw_gopher": false, // set to true to log raw gopher responses to the console for debugging
+      "handler_module": "WTVGopher"
+    },
     "wtv-passport": {
       // wtv-passport (for messenger)
       "port": 1654
@@ -404,7 +427,8 @@
       "descriptor_after_hello_ms": 85,
       "burst_prestart_ms": 5000,
       "debug": false,
-      "allow_indexing": true
+      "allow_indexing": true,
+      "handler_module": "WTVPNM"
     }
   },
   "favorites": {

zefie_wtvp_minisrv/ra_win/build.cmd

@@ -1,4 +1,4 @@
 @echo OFF
 cl rpcli.cpp /O2 /MT /Fe:rpcli.exe
-del rpcli.obj prct3260.thl
+del rpcli.obj prct3260.tlh