zefie/minisrv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more client_emu work

Browse Source
This commit is contained in:
zefie
2025-08-10 16:03:33 -04:00
parent 3378fa980b
commit 26bd129e38
3 changed files with 503 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

484
zefie_wtvp_minisrv/client_emu.js
View File

@@ -4,6 +4,7 @@ require(classPath + "Prototypes.js");
const WTVSec = require(classPath + "WTVSec.js");
const WTVShared = require(classPath + "/WTVShared.js")['WTVShared'];
const LZPF = require(classPath + "/LZPF.js");
const WTVMime = require(classPath + "/WTVMime.js");
const net = require('net');
const crypto = require('crypto');
@@ -18,7 +19,7 @@ const AdmZip = require('adm-zip');
* using the WTVP protocol with proper authentication and service discovery.
*/
class WebTVClientSimulator {
constructor(host, port, ssid, url, outputFile = null, maxRedirects = 10, useEncryption = false, request_type_download = false, debug = false, tricks = false, followImages = false, followAll = false, maxDepth = 5, maxRetries = 5) {
constructor(host, port, ssid, url, outputFile = null, maxRedirects = 10, useEncryption = false, request_type_download = false, debug = false, tricks = false, followImages = false, followAll = false, maxDepth = 5, maxRetries = 5, requestDelay = 250, boxType = null, username = null) {
this.host = host;
this.port = port;
this.ssid = ssid;
@@ -29,7 +30,9 @@ class WebTVClientSimulator {
this.followAll = followAll;
this.maxDepth = maxDepth;
this.maxRetries = maxRetries;
this.requestDelay = requestDelay;
this.currentDepth = 0;
this.authenticated = false;
this.downloadedUrls = new Set(); // Track what we've already downloaded
this.pendingDownloads = []; // Queue of {url, referrer} objects to download
this.allContent = new Map(); // Store all downloaded content
@@ -55,7 +58,9 @@ class WebTVClientSimulator {
this.hasSeenEncryptedResponse = false; // Track if we've seen an encrypted response
this.previousUrl = null; // Store previous URL for Referer header
this.debug = debug;
this.defaultBox = "plus";
this.connectSessionId = Math.random().toString(16).substr(2, 8).padEnd(8, '0');
this.username = username;
// Load minisrv config to get the initial shared key
this.minisrv_config = this.wtvshared.readMiniSrvConfig(true, false);
@@ -67,6 +72,122 @@ class WebTVClientSimulator {
if (outputFile) {
this.debugLog(`Output file: ${outputFile}`);
}
this.boxConfigs = {
"classic": [
"wtv-system-version: 1235",
"wtv-capability-flags: 1009c93bef",
"wtv-client-bootrom-version: 105",
"wtv-client-rom-type: bf0app",
"wtv-system-chipversion: 16842752",
"User-Agent: Mozilla/4.0 WebTV/1.4.2 (compatible; MSIE 3.0)",
"wtv-system-cpuspeed: 112790760",
"wtv-system-sysconfig: 736935823"
],
"plus": [
"wtv-system-version: 7181",
"wtv-capability-flags: 10935ffc8f",
"wtv-client-bootrom-version: 2046",
"wtv-client-rom-type: US-LC2-disk-0MB-8MB",
"wtv-system-chipversion: 51511296",
"User-Agent: Mozilla/4.0 WebTV/2.2.6.1 (compatible; MSIE 4.0)",
"wtv-system-cpuspeed: 166187148",
"wtv-system-sysconfig: 4163328",
"wtv-disk-size: 8006"
],
"derby": [
"wtv-system-version: 7253",
"wtv-capability-flags: f1d9bdfefef",
"wtv-client-bootrom-version: 2243",
"wtv-client-rom-type: US-LC2-disk-0MB-8MB-softmodem-CPU5230",
"wtv-system-chipversion: 53608448",
"User-Agent: Mozilla/4.0 WebTV/2.2.6.1 (compatible; MSIE 4.0)",
"wtv-system-cpuspeed: 166164434",
"wtv-system-sysconfig: 3115520",
"wtv-disk-size: 8006"
],
"echostar": [
"wtv-system-version: 17015",
"wtv-capability-flags: 21816935fec8f",
"wtv-client-rom-type: US-WEBSTAR-disk-0MB-16MB-softmodem-CPU5230",
"wtv-client-bootrom-version: 2524",
"wtv-system-chipversion: 53608448",
"wtv-system-sysconfig: 3130128",
"wtv-system-cpuspeed: 166164662",
"User-Agent: Mozilla/4.0 WebTV/2.8.2 (compatible; MSIE 4.0)"
],
"newclassic": [
"wtv-system-version: 5792",
"wtv-capability-flags: 5499dbafef",
"wtv-client-bootrom-version: 2525",
"wtv-client-rom-type: US-BPS-flashdisk-0MB-8MB-softmodem-CPU5230",
"wtv-system-chipversion: 84017152",
"User-Agent: Mozilla/4.0 WebTV/2.8.2 (compatible; MSIE 4.0)",
"wtv-system-cpuspeed: 148141518",
"wtv-system-sysconfig: 3133702",
"wtv-disk-size: 3990"
],
"newplus": [
"wtv-system-version: 5792",
"wtv-capability-flags: 5c9bdfefef",
"wtv-client-bootrom-version: 2525",
"wtv-client-rom-type: US-LC2-flashdisk-0MB-16MB-softmodem-CPU5230",
"wtv-system-chipversion: 53608448",
"User-Agent: Mozilla/4.0 WebTV/2.8.2 (compatible; MSIE 4.0)",
"wtv-system-cpuspeed: 166330740",
"wtv-system-sysconfig: 3116068",
"wtv-disk-size: 8006"
],
"ultimatetv": [
"wtv-system-version: 28220",
"wtv-capability-flags: 6f217b935dec8e",
"wtv-client-bootrom-version: 2545",
"wtv-client-rom-type: US-DTV-disk-0MB-32MB-softmodem-CPU5230",
"wtv-system-chipversion: 0x04120000",
"User-Agent: Mozilla/4.0 WebTV/2.8.2 (compatible; MSIE 4.0)",
"wtv-system-cpuspeed: 249088032",
"wtv-system-sysconfig: 0x034dea33",
"wtv-disk-size: 156330720"
],
"dreamcast": [
"wtv-system-version: 5254",
"wtv-capability-flags: 19d4928cf",
"wtv-client-bootrom-version: 5254",
"wtv-client-rom-type: JP-Fiji",
"User-Agent: Mozilla/4.0 WebTV/2.8.2 (compatible; MSIE 4.0)"
]
};
this.defaultBoxConfig = this.boxConfigs[this.defaultBox];
this.boxType = this.boxConfigs[boxType] || this.defaultBoxConfig;
}
getBoxConfig(box) {
// Aliases
switch (box) {
case "bf0":
case "bf0app":
return this.boxConfigs["classic"];
case "webstar":
case "dishplayer":
return this.boxConfigs["echostar"];
case "lc2":
case "lucy":
return this.boxConfigs["plus"];
case "lc2.5":
return this.boxConfigs["newplus"];
case "utv":
return this.boxConfigs["ultimatetv"];
case "dc":
return this.boxConfigs["dreamcast"];
}
return this.boxConfigs[box] || this.defaultBoxConfig;
}
getBoxHeaders(box) {
const config = this.getBoxConfig(box);
console.log(config);
return config.join("\r\n")+"\r\n";
}
debugLog(...args) {
@@ -140,10 +261,16 @@ class WebTVClientSimulator {
/**
* Make a WTVP request to a service
*/
async makeRequest(serviceName, path, data = null, skipRedirects = false) {
async makeRequest(serviceName, path, data = null, skipRedirects = false, referrerUrl = null) {
return new Promise((resolve, reject) => {
const currentUrl = `${serviceName}:${path}`;
// Prevent requests to client: URLs
if (serviceName.startsWith('client')) {
this.debugLog(`Blocking request to client: URL: ${currentUrl}`);
return;
}
// Increment incarnation for each new request (like real WebTV client)
this.incarnation++;
this.debugLog(`Using incarnation: ${this.incarnation} for ${serviceName}:${path}`);
@@ -196,9 +323,10 @@ class WebTVClientSimulator {
let requestData;
if (this.encryptionEnabled && this.wtvsec) {
if (this.encryptionEnabled && this.wtvsec && this.authenticated) {
// For encrypted requests, first send SECURE ON, then immediately send the encrypted request
// This matches the real WebTV client behavior seen in packet captures
// Only send SECURE ON after successful authentication
this.debugLog('Sending SECURE ON request...');
const secureOnBuffer = this.buildSecureOnRequest();
socket.write(secureOnBuffer);
@@ -206,12 +334,12 @@ class WebTVClientSimulator {
// Send encrypted request immediately after (as seen in pcap analysis)
setImmediate(() => {
this.debugLog('Sending encrypted request...');
const encryptedRequestData = this.buildEncryptedRequest(serviceName, path, data);
const encryptedRequestData = this.buildEncryptedRequest(serviceName, path, data, referrerUrl);
socket.write(encryptedRequestData);
});
} else {
// Send regular request
requestData = this.buildRegularRequest(serviceName, path, data);
requestData = this.buildRegularRequest(serviceName, path, data, referrerUrl);
this.debugLog('Sending request:');
this.debugLog(requestData.toString());
socket.write(requestData);
@@ -315,23 +443,19 @@ class WebTVClientSimulator {
/**
* Build a regular (unencrypted) WTVP request
*/
buildRegularRequest(serviceName, path, data = null) {
buildRegularRequest(serviceName, path, data = null, referrerUrl = null) {
const method = data ? 'POST' : 'GET';
let request = `${method} ${serviceName}:${path}\r\n`;
// Add Referer header if we have a previous URL
if (this.previousUrl) {
request += `Referer: ${this.previousUrl}\r\n`;
// Add Referer header - prefer explicit referrerUrl, fallback to previousUrl
const refererToUse = referrerUrl || this.previousUrl;
if (refererToUse) {
request += `Referer: ${refererToUse}\r\n`;
}
// Add required headers (matching real WebTV client from PCAP)
request += `wtv-request-type: ${((this.request_type_download) ? 'download' : 'primary')}\r\n`;
request += `wtv-client-serial-number: ${this.ssid}\r\n`;
request += `wtv-client-bootrom-version: 2046\r\n`;
request += `wtv-client-rom-type: US-LC2-disk-0MB-8MB\r\n`;
request += `wtv-system-cpuspeed: 166187148\r\n`;
request += `wtv-system-sysconfig: 4163328\r\n`;
request += `wtv-disk-size: 8006\r\n`;
request += `Accept-Language: en\r\n`;
request += `wtv-incarnation: ${this.incarnation}\r\n`;
// Generate a random 8 character (4 byte) hex code for wtv-connect-session-id
@@ -339,12 +463,10 @@ class WebTVClientSimulator {
request += `wtv-connect-session-id: ${this.connectSessionId}\r\n`
// Add additional headers that real client sends (from PCAP analysis)
request += `User-Agent: Mozilla/4.0 WebTV/2.2.6.1 (compatible; MSIE 4.0)\r\n`;
request += `wtv-system-version: 7181\r\n`;
request += `wtv-capability-flags: 10935ffc8f\r\n`;
request += `wtv-system-chipversion: 51511296\r\n`;
if (this.useEncryption) request += `wtv-encryption: true\r\n`;
if (!this.challengeResponse) request += `wtv-script-id: -1896417432\r\n`;
if (!this.challengeResponse) request += `wtv-script-mod: 1754789923\r\n`;
request += this.getBoxHeaders(this.boxType);
request += `wtv-client-address: 0.0.0.0\r\n`;
// Add challenge response if we have one
@@ -384,12 +506,7 @@ class WebTVClientSimulator {
}
request += `wtv-connect-session-id: ${Math.random().toString(16).substr(2, 8)}\r\n`;
request += `wtv-client-serial-number: ${this.ssid}\r\n`;
request += `wtv-system-version: 7181\r\n`;
request += `wtv-capability-flags: 10935ffc8f\r\n`;
request += `wtv-client-bootrom-version: 2046\r\n`;
request += `wtv-client-rom-type: US-LC2-disk-0MB-8MB\r\n`;
request += `wtv-system-chipversion: 51511296\r\n`;
request += `User-Agent: Mozilla/4.0 WebTV/2.2.6.1 (compatible; MSIE 4.0)\r\n`;
request += this.getBoxHeaders(this.boxType);
request += `wtv-encryption: true\r\n`;
request += `wtv-script-id: -154276969\r\n`;
request += `wtv-script-mod: ${Math.floor(Date.now() / 1000)}\r\n`;
@@ -402,13 +519,14 @@ class WebTVClientSimulator {
/**
* Build an encrypted WTVP request
*/
buildEncryptedRequest(serviceName, path, data = null) {
buildEncryptedRequest(serviceName, path, data = null, referrerUrl = null) {
const method = data ? 'POST' : 'GET';
let request = `${method} ${serviceName}:${path}\r\n`;
// Add Referer header if we have a previous URL
if (this.previousUrl) {
request += `Referer: ${this.previousUrl}\r\n`;
// Add Referer header - prefer explicit referrerUrl, fallback to previousUrl
const refererToUse = referrerUrl || this.previousUrl;
if (refererToUse) {
request += `Referer: ${refererToUse}\r\n`;
}
// For encrypted requests, only include the minimal necessary headers
// The SECURE ON already sent the auth and session info
@@ -572,7 +690,7 @@ class WebTVClientSimulator {
this.followVisit(redirectUrl)
.then(resolve)
.catch(reject);
}, 100);
}, this.requestDelay);
return true; // Redirect is being followed
}
@@ -639,8 +757,58 @@ class WebTVClientSimulator {
}
this.processHeaders(headers);
this.debugLog("srv headers:", headers);
// Decompress the body if needed
bodyBuf = this.decompressBody(bodyBuf, headers);
// Special handler for wtv-head-waiter:/login-stage-two
if (currentUrl && currentUrl.startsWith('wtv-head-waiter:/login-stage-two')) {
const contentLength = headers['content-length'];
if (contentLength && parseInt(contentLength) > 0) {
this.debugLog('Special handling for wtv-head-waiter:/login-stage-two with content-length > 0');
this.debugLog(`Content-Length: ${contentLength}`);
// Decrypt the content if we have encryption enabled
if (this.wtvsec && bodyBuf.length > 0) {
try {
this.debugLog('Decrypting login-stage-two content...');
this.wtvsec.set_incarnation(this.incarnation); // Ensure WTVSec has the correct incarnation
const decryptedBuffer = this.wtvsec.Decrypt(1, bodyBuf);
bodyBuf = Buffer.from(decryptedBuffer);
this.debugLog(`Content decrypted successfully: ${bodyBuf.length} bytes`);
// Re-decompress after decryption in case it was compressed
bodyBuf = this.decompressBody(bodyBuf, headers);
this.debugLog(`Final content after decrypt+decompress: ${bodyBuf.length} bytes`);
} catch (error) {
console.error('Error decrypting login-stage-two content:', error);
}
}
// Parse the HTML to extract usernames and their href links
if (bodyBuf.length > 0) {
const parseResult = this.parseLoginStageTwoHTML(bodyBuf);
if (parseResult.selectedUser) {
// User was specified and found, automatically follow their link
this.debugLog(`Following link for user: ${parseResult.selectedUser.username}`);
setTimeout(() => {
this.followVisit(parseResult.selectedUser.href)
.then(resolve)
.catch(reject);
}, this.requestDelay);
return; // Exit early to follow the user's link
}
// If we get here, either no username was specified or there was an error
// The parseLoginStageTwoHTML method already handles displaying users and exiting
}
}
} else {
if (this.encryptionEnabled && bodyBuf.length > 0) {
this.debugLog('Decrypting response body...');
bodyBuf = this.wtvsec.Decrypt(1, bodyBuf);
}
// Decompress the body if needed
bodyBuf = this.decompressBody(bodyBuf, headers);
}
// Mark that we've seen an encrypted response
if (headers['wtv-encrypted'] === 'true') {
@@ -676,10 +844,10 @@ class WebTVClientSimulator {
this.fetchTargetUrl()
.then(resolve)
.catch(reject);
}, 100);
}, this.requestDelay);
return;
}
if ((headers['wtv-visit'] || headers['Location']) && !skipRedirects) {
if ((headers['wtv-visit'] || headers['location']) && !skipRedirects) {
if (this.redirectCount >= this.maxRedirects) {
// Check if we can use tricks access for one last redirect
if (this.useTricksAccess && !this.tricksAccessUsed) {
@@ -687,10 +855,10 @@ class WebTVClientSimulator {
this.tricksAccessUsed = true;
this.redirectCount++;
setTimeout(() => {
this.followVisit(headers['wtv-visit'] || headers['Location'])
this.followVisit(headers['wtv-visit'] || headers['location'])
.then(resolve)
.catch(reject);
}, 100);
}, this.requestDelay);
return;
} else {
this.debugLog(`Maximum redirects (${this.maxRedirects}) reached, stopping`);
@@ -700,17 +868,17 @@ class WebTVClientSimulator {
}
this.redirectCount++;
setTimeout(() => {
this.followVisit(headers['wtv-visit'] || headers['Location'])
this.followVisit(headers['wtv-visit'] || headers['location'])
.then(resolve)
.catch(reject);
}, 100);
}, this.requestDelay);
} else if (headers['wtv-phone-log-url'] && headers['wtv-phone-log-url'].includes("post")) {
this.debugLog(`Following wtv-phone-log-url: ${headers['wtv-phone-log-url']}`);
setTimeout(() => {
this.followVisit(headers['wtv-phone-log-url'], true)
.then(resolve)
.catch(reject);
}, 100);
}, this.requestDelay);
} else {
if (skipRedirects && headers['wtv-visit']) {
// Check if we can use tricks access for one last redirect
@@ -722,13 +890,15 @@ class WebTVClientSimulator {
this.followVisit(headers['wtv-visit'])
.then(resolve)
.catch(reject);
}, 100);
}, this.requestDelay);
return;
} else {
this.debugLog(`Skipping wtv-visit redirect: ${headers['wtv-visit']}`);
}
} else {
this.debugLog('No wtv-visit header found, resolving...');
this.cleanup();
process.exit(0);
}
resolve({ headers, body: bodyBuf, status: statusLine });
}
@@ -803,19 +973,67 @@ class WebTVClientSimulator {
this.incarnation = parseInt(headers["wtv-incarnation"]);
}
// Use the server's initial key for challenge processing
const keyToUse = this.initial_key ? CryptoJS.enc.Base64.parse(this.initial_key) : this.wtvsec.current_shared_key;
// Use the appropriate key for challenge processing
// For subsequent challenges (like during user login), use the current shared key
// For the first challenge, use the initial key if provided
let keyToUse = this.wtvsec.current_shared_key || this.initial_key
this.debugLog(`Using key for challenge: ${keyToUse.toString(CryptoJS.enc.Base64)}`);
this.wtvsec.set_incarnation(this.incarnation);
this.debugLog(`Using incarnation for challenge: ${this.wtvsec.incarnation}`);
const challengeResponse = this.wtvsec.ProcessChallenge(headers['wtv-challenge'], keyToUse);
if (challengeResponse && challengeResponse.toString(CryptoJS.enc.Base64)) {
this.debugLog('Challenge processed successfully, preparing response');
this.debugLog(`Challenge response: ${challengeResponse.toString(CryptoJS.enc.Base64)}`);
// We'll send the challenge response in the next request
this.challengeResponse = challengeResponse.toString(CryptoJS.enc.Base64);
this.debugLog('Setting wtv-challenge-response header for next request');
// Enable encryption preparation if requested (but don't enable encrypted communication yet)
if (this.useEncryption) {
this.debugLog('*** Encryption requested - preparing encryption after challenge processing ***');
this.wtvsec.SecureOn(); // Initialize RC4 sessions
// Note: this.encryptionEnabled will be set to true only after authentication
}
} else {
console.error('Failed to process challenge - no response generated');
this.debugLog('Challenge processing failed, attempting with different key...');
// Try with a different key if the first attempt failed
try {
let alternativeKey;
if (this.challengeResponse && this.initial_key) {
// Try with initial key if we used current shared key
alternativeKey = CryptoJS.enc.Base64.parse(this.initial_key);
this.debugLog('Retrying challenge with initial key');
} else if (!this.challengeResponse && this.wtvsec.current_shared_key) {
// Try with current shared key if we used initial key
alternativeKey = this.wtvsec.current_shared_key;
this.debugLog('Retrying challenge with current shared key');
}
if (alternativeKey) {
this.debugLog(`Retry key: ${alternativeKey.toString(CryptoJS.enc.Base64)}`);
const retryResponse = this.wtvsec.ProcessChallenge(headers['wtv-challenge'], alternativeKey);
if (retryResponse && retryResponse.toString(CryptoJS.enc.Base64)) {
this.debugLog('Challenge retry successful!');
this.challengeResponse = retryResponse.toString(CryptoJS.enc.Base64);
this.debugLog('Setting wtv-challenge-response header for next request');
// Enable encryption preparation if requested
if (this.useEncryption) {
this.debugLog('*** Encryption requested - preparing encryption after challenge retry ***');
this.wtvsec.SecureOn(); // Initialize RC4 sessions
}
} else {
console.error('Challenge retry also failed');
}
}
} catch (retryError) {
console.error('Challenge retry failed:', retryError.message);
}
}
} catch (error) {
console.error('Error processing challenge:', error.message);
@@ -834,18 +1052,18 @@ class WebTVClientSimulator {
this.debugLog(`*** Authentication successful! user-id detected: ${headers['user-id']} ***`);
this.userIdDetected = true;
// Enable encryption if requested and we have WTVSec
if (this.useEncryption) {
this.debugLog('*** Enabling encryption after successful authentication ***');
if (!this.wtvsec) {
// Initialize with current incarnation (which was incremented when we got wtv-encrypted: true)
this.wtvsec = new WTVSec(this.minisrv_config, this.incarnation);
}
// Enable encrypted communication if requested and we have WTVSec
if (this.useEncryption && this.wtvsec) {
this.debugLog('*** Enabling encrypted communication after successful authentication ***');
this.encryptionEnabled = true;
} else if (this.useEncryption && !this.wtvsec) {
this.debugLog('*** Encryption requested but no WTVSec instance - initializing ***');
// Initialize with current incarnation
this.wtvsec = new WTVSec(this.minisrv_config, this.incarnation);
this.wtvsec.SecureOn(); // Initialize RC4 sessions
this.encryptionEnabled = true;
}
this.authenticated = true;
return; // Stop processing other headers since we're authenticated
}
}
@@ -892,6 +1110,11 @@ class WebTVClientSimulator {
if (result.body) {
this.debugLog('\n*** Target URL Response Body ***');
if (this.outputFile) {
// Check if target URL returned a download-list and --follow is enabled
const contentType = result.headers['content-type'] || '';
const normalizedContentType = contentType.toLowerCase().split(';')[0].trim();
const isDownloadList = normalizedContentType === 'wtv/download-list';
if (this.followAll) {
// Store the main content first
this.storeContent(this.url, result);
@@ -901,6 +1124,9 @@ class WebTVClientSimulator {
// Create comprehensive archive
await this.createComprehensiveArchive();
} else if (this.followImages && isDownloadList) {
this.debugLog('Target URL returned download-list content with --follow enabled, creating archive...');
await this.createDownloadListArchive(result.body, result.headers);
} else if (this.followImages) {
await this.saveToFile(result.body, result.headers);
} else {
@@ -1054,7 +1280,7 @@ class WebTVClientSimulator {
const downloadedImages = new Set();
for (const imageUrl of imageUrls) {
try {
const imageResult = await this.downloadImage(imageUrl);
const imageResult = await this.downloadImage(imageUrl, this.url);
if (imageResult && imageResult.body && !downloadedImages.has(imageUrl)) {
const imagePath = this.getServicePath(imageUrl, imageResult.headers || {});
zip.addFile(imagePath, imageResult.body);
@@ -1124,7 +1350,7 @@ class WebTVClientSimulator {
}
// Small delay to avoid overwhelming the server
await new Promise(resolve => setTimeout(resolve, 100));
await new Promise(resolve => setTimeout(resolve, this.requestDelay));
} catch (error) {
console.warn(`Failed to download referenced file ${fileUrl}: ${error.message}`);
}
@@ -1211,7 +1437,7 @@ class WebTVClientSimulator {
/**
* Download an image from a WebTV service URL
*/
async downloadImage(imageUrl) {
async downloadImage(imageUrl, referrerUrl = null) {
this.debugLog(`Downloading image: ${imageUrl}`);
try {
@@ -1225,7 +1451,7 @@ class WebTVClientSimulator {
const path = '/' + (match[2] || '');
// Make request to download the image
const result = await this.makeRequestWithRetry(serviceName, path, null, true); // Skip redirects for images
const result = await this.makeRequestWithRetry(serviceName, path, null, true, referrerUrl); // Skip redirects for images
if (result.body && result.body.length > 0) {
this.debugLog(`Downloaded image: ${imageUrl} (${result.body.length} bytes)`);
@@ -1258,9 +1484,9 @@ class WebTVClientSimulator {
if (this.currentDepth < this.maxDepth) {
const newUrls = this.extractAllUrls(response.body, response.headers, url);
for (const newUrl of newUrls) {
if (!this.downloadedUrls.has(newUrl) && !this.pendingDownloads.includes(newUrl)) {
this.pendingDownloads.push(newUrl);
this.debugLog(`Queued for download: ${newUrl}`);
if (!this.downloadedUrls.has(newUrl) && !this.pendingDownloads.some(item => item.url === newUrl)) {
this.pendingDownloads.push({ url: newUrl, referrer: url });
this.debugLog(`Queued for download: ${newUrl} (referrer: ${url})`);
}
}
}
@@ -1596,15 +1822,15 @@ class WebTVClientSimulator {
/**
* Make request with retry logic for ECONNREFUSED errors
*/
async makeRequestWithRetry(serviceName, path, postData = null, downloadMode = false, retryCount = 0) {
async makeRequestWithRetry(serviceName, path, postData = null, downloadMode = false, referrerUrl = null, retryCount = 0) {
try {
return await this.makeRequest(serviceName, path, postData, downloadMode);
return await this.makeRequest(serviceName, path, postData, downloadMode, referrerUrl);
} catch (error) {
if (error.code === 'ECONNREFUSED' && retryCount < this.maxRetries) {
const retryDelay = 5000; // 5 seconds
this.debugLog(`Connection refused, retrying in ${retryDelay/1000}s (attempt ${retryCount + 1}/${this.maxRetries})`);
await new Promise(resolve => setTimeout(resolve, retryDelay));
return this.makeRequestWithRetry(serviceName, path, postData, downloadMode, retryCount + 1);
return this.makeRequestWithRetry(serviceName, path, postData, downloadMode, referrerUrl, retryCount + 1);
}
throw error; // Re-throw if not ECONNREFUSED or max retries exceeded
}
@@ -1623,18 +1849,19 @@ class WebTVClientSimulator {
this.debugLog(`\n--- Processing depth ${this.currentDepth} (${currentBatch.length} URLs) ---`);
for (const url of currentBatch) {
for (const item of currentBatch) {
const { url, referrer } = item;
if (this.downloadedUrls.has(url)) {
continue; // Skip if already downloaded
}
try {
this.debugLog(`Downloading: ${url}`);
this.debugLog(`Downloading: ${url} (referrer: ${referrer})`);
const match = url.match(/^([\w-]+):\/?(.*)/);
if (match) {
const serviceName = match[1];
const path = '/' + (match[2] || '');
const result = await this.makeRequestWithRetry(serviceName, path, null, true);
const result = await this.makeRequestWithRetry(serviceName, path, null, true, referrer);
this.storeContent(url, result);
}
} catch (error) {
@@ -1642,7 +1869,7 @@ class WebTVClientSimulator {
}
// Small delay to avoid overwhelming the server
await new Promise(resolve => setTimeout(resolve, 50));
await new Promise(resolve => setTimeout(resolve, this.requestDelay));
}
}
@@ -1889,6 +2116,113 @@ class WebTVClientSimulator {
return `content_${hash}${ext}`;
}
/**
* Parse login-stage-two HTML to extract usernames and their respective href links
*/
parseLoginStageTwoHTML(htmlContent) {
const userList = [];
try {
const htmlString = Buffer.isBuffer(htmlContent) ? htmlContent.toString('utf8') : htmlContent;
this.debugLog('Parsing login-stage-two HTML for usernames and links...');
// Pattern to match user entries in the HTML
// Looking for <a href="/ValidateLoginName-..." followed by username in <font> tags
// Note: Some usernames have <b> tags, others don't, so we make <b> optional
const userPattern = /<a\s+href="([^"]*ValidateLoginName[^"]*)"[^>]*>[\s\S]*?<font[^>]*color="#FFEA9C"[^>]*>(?:<b>)?([^<]+?)(?:<\/b>)?<\/font>/gi;
let match;
while ((match = userPattern.exec(htmlString)) !== null) {
if (match[1].slice(0,1) === "/") {
match[1] = "wtv-head-waiter:" + match[1];
}
const href = match[1];
const username = match[2].trim();
if (username && href) {
userList.push({
username: username,
href: href
});
this.debugLog(`Found user: ${username} -> ${href}`);
}
}
// Alternative pattern in case the first one doesn't catch all cases
// Look for any ValidateLoginName links and try to find nearby usernames
if (userList.length === 0) {
this.debugLog('Primary pattern found no users, trying alternative pattern...');
const linkPattern = /href="([^"]*ValidateLoginName[^"]*)"/gi;
const fontPattern = /<font[^>]*color="#FFEA9C"[^>]*>(?:<b>)?([^<]+?)(?:<\/b>)?<\/font>/gi;
const links = [];
const usernames = [];
let linkMatch;
while ((linkMatch = linkPattern.exec(htmlString)) !== null) {
if (linkMatch[1].slice(0,1) === "/") {
linkMatch[1] = "wtv-head-waiter:" + linkMatch[1];
}
links.push(linkMatch[1]);
}
let fontMatch;
while ((fontMatch = fontPattern.exec(htmlString)) !== null) {
usernames.push(fontMatch[1].trim());
}
// Try to pair them up (assuming they appear in the same order)
const minLength = Math.min(links.length, usernames.length);
for (let i = 0; i < minLength; i++) {
userList.push({
username: usernames[i],
href: links[i]
});
this.debugLog(`Found user (alternative): ${usernames[i]} -> ${links[i]}`);
}
}
this.debugLog(`Parsed ${userList.length} users from login-stage-two HTML`);
if (this.username) {
// Find the specified user
const selectedUser = userList.find(user => user.username.toLowerCase() === this.username.toLowerCase());
if (selectedUser) {
this.debugLog(`*** Selecting user: ${selectedUser.username} ***`);
console.log(`Selecting user: ${selectedUser.username}`);
// Return the user list with the selected user marked for automatic following
return { userList, selectedUser };
} else {
console.error(`\nUser '${this.username}' not found. Available users: ${userList.map(user => user.username).join(', ')}`);
this.cleanup();
process.exit(1);
}
} else {
// No username specified, list available users and exit
if (userList.length > 0) {
console.log('\n*** Available users from login-stage-two ***');
userList.forEach((user, index) => {
console.log(`${index + 1}. ${user.username} -> ${user.href}`);
});
console.log('*** End of user list ***\n');
console.error(`Please select a --user: ${userList.map(user => user.username).join(' ')}`);
this.cleanup();
process.exit(1);
} else {
console.error('No users found in login-stage-two HTML');
this.cleanup();
process.exit(1);
}
}
return { userList };
} catch (error) {
console.error('Error parsing login-stage-two HTML:', error);
return [];
}
}
/**
* Clean up resources
*/
@@ -1926,7 +2260,9 @@ function parseArgs() {
followAll: false,
maxDepth: 3,
maxRetries: 5,
debug: false
requestDelay: 250,
debug: false,
username: null
};
for (let i = 0; i < args.length; i++) {
@@ -1936,6 +2272,11 @@ function parseArgs() {
config.host = args[++i];
}
break;
case '--boxtype':
if (i + 1 < args.length) {
config.boxType = args[++i];
}
break;
case '--port':
if (i + 1 < args.length) {
config.port = parseInt(args[++i]);
@@ -1991,6 +2332,16 @@ function parseArgs() {
config.maxRetries = parseInt(args[++i]);
}
break;
case '--delay':
if (i + 1 < args.length) {
config.requestDelay = parseInt(args[++i]);
}
break;
case '--user':
if (i + 1 < args.length) {
config.username = args[++i];
}
break;
case '--help':
console.log(`
WebTV Client Simulator
@@ -2001,7 +2352,9 @@ Options:
--host <ip> Target server IP address (default: 127.0.0.1)
--port <port> Target server port (default: 1615)
--ssid <ssid> WebTV client SSID (default: 8100000000000001)
--boxtype <type> Set the WebTV box type (default: ${this.defaultBox})
--url <url> Target URL to fetch after authentication (default: wtv-home:/home)
--user <username> Seelect username during login-stage-two (required if the account has multiple users)
--file <filename> Save response body to file instead of echoing to CLI
--max-redirects <num> Maximum number of wtv-visit redirects (default: 10)
--dl-mode Enable 'wtv-request-type: download' for diskmap testing on minisrv
@@ -2011,6 +2364,7 @@ Options:
--follow-all Aggressively download everything encountered (spider mode)
--depth <num> Maximum crawl depth for --follow-all mode (default: 5)
--retries <num> Maximum number of retries for ECONNREFUSED errors (default: 5)
--delay <num> Delay between requests in milliseconds (default: 250)
--debug Enable debug logging
--help Show this help message
@@ -2031,7 +2385,7 @@ Example:
*/
async function main() {
const config = parseArgs();
const simulator = new WebTVClientSimulator(config.host, config.port, config.ssid, config.url, config.outputFile, config.maxRedirects, config.useEncryption, config.request_type_download, config.debug, config.useTricksAccess, config.followImages, config.followAll, config.maxDepth, config.maxRetries);
const simulator = new WebTVClientSimulator(config.host, config.port, config.ssid, config.url, config.outputFile, config.maxRedirects, config.useEncryption, config.request_type_download, config.debug, config.useTricksAccess, config.followImages, config.followAll, config.maxDepth, config.maxRetries, config.requestDelay, config.boxType, config.username);
// Handle graceful shutdown
process.on('SIGINT', () => {

85
zefie_wtvp_minisrv/includes/ServiceVault/http_pc/viewergen/index.js
View File

@@ -329,6 +329,20 @@ function buildProfile(build) {
}
break;
case 7253:
buildProfile = {
"wtv-system-version": build,
"wtv-capability-flags": "f1d9bdfefef",
"wtv-client-bootrom-version": 2243,
"wtv-client-rom-type": "US-LC2-disk-0MB-8MB-softmodem-CPU5230",
"wtv-system-chipversion": 53608448,
"User-Agent": "Mozilla/4.0 WebTV/2.2.6.1 (compatible; MSIE 4.0)",
"wtv-system-cpuspeed": 166164434,
"wtv-system-sysconfig": 3115520,
"wtv-disk-size": 8006
}
break;
case 71810:
buildProfile = {
"wtv-capability-flags": "d10094938ef",
@@ -338,7 +352,7 @@ function buildProfile(build) {
"wtv-system-chipversion": 16842752,
"wtv-system-sysconfig": 736935823,
"wtv-system-cpuspeed": 112790760,
"User-Agent": "Mozilla/4.0 WebTV/2.5 (compatible; MSIE 4.0)",
"User-Agent": "Mozilla/4.0 WebTV/2.5 (compatible; MSIE 4.0)"
}
break;
case 16276:
@@ -354,6 +368,67 @@ function buildProfile(build) {
"wtv-disk-size": 8006
}
break;
case 17015:
buildProfile = {
"wtv-capability-flags": "21816935fec8f",
"wtv-system-version": 17015,
"wtv-client-rom-type": "US-WEBSTAR-disk-0MB-16MB-softmodem-CPU5230",
"wtv-client-bootrom-version": 2524,
"wtv-system-chipversion": 53608448,
"wtv-system-sysconfig": 3130128,
"wtv-system-cpuspeed": 166164662,
"User-Agent": "Mozilla/4.0 WebTV/2.8.2 (compatible; MSIE 4.0)"
}
break;
case 5792:
buildProfile = {
"wtv-system-version": build,
"wtv-capability-flags": "5c9bdfefef",
"wtv-client-bootrom-version": 2525,
"wtv-client-rom-type": "US-LC2-flashdisk-0MB-16MB-softmodem-CPU5230",
"wtv-system-chipversion": 53608448,
"User-Agent": "Mozilla/4.0 WebTV/2.8.2 (compatible; MSIE 4.0)",
"wtv-system-cpuspeed": 166330740,
"wtv-system-sysconfig": 3116068,
"wtv-disk-size": 8006
}
break;
case 57920:
buildProfile = {
"wtv-system-version": build,
"wtv-capability-flags": "5499dbafef",
"wtv-client-bootrom-version": 2525,
"wtv-client-rom-type": "US-BPS-flashdisk-0MB-8MB-softmodem-CPU5230",
"wtv-system-chipversion": 84017152,
"User-Agent": "Mozilla/4.0 WebTV/2.8.2 (compatible; MSIE 4.0)",
"wtv-system-cpuspeed": 148141518,
"wtv-system-sysconfig": 3133702,
"wtv-disk-size": 3990
}
break;
case 28220:
buildProfile = {
"wtv-system-version": build,
"wtv-capability-flags": "6f217b935dec8e",
"wtv-client-bootrom-version": 2545,
"wtv-client-rom-type": "US-DTV-disk-0MB-32MB-softmodem-CPU5230",
"wtv-system-chipversion": 0x04120000,
"User-Agent": "Mozilla/4.0 WebTV/2.8.2 (compatible; MSIE 4.0)",
"wtv-system-cpuspeed": 249088032,
"wtv-system-sysconfig": 0x034dea33,
"wtv-disk-size": 156330720
}
break;
case 5254:
buildProfile = {
"wtv-system-version": build,
"wtv-capability-flags": "19d4928cf",
"wtv-client-bootrom-version": 5254,
"wtv-client-rom-type": "JP-Fiji",
"User-Agent": "Mozilla/4.0 WebTV/2.8.2 (compatible; MSIE 4.0)"
}
break;
}
return buildProfile;
}
@@ -659,8 +734,14 @@ unless you are intentionally trying to spoof a box.</em>
<td><select name="build" id="build">
<option value="1235">Build 1235 (Old Classic)</option>
<option value="71810">Build 7181 (Old Classic)</option>
<option selected value="7181">Build 7181 (Old Plus)</option>
<option value="7253">Build 7253 (Old Plus Derby)</option>
<option selected value="7181">Build 7181 (Old Plus **Recommended**)</option>
<option value="16276">Build 16276 (Old Plus)</option>
<option value="17015">Build 17015 (Dish-Echostar)</option>
<option value="5792">Build 5792 (New Plus LC2.5)</option>
<option value="57920">Build 5792 (New Classic bps)</option>
<option value="28220">Build 28220 (Ultimate TV)</option>
<option value="5254">Sega DreamCast</option>
</select><br>
<em>This legacy option has little impact on minisrv servers,<br>
although certain advanced server operators may use these flags<br>

2
zefie_wtvp_minisrv/includes/ServiceVault/wtv-flashrom/big-willie.js
View File

@@ -24,7 +24,7 @@ var req = https.request(options, function(res) {
res.on('error', function (e) {
if (!minisrv_config.config.debug_flags.quiet) console.log(" * Upstream Big Willies HTTP Error:", e);
var errpage = wtvshared.doErrorPage(400)
var errpage = wtvshared.doErrorPage(400, "A required service is not responding. Please try again in a few moments.");
headers = errpage[0];
data = errpage[1];
sendToClient(socket, headers, data);