wip usenet stuff

zefie_wtvp_minisrv/ServiceDeps/wtv-news/localserver_ca.pem

@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID1TCCAr2gAwIBAgIUQzfMpYgezM0u/aOpBIYzCTSNuSgwDQYJKoZIhvcNAQEL
+BQAwejELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAk5ZMRcwFQYDVQQKDA5aZWZpZSBO
+ZXR3b3JrczEQMA4GA1UECwwHbWluaXNydjETMBEGA1UEAwwKbWluaXNydl9jYTEe
+MBwGCSqGSIb3DQEJARYPemVmaWVAemVmaWUubmV0MB4XDTIyMTAxMjAzMzQzOVoX
+DTIyMTExMTAzMzQzOVowejELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAk5ZMRcwFQYD
+VQQKDA5aZWZpZSBOZXR3b3JrczEQMA4GA1UECwwHbWluaXNydjETMBEGA1UEAwwK
+bWluaXNydl9jYTEeMBwGCSqGSIb3DQEJARYPemVmaWVAemVmaWUubmV0MIIBIjAN
+BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzo8+u/9kJcQY+mOKLxYNOtNSWqlu
+fwSKpo/kCMbCkM4TCJiaKHWwtXeT5tgrYPqg6s3nHpG2arwI/sxX6fy86s9i8bu5
+NEaO6/Czii0gQLXznURPaCPKp2dJEpcrFvqsB/Ej7Bm2cZH4BaOW9CeaMdrSXK1e
+Tlwem1XHPeFnIPGQr3sYp2CNcB3TWzNkGGDPB2CBcR+qB2Fnjb7aEcd5t4imJqT6
+BPTiDB234v3glcEaDBIr3bT/YGZuRn7Ou8qAVKlfT4iDsgHkpW+V7RYRocfAynPF
+mSxQf2i4hIfvK2mGhdpcQGH5WTOp6K9xIv/lGIZcb29xoAeICplq3/u24wIDAQAB
+o1MwUTAdBgNVHQ4EFgQUG8IOPoOiwIA1GI/3pANtftkD7BwwHwYDVR0jBBgwFoAU
+G8IOPoOiwIA1GI/3pANtftkD7BwwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B
+AQsFAAOCAQEAaG63vM/KbOfx/wzg8dZE6j/h8Ai0wJ1iXXvBLa23+fpZwfHIMH/A
+uvkap+OzX3oI8fdalReY11+WONfzA4pyuNPtluwkngSLjT8YSfHfM3MUM6LKt5x4
+rRNBNN74H8A7q5UDA/SD2M1HjcOkFNlH4XrE6Uy/ColCVV9cYGPdx1rKwiQiKK7Q
+Nr+6awAYHzQ2kdrZ4kneBZqAWAuIzjjWMxcgVwCHpb4rV0u3j/uhIos/8Q6NtLA7
+EqW0qMWmpJ9DlqfhLXqVCgfPyxGNpvTom0lw6J8OL8U6d+oaeVTZW44/3A8wNjJ+
+Gd2qQMxupodfV3LVyREvDpM+cowW6q+qHA==
+-----END CERTIFICATE-----

zefie_wtvp_minisrv/ServiceDeps/wtv-news/localserver_cert.pem

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDhjCCAm4CFCwnaIlHV8E1xd6rMWruYTncUoieMA0GCSqGSIb3DQEBCwUAMHox
+CzAJBgNVBAYTAlVTMQswCQYDVQQIDAJOWTEXMBUGA1UECgwOWmVmaWUgTmV0d29y
+a3MxEDAOBgNVBAsMB21pbmlzcnYxEzARBgNVBAMMCm1pbmlzcnZfY2ExHjAcBgkq
+hkiG9w0BCQEWD3plZmllQHplZmllLm5ldDAgFw0yMjEwMTIwMzM3MDlaGA8yMTIy
+MDkxODAzMzcwOVowgYIxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJOWTEXMBUGA1UE
+CgwObWluaXNydiB1c2VuZXQxFzAVBgNVBAsMDm1pbmlzcnYgdXNlbmV0MRcwFQYD
+VQQDDA5taW5pc3J2IHVzZW5ldDEbMBkGCSqGSIb3DQEJARYMdXNlckBtaW5pc3J2
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3lH9XWgXr1m8rLPS2gDV
+/b5vUFSRiK+2JViVJq8CpLYSKqEoI1l2t4Ui4gwejy/SemKvdXFfHDBUEzb4M9Cy
+nTzBQK72cuJ1hgEyXiqVFEA2Z3mLq5DQSstx5I+/8p+QN5V93reusq9ALEGqEWI4
+j9kj5vGQr4/t8Ce0onflpsoNDIng1uUBv4BRe/Gj4ewJpfwbKjHxxssUauQ0gJg9
+71hy5jz306G5j+8EtOSQtcrLKcbuctoZhzf240aD9IjRlZwAQLWrS5FUHl53dfsF
++kiU/rTDEuLx5fsMA0PKhEn+85IXlEiKWas8zptecAxhGaxRv5ndaNzw1EFkIL/l
+rQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBEzoRbk8D0a/ETOtQtnGpFFpMkxNkx
+b015xPLpQEnQ3/CWs8jUTJiLQdgL7BkmDx2CGVbONKYMr9YO3kzCn+wAiX1FD+NP
+9CMRj1XEkjxoImYv75rmyKtspNMFq9r/PiLwr0kRmAASnbYn5+15D68FTyxIw/zJ
+7M0jZmf/hW+QT1WVlvO31gwyGcWQFvJctYMjImtAOlZE6LJQI5T4Ld10L1EQyj2q
+l9XVM1HrggecgaSsdMxGIi02XZ/LV3gKLuTp7KJ/8YcjThQVOVS4wycmuGF0I71t
+IpZVAjDxH/pzL79uyW8oEzkhF4H0QDA5I2cd4h6kGPeOXH76JvdhlQ5Q
+-----END CERTIFICATE-----

zefie_wtvp_minisrv/ServiceDeps/wtv-news/localserver_key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA3lH9XWgXr1m8rLPS2gDV/b5vUFSRiK+2JViVJq8CpLYSKqEo
+I1l2t4Ui4gwejy/SemKvdXFfHDBUEzb4M9CynTzBQK72cuJ1hgEyXiqVFEA2Z3mL
+q5DQSstx5I+/8p+QN5V93reusq9ALEGqEWI4j9kj5vGQr4/t8Ce0onflpsoNDIng
+1uUBv4BRe/Gj4ewJpfwbKjHxxssUauQ0gJg971hy5jz306G5j+8EtOSQtcrLKcbu
+ctoZhzf240aD9IjRlZwAQLWrS5FUHl53dfsF+kiU/rTDEuLx5fsMA0PKhEn+85IX
+lEiKWas8zptecAxhGaxRv5ndaNzw1EFkIL/lrQIDAQABAoIBAH8A+aulrnom2okS
+hW6ysYyugKq0VRtvva9iBnq92dW8iW4++E4Raqd2cDIQWFjWKfhubeu6o2nYFf+Q
+FISNQaxfq8rG6NVCl37BKkPvMZELCclc/zQ4mzVPObYnhLEXBG0sgmZ8Klm7kf52
+WLBrmwJaOo9RCqWwpG3gUOnLcq8vhvqKEXRzqaHqPtWJEuU09FgXt55/8eydykvE
+JqP9W1iLpwYgknSkC65QKxpWrKx1h9JbkKUzOx4H51WzR2qVDT3VVK+5J7LvonSI
+p1eEDGH8g3kWp6T0qwaUTkVINS4hhg/l5XPEqKkTNayKIJeSrEkfgs0sL/D9cxTI
+gswml6ECgYEA/E7RJa8xXDqthtw/Uxss5wfANquufBxl9c0Spp2M1SfMSXGTQ+SQ
+6mUGsWdiFhv98vPVv4JDfjvOSonppG9+cd2+kSignCeEt33hMb6cgp9QpG0PC22C
+B2JqWtN4YuS8D+Hms5ZwC1XSavXCD8gPspXlo6jM6Q/3fPZxZr+xvzkCgYEA4ZLV
+rOwODiPRB3DL+Y01KJZQvHGbU9QRAjAfqpdJ6UagueKQB6I/GxCKwtZbG1l2RreM
+vjhvDf8oWW4Oe+Ff9PXp/A4C42RbPKTKIUNLiDd+qVsn9yT2dypibb3W6pzLQru7
+gT72ERCwb/yAoIbwLSUpudarRxASOBiCxgJO5hUCgYAdQE4DHnKMjMj9b59v1SC5
+kC56qCDMhsZTXvkgk/d15u1KPn6iSmd6pUfHXYMDqS8h8Z8AWuvUQL7D6YVK6Rox
+vVGQprMoJ3S9iTyIdd7TTgEdbv9lhcajcIeADDBS1s3u0XPbsj8/MS359JARIFKh
+k7tR5AoNoypANJolHWhQEQKBgQC8iPV+6m109AqppWVW1ucvXTNtzaZ9kioFIiPV
+eU2VZCrCMtMfDkSgYlsYLgFIAqYQxSkldBWeJT2cQZhpgh2pqEBBI5TWafK35SPs
+WUqwN3HuKvpkXsiBITPTdXBD93kL9czqYa1Y5hISB4Gaqi3kDcOqR/owEwanTBVB
+WnyHYQKBgQChNn7uvIIUzQKoxYyvXk5NVIb1xey64deOVnayg66s6oHF9UHtynay
+equq5maSgyF3Ft597koiG7ylMkcXkdsmUPos26ZSDlgqYC9v+VopqQB3T06ltyGA
+lOjjVjzofTSzMvQ1UFbDoogelpnZNn8koKrgrUuZL+A0VLEYQTFITg==
+-----END RSA PRIVATE KEY-----

zefie_wtvp_minisrv/ServiceVault/wtv-news/news.js

@@ -1,12 +1,32 @@
 var minisrv_service_file = true;
+
 const wtvnews = new WTVNews(minisrv_config, service_name);
+var service_config = minisrv_config.services[service_name];
+if (service_config.local_nntp_port && wtvnewsserver) {
+    var tls_path = this.wtvshared.getAbsolutePath(this.minisrv_config.config.ServiceDeps + '/wtv-news');
+    var tls_options = {
+        ca: this.fs.readFileSync(tls_path + '/localserver_ca.pem'),
+        key: this.fs.readFileSync(tls_path + '/localserver_key.pem'),
+        cert: this.fs.readFileSync(tls_path + '/localserver_cert.pem'),
+        checkServerIdentity: () => { return null; }
+    }
+    if (wtvnewsserver.username)
+        wtvnews.initializeUsenet("127.0.0.1", minisrv_config.services[service_name].local_nntp_port, tls_options, wtvnewsserver.username, wtvnewsserver.password);
+    else
+        wtvnews.initializeUsenet("127.0.0.1", minisrv_config.services[service_name].local_nntp_port, tls_options);
+} else {
+    if (service_config.upstream_auth)
+        wtvnews.initializeUsenet(service_config.upstream_address, service_config.upstream_port, service_config.upstream_tls || null, service_config.upstream_auth.username || null, service_config.upstream_auth.password || null);
+    else
+        wtvnews.initializeUsenet(service_config.upstream_address, service_configupstream_port, service_config.upstream_tls || null);
+}
  
 async function throwError(e) {
+    console.log(e);
     var errpage = wtvshared.doErrorPage(400, null, e.toString());
     sendToClient(socket, errpage[0], errpage[1]);
 }
 
-
 function isToday (chkdate) {
     const today = new Date()
     return chkdate.getDate() == today.getDate() &&
@@ -28,7 +48,6 @@ async function WebTVListGroup(group) {
                     page_start = (limit_per_page * page) + 1;
                     page_end = (page + 1) * limit_per_page;
                     if (page_end > NGCount) page_end = NGCount;
-
                     wtvnews.getHeaderObj(NGArticles).then((messages) => {
                         messages = wtvnews.sortByResponse(messages);
                         wtvnews.quitUsenet();
@@ -157,7 +176,9 @@ Group: ${request_headers.query.group}
 </font>
 <br>
 <img src="wtv-home:/ROMCache/Spacer.gif" width=0 height=8>
-
+`;
+                        if (NGCount > 0) {
+                            data += `
 <td width=180 valign=bottom align=right>
 <table cellspacing=0 cellpadding=0>
 <td rowspan=4 height=26 width=30>
@@ -182,8 +203,9 @@ ${page_start}-${page_end}
 <img src="wtv-home:/ROMCache/Spacer.gif" width=1 height=1>
 <tr>
 <td colspan=5 height=3>
-</table>	</table>
+</table>	`;
-
+                        }
+                        data += `</table>
 <TABLE width=446 cellspacing=0 cellpadding=0>
 <tr>
 <td rowspan=4>
@@ -206,6 +228,7 @@ ${page_start}-${page_end}
                                 var has_relation = (messages[k].relation !== null) ? true : false;
                                 var date_obj = new Date(Date.parse(message.headers.DATE));
                                 var date = (isToday(date_obj)) ? strftime("%I:%M %p", date_obj) : strftime("%b %d", date_obj)
+                                console.log(message);
                                 data += `
 <table cellspacing=0 cellpadding=0>
 <tr>
@@ -230,8 +253,6 @@ ${(message.headers.FROM.indexOf(' ') > 0) ? message.headers.FROM.split(' ')[0] :
 <TABLE width=446 cellspacing=0 cellpadding=0>
 <tr>
 <td rowspan=4 width=10 height=1>
-<img src="wtv-home:/ROMCache/Spacer.gif" width=10 height=1>
-<td height=2 width=436 bgcolor="2B2B2B">
 <img src="wtv-home:/ROMCache/Spacer.gif" width=436 height=1>
 <tr>
 <td height=1>
@@ -264,7 +285,6 @@ async function WebTVShowMessage(group, article) {
             wtvnews.getArticle(article).then((response) => {
                 wtvnews.quitUsenet();
                 if (response.code == 220) {
-                    console.log(response);
                     headers = `200 OK
 Content-type: text/html
 wtv-expire-all: wtv-news:/news?group=${group}&article=`;
@@ -573,7 +593,7 @@ ${wtvshared.htmlEntitize(message_body, true)}
     });;
 }
 
-if (!minisrv_config.services[service_name].upstream_address || !minisrv_config.services[service_name].upstream_port) {
+if (!wtvnews.client) {
     var errpage = doErrorPage();
     headers = errpage[0];
     data = errpage[1];

zefie_wtvp_minisrv/ServiceVault/wtv-setup/edit-password.js

@@ -123,7 +123,7 @@ MAXLENGTH="${minisrv_config.config.passwords.max_length}">
 <tr>
 <td colspan=3 align=left>
 <br>Type again to confirm<br>
-<INPUT noSubmit name="verify" id="verify" Value=""
+<INPUT noSubmit name="password_verify" id="password_verify" Value=""
 bgcolor=#444444 text=#ffdd33 cursor=#cc9933
 TYPE="password" ASCIIONLY
 SIZE="${minisrv_config.config.passwords.form_size}"

zefie_wtvp_minisrv/ServiceVault/wtv-setup/validate-change-password.js

@@ -36,7 +36,7 @@ Content-Type: text/html`
         data = errpage[1];
     }
     else {
-        if (request_headers.query.password.length == 0 && request_headers.query.verify.length == 0) {
+        if (request_headers.query.password.length == 0 && request_headers.query.password_verify.length == 0) {
             userSession.disableUserPassword();
             headers = `300 OK
 Content-type: text/html
@@ -50,7 +50,7 @@ Location: ${request_headers.query.return_to}`;
         }
         else if (request_headers.query.password.length < minisrv_config.config.passwords.min_length) errpage = wtvshared.doErrorPage(400, "Your password must contain at least " + minisrv_config.config.passwords.min_length + " characters.");
         else if (request_headers.query.password.length > minisrv_config.config.passwords.max_length) errpage = wtvshared.doErrorPage(400, "Your password must contain no more than than " + minisrv_config.config.passwords.max_length + " characters.");
-        else if (request_headers.query.password !== request_headers.query.verify) errpage = wtvshared.doErrorPage(400, "The passwords you entered did not match. Please check them and try again.");
+        else if (request_headers.query.password !== request_headers.query.password_verify) errpage = wtvshared.doErrorPage(400, "The passwords you entered did not match. Please check them and try again.");
         else {
             if (errpage) {
                 headers = errpage[0];

zefie_wtvp_minisrv/includes/WTVNews.js

@@ -6,23 +6,55 @@ class WTVNews {
     wtvshared = null;
     service_name = null;
     client = null;
+    username = null;
+    password = null;
 
     constructor(minisrv_config, service_name) {
         this.minisrv_config = minisrv_config;
         this.service_name = service_name;
         const { WTVShared } = require("./WTVShared.js");
         this.wtvshared = new WTVShared(minisrv_config);
-        this.client = new this.newsie({
+    }
-            host: this.minisrv_config.services[service_name].upstream_address,
+
-            port: this.minisrv_config.services[service_name].upstream_port
+    initializeUsenet(host, port = 119, tls_options = null, username = null, password = null) {
-        })
+        // use local self-signed cert for local server
+        var newsie_options = {
+            host: host,
+            port: port,
+            tlsPort: (tls_options !== null) ? true : false,            
+        }
+        if (newsie_options.tlsPort) newsie_options.tlsOptions = tls_options;
+        this.client = new this.newsie(newsie_options);
+        if (username && password) {
+            this.username = username;
+            this.password = password;
+        }
     }
 
     connectUsenet() {
         return new Promise((resolve, reject) => {
             this.client.connect().then((response) => {
-                if (response.code == 200) {
+                if (response.code == 200 || response.code == 201) {
-                    resolve(true);
+                    if (this.username && this.password) {
+                        this.client.authInfoUser(this.username).then((res) => {
+                            if (res.code == "381") {
+                                res.authInfoPass(this.password).then((res) => {
+                                    if (res.code == 281) resolve(true);
+                                    else reject(res.description);
+                                }).catch((e) => {
+                                    console.error(" * WTVNews Error:", "Command: connect", e);
+                                    reject("Could not connect to upstream usenet server");
+                                });
+                            } else {
+                                reject(res.description)
+                            }
+                        }).catch((e) => {
+                            console.error(" * WTVNews Error:", "Command: connect", e);
+                            reject("Could not connect to upstream usenet server");
+                        });
+                    } else {
+                        resolve(true);
+                    }
                 }
             }).catch((e) => {
                 console.error(" * WTVNews Error:", "Command: connect", e);
@@ -31,16 +63,19 @@ class WTVNews {
         }); 
     }
 
-    listGroup(group, page = 0, limit = 100) {
+    listGroup(group, page = 0, limit = 100, raw_range = null) {
         return new Promise((resolve, reject) => {
             this.selectGroup(group).then((res) => {
-                var range = {
+                if (!raw_range) {
-                    start: (limit * page) + res.group.low,
+                    var range = {
+                        start: (limit * page) + res.group.low,
+                    }
+                    range.end = range.start + limit;
+                    if (page) range.start++;
+                    if (range.end > res.high) delete range.group.end;
+                } else {
+                    range = raw_range;
                 }
-                range.end = range.start + limit;
-                if (page) range.start++;
-                if (range.end > res.high) delete range.group.end;
-                console.log(res, range);
                 this.client.listGroup(group, range).then((data) => {
                     resolve(data);
                 }).catch((e) => {
@@ -75,7 +110,8 @@ class WTVNews {
                     this.client.next().then((res) => {
                         data.next_article = res.article.articleNumber;
                         resolve(data.next_article);
-                    }).catch(() => {
+                    }).catch((e) => {
+                        console.log(e);
                         data.next_article = null;
                         resolve(data.next_article);
                     })
@@ -150,15 +186,12 @@ class WTVNews {
                         promises.push(new Promise((resolve, reject) => {
                             this.selectGroup(group).then((res) => {
                                 this.getArticleMessageID(article).then((data) => {
-                                    console.log(data);
                                     messageid = data;
                                     resolve(data);
                                 }).catch((e) => {
-                                    console.log(e);
                                     reject(e)
                                 });
                             }).catch((e) => {
-                                console.log(e);
                                 reject(e)
                             });
                         }));
@@ -185,7 +218,6 @@ class WTVNews {
                                     } else {
                                         articleData.body = [];
                                     }
-                                    console.log(articleData);
                                     response.send(articleData).then((response) => {
                                         this.client.quit();
                                         if (response.code !== 240) {
@@ -233,7 +265,6 @@ class WTVNews {
                         if (data.article) messages.push(data.article)
                         resolve();
                     }).catch((e) => {
-                        console.log(e, article);
                         reject(e);
                     });
                 }));
@@ -285,9 +316,6 @@ class WTVNews {
                                 });
                             }
                         })
-                        if (!found) {
-                            message_id_roots.push({ "messageId": messageId, "index": j });
-                        }
                     } else {
                         message_id_roots.push({ "messageId": messageId, "index": k });
                     }
@@ -306,7 +334,7 @@ class WTVNews {
             var article_date = Date.parse(article.headers.DATE);
             message_roots_sorted.push({ "article": article, "relation": null, "date": article_date });
         });
-        message_roots_sorted.sort((a, b) => { return (a.date > b.date) });
+        message_roots_sorted.sort((a, b) => { return (a.date - b.date) });
         Object.keys(message_roots_sorted).forEach((k) => {
             sorted.push(message_roots_sorted[k]);
 
@@ -318,7 +346,7 @@ class WTVNews {
                     var article_date = Date.parse(article.headers.DATE);
                     relations.push({ "article": article, "relation": message_id_roots[k].messageId, "date": article_date })
                 });
-                relations.sort((a, b) => { return (a.date > b.date) });
+                relations.sort((a, b) => { return (a.date - b.date) });
                 Object.keys(relations).forEach((j) => {
                     sorted.push(relations[j]);
                 });

zefie_wtvp_minisrv/includes/WTVNewsServer.js

@@ -0,0 +1,216 @@
+class WTVNewsServer {
+    fs = require('fs');
+    path = require('path');
+    minisrv_config = null;
+    strftime = require('strftime');
+    wtvshared = null;
+    username = null;
+    password = null;
+    using_auth = false;
+    local_server = null;
+    data_path = null;
+
+    constructor(minisrv_config, local_server_port, using_auth = false, username = null, password = null) {
+        this.minisrv_config = minisrv_config;
+        const { WTVShared } = require("./WTVShared.js");
+        this.wtvshared = new WTVShared(minisrv_config);
+        const nntp_server = require('nntp-server');
+        var nntp_commands = {
+            ...nntp_server.commands,
+            "LAST": {
+                head: 'LAST',
+                validate: /^LAST( [^\s]+)$/i,
+
+                // All supported params are defined in separate files
+                run() {
+                    console.log('hi');
+                    throw new Error('method LAST is not implemented');
+                }
+            },
+            "NEXT": {
+                head: 'NEXT',
+                validate: /^NEXT( [^\s]+)$/i,
+
+                // All supported params are defined in separate files
+                run() {
+                    console.log('hi');
+                    throw new Error('method NEXT is not implemented');
+                }
+            }
+        }
+
+        console.log(nntp_commands);
+
+        this.username == username || null;
+        this.password == password || null;
+        this.using_auth = using_auth;
+        if (using_auth && (!username && !password)) {
+            // using auth, but no auth info specified, so randomly generate it
+            this.username = this.wtvshared.generatePassword(8);
+            this.password = this.wtvshared.generatePassword(16);
+        }
+
+        // nntp-server module overrides
+        var self = this;
+
+        nntp_server.prototype = {
+            ...nntp_server.prototype,
+            _authenticate: function (session) {
+                // authenticte
+                if (session.authinfo_user == self.username && session.authinfo_pass == self.password) return Promise.resolve(true);
+                return Promise.resolve(false);
+            },
+
+            _selectGroup: function (session, name) {
+                // selectGroup
+                var res = self.selectGroup(name);
+                if (!res.failed) {
+                    session.group = res;
+                    return true;
+                }
+                return false;
+            },
+
+            _buildHead: function (session, message) {
+                var out = "";
+                Object.keys(message.headers).forEach((k) => {
+                    if (k.length > 0) out += `${k}: ${message.headers[k]}\r\n`;
+                });
+                out = out.substr(0,out.length - 2);
+                return out;
+            },
+
+            _buildHeaderField: function (session, message, field) {
+                var search = Object.keys(message.headers).find(e => (e.toLowerCase() == field.toLowerCase()));
+                if (search) return message.headers[search];
+                else return null;
+            },
+
+            _getArticle: function (session, message_id) {
+                // getArticle
+                return new Promise((resolve, reject) => {
+                    self.getArticle(session.group.name, message_id).then((res) => {
+                        resolve(res);
+                    }).catch((e) => {
+                        console.log(" * WTVNewsServer Error:", e);
+                        reject(e);
+                    });
+                });
+            },
+
+            _buildBody: function (session, message) {
+                return message.body;
+            },
+
+            _getRange: function (session, first, last) {
+                var res = self.listGroup(session.group.name, first, last)
+                if (res.failed) return false;
+                session.group = res.group_data;
+                return res.articleNumbers;
+            }
+
+        }
+
+        this.data_path = this.wtvshared.getAbsolutePath(this.minisrv_config.config.SessionStore + '/minisrv_internal_nntp');
+        this.createDataStore();
+
+        var tls_path = this.wtvshared.getAbsolutePath(this.minisrv_config.config.ServiceDeps + '/wtv-news');
+        var tls_options = {
+            ca: this.fs.readFileSync(tls_path + this.path.sep + 'localserver_ca.pem'),
+            key: this.fs.readFileSync(tls_path + this.path.sep + 'localserver_key.pem'),
+            cert: this.fs.readFileSync(tls_path + this.path.sep + 'localserver_cert.pem'),
+        }
+        this.local_server = new nntp_server({ requireAuth: using_auth, tls: tls_options, secure: true, commands: nntp_commands });
+        this.local_server.listen('nntps://localhost:' + local_server_port);
+    }
+
+    createDataStore() {
+        if (!this.fs.existsSync(this.data_path)) return this.fs.mkdirSync(this.data_path);
+        return true;
+    }
+
+    getGroupPath(group) {
+        return this.data_path + this.path.sep + group;
+    }
+
+    getArticlePath(group, article) {
+        return this.getGroupPath(group) + this.path.sep + article + ".newz";
+    }
+
+    createGroup(group) {
+        if (!this.fs.existsSync(getGroupPath(group))) return this.fs.mkdirSync(getGroupPath(group));
+        return true;
+    }
+
+    getArticle(group, article) {
+        return new Promise((resolve, reject) => {
+            const g = this.getArticlePath(group, article);
+            if (!this.fs.existsSync(g)) return false;
+            try {
+                var data = JSON.parse(this.fs.readFileSync(g));
+                resolve(data);
+            } catch (e) {
+                console.log(" * WTVNewsServer Error:", e);
+                reject(e)
+            }
+        });
+    }
+
+    selectGroup(group) {
+        var g = this.getGroupPath(group);
+        var out = {
+            total: 0,
+            min_index: null,
+            max_index: 0,
+            name: group
+        }
+        try {
+            this.fs.readdirSync(g).forEach(file => {
+                var articleNumber = parseInt(file.split('.')[0]);
+                if (out.min_index == null) out.min_index = articleNumber;
+                else if (articleNumber < out.min_index) out.min_index = articleNumber;
+
+                if (articleNumber > out.max_index) out.max_index = articleNumber;
+                out.total++;
+            });
+        } catch (e) {
+            out.failed = e;
+        }
+        if (out.min_index === null) out.min_index = 0;
+        return out;
+    }
+
+    listGroup(group, start, end) {
+        var g = this.getGroupPath(group);
+        var out = {
+            total: 0,
+            min_index: null,
+            max_index: 0,
+            name: group
+        }
+        var articleNumbers = [];
+        try {
+            this.fs.readdirSync(g).forEach(file => {
+                var articleNumber = parseInt(file.split('.')[0]);
+                if (articleNumber < start) return;
+                if (articleNumber > end) return false;
+                if (out.min_index == null) out.min_index = articleNumber;
+                else if (articleNumber < out.min_index) out.min_index = articleNumber;
+
+                if (articleNumber > out.max_index) out.max_index = articleNumber;
+                articleNumbers.push({ index: articleNumber });
+                out.total++;
+            });
+        } catch (e) {
+            console.log(e);
+            out.failed = e;
+        }
+        if (out.min_index === null) out.min_index = 0;
+        return {
+            articleNumbers: articleNumbers,
+            group_data: out
+        }
+    }
+}
+
+module.exports = WTVNewsServer;

zefie_wtvp_minisrv/includes/config.json

@@ -6,7 +6,7 @@
       "UserServiceVault",
       "ServiceVault"
     ],
-    "ServiceDeps":  "ServiceDeps",
+    "ServiceDeps": "ServiceDeps",
     "SessionStore": "SessionStore",
     "SharedROMCache": "SharedROMCache",
     "enable_shared_romcache": true,
@@ -31,8 +31,12 @@
     "show_detailed_splash": true,
     "show_diskmap": false,
     "unauthorized_url": "wtv-1800:/unauthorized?",
+    "enable_port_isolation": true,
     "allow_guests": true,
     "domain_name": "wtv.zefie.com",
+    "ssid_block_list": [
+      "minisrv_internal_nntp"
+    ],
     "user_accounts": {
       "max_users_per_account": 6,
       "min_username_length": 5,
@@ -60,7 +64,8 @@
     },
     "wtv-news": {
       "port": 1605,
-      "disabled": true,
+      "local_nntp_port": 57319,
+      "local_nntp_requires_auth": true,
       "modules": [
         "WTVNews"
       ]

zefie_wtvp_minisrv/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "zefie_wtvp_minisrv",
-  "version": "0.9.32",
+  "version": "0.9.33",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "zefie_wtvp_minisrv",
-      "version": "0.9.32",
+      "version": "0.9.33",
       "license": "GPL3",
       "dependencies": {
         "adm-zip": "^0.5.9",
@@ -18,6 +18,7 @@
         "iconv-lite": "^0.6.3",
         "mime-types": "^2.1.35",
         "newsie": "^1.2.1",
+        "nntp-server": "^3.1.0",
         "proxy-agent": "^5.0.0",
         "sanitize-html": "^2.7.2",
         "socks-proxy-agent": "^6.2.1",
@@ -263,6 +264,14 @@
         "node": ">= 6"
       }
     },
+    "node_modules/denque": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
+      "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
+      "engines": {
+        "node": ">=0.10"
+      }
+    },
     "node_modules/depd": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
@@ -861,6 +870,17 @@
         "node": ">=8.9.0"
       }
     },
+    "node_modules/nntp-server": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/nntp-server/-/nntp-server-3.1.0.tgz",
+      "integrity": "sha512-L8U2CnFPHXfDKF+eaYpx07fCW7gbk7gNB0faDWg5fLIEofEP7O/RIUEonl0jGRDf9/eGMj7CRil1faG5YZq4bw==",
+      "dependencies": {
+        "debug": "^4.3.3",
+        "denque": "^2.0.1",
+        "serialize-error": "^8.1.0",
+        "split2": "^4.1.0"
+      }
+    },
     "node_modules/object-inspect": {
       "version": "1.12.2",
       "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz",
@@ -1179,6 +1199,20 @@
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
     },
+    "node_modules/serialize-error": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz",
+      "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==",
+      "dependencies": {
+        "type-fest": "^0.20.2"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/serve-static": {
       "version": "1.15.0",
       "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz",
@@ -1268,6 +1302,14 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/split2": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz",
+      "integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ==",
+      "engines": {
+        "node": ">= 10.x"
+      }
+    },
     "node_modules/statuses": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
@@ -1313,6 +1355,17 @@
         "node": ">= 0.8.0"
       }
     },
+    "node_modules/type-fest": {
+      "version": "0.20.2",
+      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz",
+      "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/type-is": {
       "version": "1.6.18",
       "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
@@ -1576,6 +1629,11 @@
         "vm2": "^3.9.8"
       }
     },
+    "denque": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
+      "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="
+    },
     "depd": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
@@ -2016,6 +2074,17 @@
       "resolved": "https://registry.npmjs.org/newsie/-/newsie-1.2.1.tgz",
       "integrity": "sha512-41jhtKmlpSc27oIBkp681fm4aLURPT/AgeBvSRicxNPWDVeHLaq7tSvG/OsEXz7g41thgv9JMV1ZjYSABMiYVA=="
     },
+    "nntp-server": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/nntp-server/-/nntp-server-3.1.0.tgz",
+      "integrity": "sha512-L8U2CnFPHXfDKF+eaYpx07fCW7gbk7gNB0faDWg5fLIEofEP7O/RIUEonl0jGRDf9/eGMj7CRil1faG5YZq4bw==",
+      "requires": {
+        "debug": "^4.3.3",
+        "denque": "^2.0.1",
+        "serialize-error": "^8.1.0",
+        "split2": "^4.1.0"
+      }
+    },
     "object-inspect": {
       "version": "1.12.2",
       "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz",
@@ -2266,6 +2335,14 @@
         }
       }
     },
+    "serialize-error": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz",
+      "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==",
+      "requires": {
+        "type-fest": "^0.20.2"
+      }
+    },
     "serve-static": {
       "version": "1.15.0",
       "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz",
@@ -2334,6 +2411,11 @@
       "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz",
       "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw=="
     },
+    "split2": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz",
+      "integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ=="
+    },
     "statuses": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
@@ -2367,6 +2449,11 @@
         "prelude-ls": "~1.1.2"
       }
     },
+    "type-fest": {
+      "version": "0.20.2",
+      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz",
+      "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ=="
+    },
     "type-is": {
       "version": "1.6.18",
       "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",

zefie_wtvp_minisrv/package.json

@@ -36,6 +36,7 @@
     "iconv-lite": "^0.6.3",
     "mime-types": "^2.1.35",
     "newsie": "^1.2.1",
+    "nntp-server": "^3.1.0",
     "proxy-agent": "^5.0.0",
     "sanitize-html": "^2.7.2",
     "socks-proxy-agent": "^6.2.1",

zefie_wtvp_minisrv/sync_nntp.js

@@ -0,0 +1,138 @@
+// no wildcard (yet?)
+const groups_to_sync = [
+	"webtv.users"
+]
+
+
+
+const fs = require('fs');
+const path = require('path');
+var classPath = __dirname + "/includes/";
+const { WTVShared } = require(classPath + "WTVShared.js");
+const wtvshared = new WTVShared(); // creates minisrv_config
+classPath = wtvshared.getAbsolutePath(classPath, __dirname);
+const minisrv_config = wtvshared.getMiniSrvConfig();
+const WTVNews = require(classPath + "/WTVNews.js");
+var data_path = wtvshared.getAbsolutePath(minisrv_config.config.SessionStore + '/minisrv_internal_nntp');
+const service_name = "wtv-news";
+
+if (!minisrv_config.services[service_name].upstream_address) {
+	console.error("minisrv config not configured with usenet upstream");
+	process.exit(1);
+}
+
+if (!minisrv_config) {
+	console.error("Something went wrong loading minisrv config");
+	process.exit(1);
+}
+
+if (!minisrv_config.config.debug_flags.quiet) console.log(" *** Successfully read minisrv configuration....");
+
+
+const service_config = minisrv_config.services[service_name];
+const wtvnews = new WTVNews(minisrv_config, service_name);
+
+if (service_config.upstream_auth) {
+	wtvnews.initializeUsenet(service_config.upstream_address, service_config.upstream_port, service_config.upstream_tls || null, service_config.upstream_auth.username || null, service_config.upstream_auth.password || null);
+} else {
+	wtvnews.initializeUsenet(service_config.upstream_address, service_configupstream_port, service_config.upstream_tls || null);
+}
+
+
+
+function createDataStore() {
+	if (!fs.existsSync(data_path)) return fs.mkdirSync(data_path);
+	return true;
+}
+
+function getGroupPath(group) {
+	return data_path + path.sep + group;
+}
+
+function createGroup(group) {
+	createDataStore();
+	if (!fs.existsSync(getGroupPath(group))) return fs.mkdirSync(getGroupPath(group));
+	return true;
+}
+
+function createArticle(group, articleNumber, article) {
+	var g = getGroupPath(group);
+	var file = g + path.sep + articleNumber + ".newz";
+	if (fs.existsSync(file)) return "exists";
+	else {
+		try {
+			article.article.index = article.article.articleNumber;
+			delete article.article.articleNumber;
+			article.article['message-id'] = article.article.messageId;
+			delete article.article.messageId;
+
+			fs.writeFileSync(file, JSON.stringify(article.article));
+			return file;
+		} catch (e) {
+			return e;
+		}
+	}
+}
+
+function deleteMissing(group, articles) {
+	var g = getGroupPath(group);
+	try {
+		fs.readdirSync(g).forEach(file => {
+			var articleNumber = parseInt(file.split('.')[0]);
+			file = g + path.sep + file;
+			if (!articles.find(e => (e == articleNumber))) {
+				console.log(" * ", group, "article", articleNumber, "deleted from upstream, removing locally")
+				fs.rmSync(file);
+			}
+		});
+		return true;
+	} catch (e) {
+		console.log(e);
+		return false;
+	}
+}
+
+wtvnews.connectUsenet().then((res) => {
+	if (res) {
+		groups_to_sync.forEach((group) => {
+			console.log(group);
+			wtvnews.selectGroup(group).then((res) => {
+				var range = {
+					start: res.group.low,
+					end: res.group.high
+				}
+				wtvnews.listGroup(group, null, null, range).then((res) => {
+					if (res.group.articleNumbers) {
+						var promises = [];
+						createGroup(group);
+						deleteMissing(group, res.group.articleNumbers)
+						res.group.articleNumbers.forEach((article) => {
+							promises.push(new Promise((resolve, reject) => {
+								wtvnews.getArticle(article).then((message) => {
+									res = createArticle(group, article, message);
+									if (res) {
+										if (res == "exists") {
+											console.log(" * ", group, "article", article, "already exists")
+										} else {
+											console.log(" * Created", group, "article", article, res)
+										}
+										resolve(true)
+									} else {
+										console.log(" ! Failed to create file for ", group, "article", article, "with error", res);
+										resolve(false)
+									}
+								});
+							}));
+						})
+						Promise.all(promises).then(() => {
+							wtvnews.quitUsenet();
+							process.exit(0);
+						});
+					} else {
+						wtvnews.quitUsenet();
+					}
+                })
+			});
+        })
+    }
+});

zefie_wtvp_minisrv/user_config.example.json

@@ -67,8 +67,13 @@
 			"upstream_address": "192.168.11.8",
 			"upstream_port": 1699,
 			"upstream_auth": {
-				"username": "not_yet",
+				"username": "myusername",
-				"password": "implemented"
+				"password": "mypassword"
+			},
+			"upstream_tls": true,
+			"local_auth": {
+				"username": "mylocaluser",
+				"password": "mylocalpass"
 			}
 		}
 	},
@@ -77,16 +82,17 @@
 			"Personal": [
 				{
 					"title": "The Midnight Archive",
-					"url": "http://archive.midnightchanne.net/",
+					"url": "http://archive.midnightchannel.net/",
 					"image_type": "url",
 					"image": "canned/midnightchannel.gif"
 				},
 				{
 					"title": "WebTV/MSNTV Secrets",
-					"url": "http://turdinc.kicks-ass.net/Msntv/",
+					"url": "http://turdinc.kicks-ass.net/Msntv/index2.html",
 					"image_type": "url",
 					"image": "canned/mattman69.gif"
 				}
 			]
 		}
 	}
+}

zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj

@@ -38,6 +38,7 @@
     <Content Include="diskmap_gen.js">
       <SubType>Code</SubType>
     </Content>
+    <Content Include="includes\WTVNewsServer.js" />
     <Content Include="packer.js" />
     <Content Include="ServiceDeps\wtv-1800\tellyscripts\bf0app\bf0app_OISP.tok" />
     <Content Include="ServiceDeps\wtv-1800\tellyscripts\bf0app\bf0app_WTV_18006138199.detok.txt" />