From 976c79c92aceac192bbb5e230ae2387959794122 Mon Sep 17 00:00:00 2001 From: zefie Date: Wed, 12 Oct 2022 12:26:30 -0400 Subject: [PATCH] wip usenet stuff --- .../ServiceDeps/wtv-news/localserver_ca.pem | 23 ++ .../ServiceDeps/wtv-news/localserver_cert.pem | 21 ++ .../ServiceDeps/wtv-news/localserver_key.pem | 27 +++ .../ServiceVault/wtv-news/news.js | 40 +++- .../ServiceVault/wtv-setup/edit-password.js | 2 +- .../wtv-setup/validate-change-password.js | 4 +- zefie_wtvp_minisrv/includes/WTVNews.js | 76 ++++-- zefie_wtvp_minisrv/includes/WTVNewsServer.js | 216 ++++++++++++++++++ zefie_wtvp_minisrv/includes/config.json | 9 +- zefie_wtvp_minisrv/package-lock.json | 91 +++++++- zefie_wtvp_minisrv/package.json | 1 + zefie_wtvp_minisrv/sync_nntp.js | 138 +++++++++++ zefie_wtvp_minisrv/user_config.example.json | 16 +- zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj | 1 + 14 files changed, 619 insertions(+), 46 deletions(-) create mode 100644 zefie_wtvp_minisrv/ServiceDeps/wtv-news/localserver_ca.pem create mode 100644 zefie_wtvp_minisrv/ServiceDeps/wtv-news/localserver_cert.pem create mode 100644 zefie_wtvp_minisrv/ServiceDeps/wtv-news/localserver_key.pem create mode 100644 zefie_wtvp_minisrv/includes/WTVNewsServer.js create mode 100644 zefie_wtvp_minisrv/sync_nntp.js diff --git a/zefie_wtvp_minisrv/ServiceDeps/wtv-news/localserver_ca.pem b/zefie_wtvp_minisrv/ServiceDeps/wtv-news/localserver_ca.pem new file mode 100644 index 00000000..6a5fbe67 --- /dev/null +++ b/zefie_wtvp_minisrv/ServiceDeps/wtv-news/localserver_ca.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIID1TCCAr2gAwIBAgIUQzfMpYgezM0u/aOpBIYzCTSNuSgwDQYJKoZIhvcNAQEL +BQAwejELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAk5ZMRcwFQYDVQQKDA5aZWZpZSBO +ZXR3b3JrczEQMA4GA1UECwwHbWluaXNydjETMBEGA1UEAwwKbWluaXNydl9jYTEe +MBwGCSqGSIb3DQEJARYPemVmaWVAemVmaWUubmV0MB4XDTIyMTAxMjAzMzQzOVoX +DTIyMTExMTAzMzQzOVowejELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAk5ZMRcwFQYD +VQQKDA5aZWZpZSBOZXR3b3JrczEQMA4GA1UECwwHbWluaXNydjETMBEGA1UEAwwK +bWluaXNydl9jYTEeMBwGCSqGSIb3DQEJARYPemVmaWVAemVmaWUubmV0MIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzo8+u/9kJcQY+mOKLxYNOtNSWqlu +fwSKpo/kCMbCkM4TCJiaKHWwtXeT5tgrYPqg6s3nHpG2arwI/sxX6fy86s9i8bu5 +NEaO6/Czii0gQLXznURPaCPKp2dJEpcrFvqsB/Ej7Bm2cZH4BaOW9CeaMdrSXK1e +Tlwem1XHPeFnIPGQr3sYp2CNcB3TWzNkGGDPB2CBcR+qB2Fnjb7aEcd5t4imJqT6 +BPTiDB234v3glcEaDBIr3bT/YGZuRn7Ou8qAVKlfT4iDsgHkpW+V7RYRocfAynPF +mSxQf2i4hIfvK2mGhdpcQGH5WTOp6K9xIv/lGIZcb29xoAeICplq3/u24wIDAQAB +o1MwUTAdBgNVHQ4EFgQUG8IOPoOiwIA1GI/3pANtftkD7BwwHwYDVR0jBBgwFoAU +G8IOPoOiwIA1GI/3pANtftkD7BwwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B +AQsFAAOCAQEAaG63vM/KbOfx/wzg8dZE6j/h8Ai0wJ1iXXvBLa23+fpZwfHIMH/A +uvkap+OzX3oI8fdalReY11+WONfzA4pyuNPtluwkngSLjT8YSfHfM3MUM6LKt5x4 +rRNBNN74H8A7q5UDA/SD2M1HjcOkFNlH4XrE6Uy/ColCVV9cYGPdx1rKwiQiKK7Q +Nr+6awAYHzQ2kdrZ4kneBZqAWAuIzjjWMxcgVwCHpb4rV0u3j/uhIos/8Q6NtLA7 +EqW0qMWmpJ9DlqfhLXqVCgfPyxGNpvTom0lw6J8OL8U6d+oaeVTZW44/3A8wNjJ+ +Gd2qQMxupodfV3LVyREvDpM+cowW6q+qHA== +-----END CERTIFICATE----- diff --git a/zefie_wtvp_minisrv/ServiceDeps/wtv-news/localserver_cert.pem b/zefie_wtvp_minisrv/ServiceDeps/wtv-news/localserver_cert.pem new file mode 100644 index 00000000..9c28dbf1 --- /dev/null +++ b/zefie_wtvp_minisrv/ServiceDeps/wtv-news/localserver_cert.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDhjCCAm4CFCwnaIlHV8E1xd6rMWruYTncUoieMA0GCSqGSIb3DQEBCwUAMHox +CzAJBgNVBAYTAlVTMQswCQYDVQQIDAJOWTEXMBUGA1UECgwOWmVmaWUgTmV0d29y +a3MxEDAOBgNVBAsMB21pbmlzcnYxEzARBgNVBAMMCm1pbmlzcnZfY2ExHjAcBgkq +hkiG9w0BCQEWD3plZmllQHplZmllLm5ldDAgFw0yMjEwMTIwMzM3MDlaGA8yMTIy +MDkxODAzMzcwOVowgYIxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJOWTEXMBUGA1UE +CgwObWluaXNydiB1c2VuZXQxFzAVBgNVBAsMDm1pbmlzcnYgdXNlbmV0MRcwFQYD +VQQDDA5taW5pc3J2IHVzZW5ldDEbMBkGCSqGSIb3DQEJARYMdXNlckBtaW5pc3J2 +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3lH9XWgXr1m8rLPS2gDV +/b5vUFSRiK+2JViVJq8CpLYSKqEoI1l2t4Ui4gwejy/SemKvdXFfHDBUEzb4M9Cy +nTzBQK72cuJ1hgEyXiqVFEA2Z3mLq5DQSstx5I+/8p+QN5V93reusq9ALEGqEWI4 +j9kj5vGQr4/t8Ce0onflpsoNDIng1uUBv4BRe/Gj4ewJpfwbKjHxxssUauQ0gJg9 +71hy5jz306G5j+8EtOSQtcrLKcbuctoZhzf240aD9IjRlZwAQLWrS5FUHl53dfsF ++kiU/rTDEuLx5fsMA0PKhEn+85IXlEiKWas8zptecAxhGaxRv5ndaNzw1EFkIL/l +rQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBEzoRbk8D0a/ETOtQtnGpFFpMkxNkx +b015xPLpQEnQ3/CWs8jUTJiLQdgL7BkmDx2CGVbONKYMr9YO3kzCn+wAiX1FD+NP +9CMRj1XEkjxoImYv75rmyKtspNMFq9r/PiLwr0kRmAASnbYn5+15D68FTyxIw/zJ +7M0jZmf/hW+QT1WVlvO31gwyGcWQFvJctYMjImtAOlZE6LJQI5T4Ld10L1EQyj2q +l9XVM1HrggecgaSsdMxGIi02XZ/LV3gKLuTp7KJ/8YcjThQVOVS4wycmuGF0I71t +IpZVAjDxH/pzL79uyW8oEzkhF4H0QDA5I2cd4h6kGPeOXH76JvdhlQ5Q +-----END CERTIFICATE----- diff --git a/zefie_wtvp_minisrv/ServiceDeps/wtv-news/localserver_key.pem b/zefie_wtvp_minisrv/ServiceDeps/wtv-news/localserver_key.pem new file mode 100644 index 00000000..f16d54f5 --- /dev/null +++ b/zefie_wtvp_minisrv/ServiceDeps/wtv-news/localserver_key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA3lH9XWgXr1m8rLPS2gDV/b5vUFSRiK+2JViVJq8CpLYSKqEo +I1l2t4Ui4gwejy/SemKvdXFfHDBUEzb4M9CynTzBQK72cuJ1hgEyXiqVFEA2Z3mL +q5DQSstx5I+/8p+QN5V93reusq9ALEGqEWI4j9kj5vGQr4/t8Ce0onflpsoNDIng +1uUBv4BRe/Gj4ewJpfwbKjHxxssUauQ0gJg971hy5jz306G5j+8EtOSQtcrLKcbu +ctoZhzf240aD9IjRlZwAQLWrS5FUHl53dfsF+kiU/rTDEuLx5fsMA0PKhEn+85IX +lEiKWas8zptecAxhGaxRv5ndaNzw1EFkIL/lrQIDAQABAoIBAH8A+aulrnom2okS +hW6ysYyugKq0VRtvva9iBnq92dW8iW4++E4Raqd2cDIQWFjWKfhubeu6o2nYFf+Q +FISNQaxfq8rG6NVCl37BKkPvMZELCclc/zQ4mzVPObYnhLEXBG0sgmZ8Klm7kf52 +WLBrmwJaOo9RCqWwpG3gUOnLcq8vhvqKEXRzqaHqPtWJEuU09FgXt55/8eydykvE +JqP9W1iLpwYgknSkC65QKxpWrKx1h9JbkKUzOx4H51WzR2qVDT3VVK+5J7LvonSI +p1eEDGH8g3kWp6T0qwaUTkVINS4hhg/l5XPEqKkTNayKIJeSrEkfgs0sL/D9cxTI +gswml6ECgYEA/E7RJa8xXDqthtw/Uxss5wfANquufBxl9c0Spp2M1SfMSXGTQ+SQ +6mUGsWdiFhv98vPVv4JDfjvOSonppG9+cd2+kSignCeEt33hMb6cgp9QpG0PC22C +B2JqWtN4YuS8D+Hms5ZwC1XSavXCD8gPspXlo6jM6Q/3fPZxZr+xvzkCgYEA4ZLV +rOwODiPRB3DL+Y01KJZQvHGbU9QRAjAfqpdJ6UagueKQB6I/GxCKwtZbG1l2RreM +vjhvDf8oWW4Oe+Ff9PXp/A4C42RbPKTKIUNLiDd+qVsn9yT2dypibb3W6pzLQru7 +gT72ERCwb/yAoIbwLSUpudarRxASOBiCxgJO5hUCgYAdQE4DHnKMjMj9b59v1SC5 +kC56qCDMhsZTXvkgk/d15u1KPn6iSmd6pUfHXYMDqS8h8Z8AWuvUQL7D6YVK6Rox +vVGQprMoJ3S9iTyIdd7TTgEdbv9lhcajcIeADDBS1s3u0XPbsj8/MS359JARIFKh +k7tR5AoNoypANJolHWhQEQKBgQC8iPV+6m109AqppWVW1ucvXTNtzaZ9kioFIiPV +eU2VZCrCMtMfDkSgYlsYLgFIAqYQxSkldBWeJT2cQZhpgh2pqEBBI5TWafK35SPs +WUqwN3HuKvpkXsiBITPTdXBD93kL9czqYa1Y5hISB4Gaqi3kDcOqR/owEwanTBVB +WnyHYQKBgQChNn7uvIIUzQKoxYyvXk5NVIb1xey64deOVnayg66s6oHF9UHtynay +equq5maSgyF3Ft597koiG7ylMkcXkdsmUPos26ZSDlgqYC9v+VopqQB3T06ltyGA +lOjjVjzofTSzMvQ1UFbDoogelpnZNn8koKrgrUuZL+A0VLEYQTFITg== +-----END RSA PRIVATE KEY----- diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-news/news.js b/zefie_wtvp_minisrv/ServiceVault/wtv-news/news.js index 5d065808..a711a67c 100644 --- a/zefie_wtvp_minisrv/ServiceVault/wtv-news/news.js +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-news/news.js @@ -1,12 +1,32 @@ var minisrv_service_file = true; -const wtvnews = new WTVNews(minisrv_config, service_name); +const wtvnews = new WTVNews(minisrv_config, service_name); +var service_config = minisrv_config.services[service_name]; +if (service_config.local_nntp_port && wtvnewsserver) { + var tls_path = this.wtvshared.getAbsolutePath(this.minisrv_config.config.ServiceDeps + '/wtv-news'); + var tls_options = { + ca: this.fs.readFileSync(tls_path + '/localserver_ca.pem'), + key: this.fs.readFileSync(tls_path + '/localserver_key.pem'), + cert: this.fs.readFileSync(tls_path + '/localserver_cert.pem'), + checkServerIdentity: () => { return null; } + } + if (wtvnewsserver.username) + wtvnews.initializeUsenet("127.0.0.1", minisrv_config.services[service_name].local_nntp_port, tls_options, wtvnewsserver.username, wtvnewsserver.password); + else + wtvnews.initializeUsenet("127.0.0.1", minisrv_config.services[service_name].local_nntp_port, tls_options); +} else { + if (service_config.upstream_auth) + wtvnews.initializeUsenet(service_config.upstream_address, service_config.upstream_port, service_config.upstream_tls || null, service_config.upstream_auth.username || null, service_config.upstream_auth.password || null); + else + wtvnews.initializeUsenet(service_config.upstream_address, service_configupstream_port, service_config.upstream_tls || null); +} + async function throwError(e) { + console.log(e); var errpage = wtvshared.doErrorPage(400, null, e.toString()); sendToClient(socket, errpage[0], errpage[1]); } - function isToday (chkdate) { const today = new Date() return chkdate.getDate() == today.getDate() && @@ -28,7 +48,6 @@ async function WebTVListGroup(group) { page_start = (limit_per_page * page) + 1; page_end = (page + 1) * limit_per_page; if (page_end > NGCount) page_end = NGCount; - wtvnews.getHeaderObj(NGArticles).then((messages) => { messages = wtvnews.sortByResponse(messages); wtvnews.quitUsenet(); @@ -157,7 +176,9 @@ Group: ${request_headers.query.group}
- +`; + if (NGCount > 0) { + data += `
@@ -182,8 +203,9 @@ ${page_start}-${page_end}
-
- + `; + } + data += `
@@ -206,6 +228,7 @@ ${page_start}-${page_end} var has_relation = (messages[k].relation !== null) ? true : false; var date_obj = new Date(Date.parse(message.headers.DATE)); var date = (isToday(date_obj)) ? strftime("%I:%M %p", date_obj) : strftime("%b %d", date_obj) + console.log(message); data += ` @@ -230,8 +253,6 @@ ${(message.headers.FROM.indexOf(' ') > 0) ? message.headers.FROM.split(' ')[0] :
- -
@@ -264,7 +285,6 @@ async function WebTVShowMessage(group, article) { wtvnews.getArticle(article).then((response) => { wtvnews.quitUsenet(); if (response.code == 220) { - console.log(response); headers = `200 OK Content-type: text/html wtv-expire-all: wtv-news:/news?group=${group}&article=`; @@ -573,7 +593,7 @@ ${wtvshared.htmlEntitize(message_body, true)} });; } -if (!minisrv_config.services[service_name].upstream_address || !minisrv_config.services[service_name].upstream_port) { +if (!wtvnews.client) { var errpage = doErrorPage(); headers = errpage[0]; data = errpage[1]; diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-setup/edit-password.js b/zefie_wtvp_minisrv/ServiceVault/wtv-setup/edit-password.js index 29d01e0a..05eb0d76 100644 --- a/zefie_wtvp_minisrv/ServiceVault/wtv-setup/edit-password.js +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-setup/edit-password.js @@ -123,7 +123,7 @@ MAXLENGTH="${minisrv_config.config.passwords.max_length}">

Type again to confirm
- minisrv_config.config.passwords.max_length) errpage = wtvshared.doErrorPage(400, "Your password must contain no more than than " + minisrv_config.config.passwords.max_length + " characters."); - else if (request_headers.query.password !== request_headers.query.verify) errpage = wtvshared.doErrorPage(400, "The passwords you entered did not match. Please check them and try again."); + else if (request_headers.query.password !== request_headers.query.password_verify) errpage = wtvshared.doErrorPage(400, "The passwords you entered did not match. Please check them and try again."); else { if (errpage) { headers = errpage[0]; diff --git a/zefie_wtvp_minisrv/includes/WTVNews.js b/zefie_wtvp_minisrv/includes/WTVNews.js index 72dd2b84..7583fa23 100644 --- a/zefie_wtvp_minisrv/includes/WTVNews.js +++ b/zefie_wtvp_minisrv/includes/WTVNews.js @@ -6,23 +6,55 @@ class WTVNews { wtvshared = null; service_name = null; client = null; + username = null; + password = null; constructor(minisrv_config, service_name) { this.minisrv_config = minisrv_config; this.service_name = service_name; const { WTVShared } = require("./WTVShared.js"); this.wtvshared = new WTVShared(minisrv_config); - this.client = new this.newsie({ - host: this.minisrv_config.services[service_name].upstream_address, - port: this.minisrv_config.services[service_name].upstream_port - }) + } + + initializeUsenet(host, port = 119, tls_options = null, username = null, password = null) { + // use local self-signed cert for local server + var newsie_options = { + host: host, + port: port, + tlsPort: (tls_options !== null) ? true : false, + } + if (newsie_options.tlsPort) newsie_options.tlsOptions = tls_options; + this.client = new this.newsie(newsie_options); + if (username && password) { + this.username = username; + this.password = password; + } } connectUsenet() { return new Promise((resolve, reject) => { this.client.connect().then((response) => { - if (response.code == 200) { - resolve(true); + if (response.code == 200 || response.code == 201) { + if (this.username && this.password) { + this.client.authInfoUser(this.username).then((res) => { + if (res.code == "381") { + res.authInfoPass(this.password).then((res) => { + if (res.code == 281) resolve(true); + else reject(res.description); + }).catch((e) => { + console.error(" * WTVNews Error:", "Command: connect", e); + reject("Could not connect to upstream usenet server"); + }); + } else { + reject(res.description) + } + }).catch((e) => { + console.error(" * WTVNews Error:", "Command: connect", e); + reject("Could not connect to upstream usenet server"); + }); + } else { + resolve(true); + } } }).catch((e) => { console.error(" * WTVNews Error:", "Command: connect", e); @@ -31,16 +63,19 @@ class WTVNews { }); } - listGroup(group, page = 0, limit = 100) { + listGroup(group, page = 0, limit = 100, raw_range = null) { return new Promise((resolve, reject) => { this.selectGroup(group).then((res) => { - var range = { - start: (limit * page) + res.group.low, + if (!raw_range) { + var range = { + start: (limit * page) + res.group.low, + } + range.end = range.start + limit; + if (page) range.start++; + if (range.end > res.high) delete range.group.end; + } else { + range = raw_range; } - range.end = range.start + limit; - if (page) range.start++; - if (range.end > res.high) delete range.group.end; - console.log(res, range); this.client.listGroup(group, range).then((data) => { resolve(data); }).catch((e) => { @@ -75,7 +110,8 @@ class WTVNews { this.client.next().then((res) => { data.next_article = res.article.articleNumber; resolve(data.next_article); - }).catch(() => { + }).catch((e) => { + console.log(e); data.next_article = null; resolve(data.next_article); }) @@ -150,15 +186,12 @@ class WTVNews { promises.push(new Promise((resolve, reject) => { this.selectGroup(group).then((res) => { this.getArticleMessageID(article).then((data) => { - console.log(data); messageid = data; resolve(data); }).catch((e) => { - console.log(e); reject(e) }); }).catch((e) => { - console.log(e); reject(e) }); })); @@ -185,7 +218,6 @@ class WTVNews { } else { articleData.body = []; } - console.log(articleData); response.send(articleData).then((response) => { this.client.quit(); if (response.code !== 240) { @@ -233,7 +265,6 @@ class WTVNews { if (data.article) messages.push(data.article) resolve(); }).catch((e) => { - console.log(e, article); reject(e); }); })); @@ -285,9 +316,6 @@ class WTVNews { }); } }) - if (!found) { - message_id_roots.push({ "messageId": messageId, "index": j }); - } } else { message_id_roots.push({ "messageId": messageId, "index": k }); } @@ -306,7 +334,7 @@ class WTVNews { var article_date = Date.parse(article.headers.DATE); message_roots_sorted.push({ "article": article, "relation": null, "date": article_date }); }); - message_roots_sorted.sort((a, b) => { return (a.date > b.date) }); + message_roots_sorted.sort((a, b) => { return (a.date - b.date) }); Object.keys(message_roots_sorted).forEach((k) => { sorted.push(message_roots_sorted[k]); @@ -318,7 +346,7 @@ class WTVNews { var article_date = Date.parse(article.headers.DATE); relations.push({ "article": article, "relation": message_id_roots[k].messageId, "date": article_date }) }); - relations.sort((a, b) => { return (a.date > b.date) }); + relations.sort((a, b) => { return (a.date - b.date) }); Object.keys(relations).forEach((j) => { sorted.push(relations[j]); }); diff --git a/zefie_wtvp_minisrv/includes/WTVNewsServer.js b/zefie_wtvp_minisrv/includes/WTVNewsServer.js new file mode 100644 index 00000000..a5780653 --- /dev/null +++ b/zefie_wtvp_minisrv/includes/WTVNewsServer.js @@ -0,0 +1,216 @@ +class WTVNewsServer { + fs = require('fs'); + path = require('path'); + minisrv_config = null; + strftime = require('strftime'); + wtvshared = null; + username = null; + password = null; + using_auth = false; + local_server = null; + data_path = null; + + constructor(minisrv_config, local_server_port, using_auth = false, username = null, password = null) { + this.minisrv_config = minisrv_config; + const { WTVShared } = require("./WTVShared.js"); + this.wtvshared = new WTVShared(minisrv_config); + const nntp_server = require('nntp-server'); + var nntp_commands = { + ...nntp_server.commands, + "LAST": { + head: 'LAST', + validate: /^LAST( [^\s]+)$/i, + + // All supported params are defined in separate files + run() { + console.log('hi'); + throw new Error('method LAST is not implemented'); + } + }, + "NEXT": { + head: 'NEXT', + validate: /^NEXT( [^\s]+)$/i, + + // All supported params are defined in separate files + run() { + console.log('hi'); + throw new Error('method NEXT is not implemented'); + } + } + } + + console.log(nntp_commands); + + this.username == username || null; + this.password == password || null; + this.using_auth = using_auth; + if (using_auth && (!username && !password)) { + // using auth, but no auth info specified, so randomly generate it + this.username = this.wtvshared.generatePassword(8); + this.password = this.wtvshared.generatePassword(16); + } + + // nntp-server module overrides + var self = this; + + nntp_server.prototype = { + ...nntp_server.prototype, + _authenticate: function (session) { + // authenticte + if (session.authinfo_user == self.username && session.authinfo_pass == self.password) return Promise.resolve(true); + return Promise.resolve(false); + }, + + _selectGroup: function (session, name) { + // selectGroup + var res = self.selectGroup(name); + if (!res.failed) { + session.group = res; + return true; + } + return false; + }, + + _buildHead: function (session, message) { + var out = ""; + Object.keys(message.headers).forEach((k) => { + if (k.length > 0) out += `${k}: ${message.headers[k]}\r\n`; + }); + out = out.substr(0,out.length - 2); + return out; + }, + + _buildHeaderField: function (session, message, field) { + var search = Object.keys(message.headers).find(e => (e.toLowerCase() == field.toLowerCase())); + if (search) return message.headers[search]; + else return null; + }, + + _getArticle: function (session, message_id) { + // getArticle + return new Promise((resolve, reject) => { + self.getArticle(session.group.name, message_id).then((res) => { + resolve(res); + }).catch((e) => { + console.log(" * WTVNewsServer Error:", e); + reject(e); + }); + }); + }, + + _buildBody: function (session, message) { + return message.body; + }, + + _getRange: function (session, first, last) { + var res = self.listGroup(session.group.name, first, last) + if (res.failed) return false; + session.group = res.group_data; + return res.articleNumbers; + } + + } + + this.data_path = this.wtvshared.getAbsolutePath(this.minisrv_config.config.SessionStore + '/minisrv_internal_nntp'); + this.createDataStore(); + + var tls_path = this.wtvshared.getAbsolutePath(this.minisrv_config.config.ServiceDeps + '/wtv-news'); + var tls_options = { + ca: this.fs.readFileSync(tls_path + this.path.sep + 'localserver_ca.pem'), + key: this.fs.readFileSync(tls_path + this.path.sep + 'localserver_key.pem'), + cert: this.fs.readFileSync(tls_path + this.path.sep + 'localserver_cert.pem'), + } + this.local_server = new nntp_server({ requireAuth: using_auth, tls: tls_options, secure: true, commands: nntp_commands }); + this.local_server.listen('nntps://localhost:' + local_server_port); + } + + createDataStore() { + if (!this.fs.existsSync(this.data_path)) return this.fs.mkdirSync(this.data_path); + return true; + } + + getGroupPath(group) { + return this.data_path + this.path.sep + group; + } + + getArticlePath(group, article) { + return this.getGroupPath(group) + this.path.sep + article + ".newz"; + } + + createGroup(group) { + if (!this.fs.existsSync(getGroupPath(group))) return this.fs.mkdirSync(getGroupPath(group)); + return true; + } + + getArticle(group, article) { + return new Promise((resolve, reject) => { + const g = this.getArticlePath(group, article); + if (!this.fs.existsSync(g)) return false; + try { + var data = JSON.parse(this.fs.readFileSync(g)); + resolve(data); + } catch (e) { + console.log(" * WTVNewsServer Error:", e); + reject(e) + } + }); + } + + selectGroup(group) { + var g = this.getGroupPath(group); + var out = { + total: 0, + min_index: null, + max_index: 0, + name: group + } + try { + this.fs.readdirSync(g).forEach(file => { + var articleNumber = parseInt(file.split('.')[0]); + if (out.min_index == null) out.min_index = articleNumber; + else if (articleNumber < out.min_index) out.min_index = articleNumber; + + if (articleNumber > out.max_index) out.max_index = articleNumber; + out.total++; + }); + } catch (e) { + out.failed = e; + } + if (out.min_index === null) out.min_index = 0; + return out; + } + + listGroup(group, start, end) { + var g = this.getGroupPath(group); + var out = { + total: 0, + min_index: null, + max_index: 0, + name: group + } + var articleNumbers = []; + try { + this.fs.readdirSync(g).forEach(file => { + var articleNumber = parseInt(file.split('.')[0]); + if (articleNumber < start) return; + if (articleNumber > end) return false; + if (out.min_index == null) out.min_index = articleNumber; + else if (articleNumber < out.min_index) out.min_index = articleNumber; + + if (articleNumber > out.max_index) out.max_index = articleNumber; + articleNumbers.push({ index: articleNumber }); + out.total++; + }); + } catch (e) { + console.log(e); + out.failed = e; + } + if (out.min_index === null) out.min_index = 0; + return { + articleNumbers: articleNumbers, + group_data: out + } + } +} + +module.exports = WTVNewsServer; \ No newline at end of file diff --git a/zefie_wtvp_minisrv/includes/config.json b/zefie_wtvp_minisrv/includes/config.json index ea196974..9342098d 100644 --- a/zefie_wtvp_minisrv/includes/config.json +++ b/zefie_wtvp_minisrv/includes/config.json @@ -6,7 +6,7 @@ "UserServiceVault", "ServiceVault" ], - "ServiceDeps": "ServiceDeps", + "ServiceDeps": "ServiceDeps", "SessionStore": "SessionStore", "SharedROMCache": "SharedROMCache", "enable_shared_romcache": true, @@ -31,8 +31,12 @@ "show_detailed_splash": true, "show_diskmap": false, "unauthorized_url": "wtv-1800:/unauthorized?", + "enable_port_isolation": true, "allow_guests": true, "domain_name": "wtv.zefie.com", + "ssid_block_list": [ + "minisrv_internal_nntp" + ], "user_accounts": { "max_users_per_account": 6, "min_username_length": 5, @@ -60,7 +64,8 @@ }, "wtv-news": { "port": 1605, - "disabled": true, + "local_nntp_port": 57319, + "local_nntp_requires_auth": true, "modules": [ "WTVNews" ] diff --git a/zefie_wtvp_minisrv/package-lock.json b/zefie_wtvp_minisrv/package-lock.json index 5fd8a344..f14c8579 100644 --- a/zefie_wtvp_minisrv/package-lock.json +++ b/zefie_wtvp_minisrv/package-lock.json @@ -1,12 +1,12 @@ { "name": "zefie_wtvp_minisrv", - "version": "0.9.32", + "version": "0.9.33", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "zefie_wtvp_minisrv", - "version": "0.9.32", + "version": "0.9.33", "license": "GPL3", "dependencies": { "adm-zip": "^0.5.9", @@ -18,6 +18,7 @@ "iconv-lite": "^0.6.3", "mime-types": "^2.1.35", "newsie": "^1.2.1", + "nntp-server": "^3.1.0", "proxy-agent": "^5.0.0", "sanitize-html": "^2.7.2", "socks-proxy-agent": "^6.2.1", @@ -263,6 +264,14 @@ "node": ">= 6" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -861,6 +870,17 @@ "node": ">=8.9.0" } }, + "node_modules/nntp-server": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/nntp-server/-/nntp-server-3.1.0.tgz", + "integrity": "sha512-L8U2CnFPHXfDKF+eaYpx07fCW7gbk7gNB0faDWg5fLIEofEP7O/RIUEonl0jGRDf9/eGMj7CRil1faG5YZq4bw==", + "dependencies": { + "debug": "^4.3.3", + "denque": "^2.0.1", + "serialize-error": "^8.1.0", + "split2": "^4.1.0" + } + }, "node_modules/object-inspect": { "version": "1.12.2", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz", @@ -1179,6 +1199,20 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, + "node_modules/serialize-error": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz", + "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==", + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/serve-static": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", @@ -1268,6 +1302,14 @@ "node": ">=0.10.0" } }, + "node_modules/split2": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz", + "integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ==", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -1313,6 +1355,17 @@ "node": ">= 0.8.0" } }, + "node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -1576,6 +1629,11 @@ "vm2": "^3.9.8" } }, + "denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" + }, "depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -2016,6 +2074,17 @@ "resolved": "https://registry.npmjs.org/newsie/-/newsie-1.2.1.tgz", "integrity": "sha512-41jhtKmlpSc27oIBkp681fm4aLURPT/AgeBvSRicxNPWDVeHLaq7tSvG/OsEXz7g41thgv9JMV1ZjYSABMiYVA==" }, + "nntp-server": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/nntp-server/-/nntp-server-3.1.0.tgz", + "integrity": "sha512-L8U2CnFPHXfDKF+eaYpx07fCW7gbk7gNB0faDWg5fLIEofEP7O/RIUEonl0jGRDf9/eGMj7CRil1faG5YZq4bw==", + "requires": { + "debug": "^4.3.3", + "denque": "^2.0.1", + "serialize-error": "^8.1.0", + "split2": "^4.1.0" + } + }, "object-inspect": { "version": "1.12.2", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz", @@ -2266,6 +2335,14 @@ } } }, + "serialize-error": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz", + "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==", + "requires": { + "type-fest": "^0.20.2" + } + }, "serve-static": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", @@ -2334,6 +2411,11 @@ "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==" }, + "split2": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz", + "integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ==" + }, "statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -2367,6 +2449,11 @@ "prelude-ls": "~1.1.2" } }, + "type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==" + }, "type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", diff --git a/zefie_wtvp_minisrv/package.json b/zefie_wtvp_minisrv/package.json index 60e5185f..d960e384 100644 --- a/zefie_wtvp_minisrv/package.json +++ b/zefie_wtvp_minisrv/package.json @@ -36,6 +36,7 @@ "iconv-lite": "^0.6.3", "mime-types": "^2.1.35", "newsie": "^1.2.1", + "nntp-server": "^3.1.0", "proxy-agent": "^5.0.0", "sanitize-html": "^2.7.2", "socks-proxy-agent": "^6.2.1", diff --git a/zefie_wtvp_minisrv/sync_nntp.js b/zefie_wtvp_minisrv/sync_nntp.js new file mode 100644 index 00000000..a547b564 --- /dev/null +++ b/zefie_wtvp_minisrv/sync_nntp.js @@ -0,0 +1,138 @@ +// no wildcard (yet?) +const groups_to_sync = [ + "webtv.users" +] + + + +const fs = require('fs'); +const path = require('path'); +var classPath = __dirname + "/includes/"; +const { WTVShared } = require(classPath + "WTVShared.js"); +const wtvshared = new WTVShared(); // creates minisrv_config +classPath = wtvshared.getAbsolutePath(classPath, __dirname); +const minisrv_config = wtvshared.getMiniSrvConfig(); +const WTVNews = require(classPath + "/WTVNews.js"); +var data_path = wtvshared.getAbsolutePath(minisrv_config.config.SessionStore + '/minisrv_internal_nntp'); +const service_name = "wtv-news"; + +if (!minisrv_config.services[service_name].upstream_address) { + console.error("minisrv config not configured with usenet upstream"); + process.exit(1); +} + +if (!minisrv_config) { + console.error("Something went wrong loading minisrv config"); + process.exit(1); +} + +if (!minisrv_config.config.debug_flags.quiet) console.log(" *** Successfully read minisrv configuration...."); + + +const service_config = minisrv_config.services[service_name]; +const wtvnews = new WTVNews(minisrv_config, service_name); + +if (service_config.upstream_auth) { + wtvnews.initializeUsenet(service_config.upstream_address, service_config.upstream_port, service_config.upstream_tls || null, service_config.upstream_auth.username || null, service_config.upstream_auth.password || null); +} else { + wtvnews.initializeUsenet(service_config.upstream_address, service_configupstream_port, service_config.upstream_tls || null); +} + + + +function createDataStore() { + if (!fs.existsSync(data_path)) return fs.mkdirSync(data_path); + return true; +} + +function getGroupPath(group) { + return data_path + path.sep + group; +} + +function createGroup(group) { + createDataStore(); + if (!fs.existsSync(getGroupPath(group))) return fs.mkdirSync(getGroupPath(group)); + return true; +} + +function createArticle(group, articleNumber, article) { + var g = getGroupPath(group); + var file = g + path.sep + articleNumber + ".newz"; + if (fs.existsSync(file)) return "exists"; + else { + try { + article.article.index = article.article.articleNumber; + delete article.article.articleNumber; + article.article['message-id'] = article.article.messageId; + delete article.article.messageId; + + fs.writeFileSync(file, JSON.stringify(article.article)); + return file; + } catch (e) { + return e; + } + } +} + +function deleteMissing(group, articles) { + var g = getGroupPath(group); + try { + fs.readdirSync(g).forEach(file => { + var articleNumber = parseInt(file.split('.')[0]); + file = g + path.sep + file; + if (!articles.find(e => (e == articleNumber))) { + console.log(" * ", group, "article", articleNumber, "deleted from upstream, removing locally") + fs.rmSync(file); + } + }); + return true; + } catch (e) { + console.log(e); + return false; + } +} + +wtvnews.connectUsenet().then((res) => { + if (res) { + groups_to_sync.forEach((group) => { + console.log(group); + wtvnews.selectGroup(group).then((res) => { + var range = { + start: res.group.low, + end: res.group.high + } + wtvnews.listGroup(group, null, null, range).then((res) => { + if (res.group.articleNumbers) { + var promises = []; + createGroup(group); + deleteMissing(group, res.group.articleNumbers) + res.group.articleNumbers.forEach((article) => { + promises.push(new Promise((resolve, reject) => { + wtvnews.getArticle(article).then((message) => { + res = createArticle(group, article, message); + if (res) { + if (res == "exists") { + console.log(" * ", group, "article", article, "already exists") + } else { + console.log(" * Created", group, "article", article, res) + } + resolve(true) + } else { + console.log(" ! Failed to create file for ", group, "article", article, "with error", res); + resolve(false) + } + }); + })); + }) + Promise.all(promises).then(() => { + wtvnews.quitUsenet(); + process.exit(0); + }); + } else { + wtvnews.quitUsenet(); + } + }) + }); + }) + } +}); diff --git a/zefie_wtvp_minisrv/user_config.example.json b/zefie_wtvp_minisrv/user_config.example.json index e09e29b1..dd46416f 100644 --- a/zefie_wtvp_minisrv/user_config.example.json +++ b/zefie_wtvp_minisrv/user_config.example.json @@ -67,8 +67,13 @@ "upstream_address": "192.168.11.8", "upstream_port": 1699, "upstream_auth": { - "username": "not_yet", - "password": "implemented" + "username": "myusername", + "password": "mypassword" + }, + "upstream_tls": true, + "local_auth": { + "username": "mylocaluser", + "password": "mylocalpass" } } }, @@ -77,16 +82,17 @@ "Personal": [ { "title": "The Midnight Archive", - "url": "http://archive.midnightchanne.net/", + "url": "http://archive.midnightchannel.net/", "image_type": "url", "image": "canned/midnightchannel.gif" }, { "title": "WebTV/MSNTV Secrets", - "url": "http://turdinc.kicks-ass.net/Msntv/", + "url": "http://turdinc.kicks-ass.net/Msntv/index2.html", "image_type": "url", "image": "canned/mattman69.gif" } ] } - } \ No newline at end of file + } +} \ No newline at end of file diff --git a/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj b/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj index 55dbdc16..686289d6 100644 --- a/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj +++ b/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj @@ -38,6 +38,7 @@ Code +