initial attachment support for usenet

- can read a few attachments posted by PCs
- can not yet attach from WebTV Plus

zefie_wtvp_minisrv/ServiceVault/wtv-news/get-attachment.js

@@ -0,0 +1,70 @@
+var minisrv_service_file = true;
+
+var request_is_async = true;
+var errpage = null;
+var group = request_headers.query.group;
+var article = request_headers.query.article;
+var attachment_id = parseInt(request_headers.query.attachment_id);
+if ((!attachment_id && attachment_id != 0) || !group || !article) {
+    errpage = wtvshared.doErrorPage(400, "Attachment ID required.");
+    sendToClient(socket, errpage[0], errpage[1]);
+} else {
+    const wtvnews = new WTVNews(minisrv_config, service_name);
+    var service_config = minisrv_config.services[service_name];
+    if (service_config.local_nntp_port && wtvnewsserver) {
+        var tls_path = this.wtvshared.getAbsolutePath(this.minisrv_config.config.ServiceDeps + '/wtv-news');
+        var tls_options = {
+            ca: this.fs.readFileSync(tls_path + '/localserver_ca.pem'),
+            key: this.fs.readFileSync(tls_path + '/localserver_key.pem'),
+            cert: this.fs.readFileSync(tls_path + '/localserver_cert.pem'),
+            checkServerIdentity: () => { return null; }
+        }
+        if (wtvnewsserver.username)
+            wtvnews.initializeUsenet("127.0.0.1", service_config.local_nntp_port, tls_options, wtvnewsserver.username, wtvnewsserver.password);
+        else
+            wtvnews.initializeUsenet("127.0.0.1", service_config.local_nntp_port, tls_options);
+    } else {
+        if (service_config.upstream_auth)
+            wtvnews.initializeUsenet(service_config.upstream_address, service_config.upstream_port, service_config.upstream_tls || null, service_config.upstream_auth.username || null, service_config.upstream_auth.password || null);
+        else
+            wtvnews.initializeUsenet(service_config.upstream_address, service_config.upstream_port, service_config.upstream_tls || null);
+    }
+    var article = parseInt(article);
+    wtvnews.connectUsenet().then(() => {
+        wtvnews.selectGroup(group).then((response) => {
+            wtvnews.getArticle(article).then((response) => {
+                wtvnews.quitUsenet();
+                if (response.code == 220) {
+                    var message_data = wtvnews.parseAttachments(response);
+                    if (message_data.attachments) {
+                        if (attachment_id < message_data.attachments.length) {
+                            var attachment = message_data.attachments[attachment_id];
+                            console.log(attachment);
+                            var encoding = attachment.content_encoding.toLowerCase()
+                            if (encoding == 'base64') {
+                                data = Buffer.from(attachment.data, encoding);
+                                headers = "200 OK\n"
+                                headers += "Content-Type: " + attachment.content_type + "\n";
+                                if (attachment.filename) headers += "Content-Disposition: attachment; filename=\"" + attachment.filename + "\"\n";
+                                sendToClient(socket, headers, data);
+                            } else {
+                                errpage = wtvshared.doErrorPage(400, "Unimplemented encoding type:", encoding);
+                                sendToClient(socket, errpage[0], errpage[1]);
+                            }
+
+                        } else {
+                            errpage = wtvshared.doErrorPage(400, "Attachment ID exceeds available attachments");
+                            sendToClient(socket, errpage[0], errpage[1]);
+                        }
+                    } else {
+                        errpage = wtvshared.doErrorPage(400, "Article does not contain attachments.");
+                        sendToClient(socket, errpage[0], errpage[1]);
+                    }
+                } else {
+                    errpage = wtvshared.doErrorPage(400);
+                    sendToClient(socket, errpage[0], errpage[1]);
+                }
+            });
+        });
+    });
+}

zefie_wtvp_minisrv/ServiceVault/wtv-news/news.js

@@ -544,25 +544,33 @@ ${(response.article.headers.SUBJECT) ? wtvshared.htmlEntitize(response.article.h
 <td abswidth=20 rowspan=99>
 <tr>
 <td>
-`;
+`;                    
-                    var message_body = response.article.body.join("\n");
+                    var message = wtvnews.parseAttachments(response);
+                    var message_body = message.text;
+                    var attachments = null;
+                    if (message.attachments) attachments = message.attachments;
                     data += `
 ${wtvshared.htmlEntitize(message_body, true)}
 <br>
 <br>`;
                     data += "<p>";
-                    /*
+                    
-                    if (message.attachments) {
+                    if (attachments) {
-                        message.attachments.forEach((v, k) => {
+                        attachments.forEach((v, k) => {
                             if (v) {
-                                console.log("*****************", v['Content-Type']);
+                                switch (v.content_type) {
-                                switch (v['Content-Type']) {
                                     case "image/jpeg":
-                                        data += `<img border=2 src="wtv-news:/get-attachment?message_id=${messageid}&attachment_id=${k}&group=${(message.to_group)}&wtv-title=Video%20Snapshot" width="380" height="290"><br><br>`;
+                                    case "image/png":
+                                    case "image/gif":
+                                        data += `<img border=2 src="wtv-news:/get-attachment?group=${group}&article=${article}&attachment_id=${k}wtv-title=Video%20Snapshot" width="380" height="290"><br><br>`;
                                         break;
                                     case "audio/wav":
-                                        data += `<table href="wtv-news:/get-attachment?message_id=${messageid}&attachment_id=${k}&group=${(message.to_group)}&wtv-title=Voice%20Mail" width=386 cellspacing=0 cellpadding=0>
+                                    case "audio/mp2":
-    <td align=left valign=middle><img src="wtv-mail:/ROMCache/FileSound.gif" align=absmiddle><font color="#189CD6">&nbsp;&nbsp;recording.wav (wav attachment)</font>
+                                    case "audio/mp3":
+                                    case "audio/mid":
+                                    case "audio/midi":
+                                        data += `<table href="wtv-news:/get-attachment?group=${group}&article=${article}&attachment_id=${k}&wtv-title=Audio%20file" width=386 cellspacing=0 cellpadding=0>
+    <td align=left valign=middle><img src="wtv-news:/ROMCache/FileSound.gif" align=absmiddle><font color="#189CD6">&nbsp;&nbsp;${(v.filename) ? (v.filename) : "Audio file"} (${v.content_type.split('/')[1]} attachment)</font>
     <td align=right valign=middle>
     </table><br><br>
     `;
@@ -571,6 +579,7 @@ ${wtvshared.htmlEntitize(message_body, true)}
                             }
                         });
                     }
+                    /*
                     if (message.url) {
                         data += `Included Page: <a href="${(message.url)}">${wtvshared.htmlEntitize(message.url_title).replace(/&apos;/gi, "'")}`;
                     }

zefie_wtvp_minisrv/includes/WTVNews.js

@@ -147,9 +147,17 @@ class WTVNews {
                     }));
 
                     Promise.all(promises).then(() => {
+                        var self = this;
+                        if (data.article.headers) Object.keys(data.article.headers).forEach((k) => {
+                            data.article.headers[k] = self.decodeCharset(data.article.headers[k])
+                        });
                         resolve(data);
                     });
                 } else {
+                    var self = this;
+                    if (data.article.headers) Object.keys(data.article.headers).forEach((k) => {
+                        data.article.headers[k] = self.decodeCharset(data.article.headers[k])
+                    });
                     resolve(data);
                 }
             }).catch((e) => {
@@ -159,9 +167,37 @@ class WTVNews {
         });
     }
 
+    decodeCharset(string) {
+        var regex = /=\?{1}(.+)\?{1}([B|Q])\?{1}(.+)\?{1}=/;
+        var decoded = null;
+        var check = string.match(regex);
+        if (check) {
+            var match = check[0];
+            var charset = check[1];
+            var encoding = check[2];
+            var encoded_text = check[3];
+            switch (encoding) {
+                case "B":
+                    var buffer = new Buffer.from(encoded_text, 'base64')
+                    decoded = buffer.toString(charset).replace(/[^\x00-\x7F]/g, "");;
+                    break;
+
+                case "Q":
+                    // unimplemented
+                    return string;
+            }
+            if (decoded) return string.replace(match, decoded);
+        }
+        return string;
+    }
+
     getHeader(articleID) {
         return new Promise((resolve, reject) => {
             this.client.head(articleID).then((data) => {
+                var self = this;
+                if (data.article.headers) Object.keys(data.article.headers).forEach((k) => {
+                    data.article.headers[k] = self.decodeCharset(data.article.headers[k])
+                });
                 resolve(data);
             }).catch((e) => {
                 reject(`Error getting header for article ID ${articleID}`);
@@ -170,6 +206,19 @@ class WTVNews {
         });
     }
 
+    getHeaderFromMessage(message, header) {
+        var response = null;
+        if (message.article.headers) {
+            Object.keys(message.article.headers).forEach((k) => {
+                if (k.toLowerCase() == header.toLowerCase()) {
+                    response = message.article.headers[k];
+                    return false;
+                }
+            })
+        }
+        return response;
+    }
+
     quitUsenet() {
         return new Promise((resolve, reject) => {
             this.client.quit().then((response) => {
@@ -220,7 +269,10 @@ class WTVNews {
                                         'Message-ID': "<" + this.wtvshared.generatePassword(16) + "@" + this.minisrv_config.config.domain_name + ">",
                                         'Date': this.strftime('%A, %d-%b-%y %k:%M:%S %z', new Date())
                                     }
-                                    if (messageid) articleData.headers.References = messageid;
+                                    if (messageid) {
+                                        articleData.headers.References = messageid;
+                                        articleData.headers['In-Reply-To'] = messageid;
+                                    }
 
                                     if (msg_body) {
                                         articleData.body = msg_body.split("\n");
@@ -290,6 +342,70 @@ class WTVNews {
         });
     }    
 
+
+    parseAttachments(message) {
+        var contype = this.getHeaderFromMessage(message, 'Content-Type');
+        if (contype) {
+            var regex = /multipart\/mixed\; boundary=\"(.+)\"/i;
+            var match = contype.match(regex);
+            if (match) {
+                var boundary = "--" + match[1];
+                var body = message.article.body.join("\n").split(boundary);
+                var attachments = [];
+                var i = 0;
+                var message_body = '';
+                var message_type = 'text/plain';
+                body.forEach((element) => {
+                    var section_type = null;
+                    var section = element.split("\n");
+                    attachments[i] = {};
+                    section.forEach((line) => {
+                        var section_header_match = line.match(/^Content\-/i)
+                        if (section_header_match) {
+                            var section_match = line.match(/^Content\-Type\: (.+)\;/i)
+                            if (section_match) {
+                                if (section_match[1].match(/text\/(html|plain)/)) {
+                                    section_type = section_match[1].match(/(text\/(html|plain))/)[1];
+                                    message_type = section_type;
+                                } else {
+                                    section_type = section_match[1];
+                                    attachments[i].content_type = section_match[1]
+                                }
+                            }
+                            section_match = line.match(/^Content\-Disposition\: (.+)\;/i)
+                            if (section_match) {
+                                section_match = line.match(/^Content\-Disposition\: (.+)\; filename=\"(.+)\"/i)
+                                if (section_match) attachments[i].filename = section_match[2];
+                            }
+                            section_match = line.match(/^Content-Transfer-Encoding: (.+)/i)
+                            if (section_match) attachments[i].content_encoding = section_match[1];
+                        } else {
+                            if (section_type != null) {
+                                if (section_type.match(/(text\/[html|plain])/)) message_body += line;
+                                else {
+                                    if (attachments[i].data) attachments[i].data += line;
+                                    else attachments[i].data = line;
+                                }
+                            }
+                        }
+                    })
+                    if (attachments[i].content_type) i++;
+                })
+                attachments.pop();
+                return {
+                    text: message_body,
+                    text_type: message_type,
+                    attachments: attachments
+                }
+            } else {
+                return { text: message.article.body.join("\n") }
+            }
+        } else {
+            return { text: message.article.body.join("\n") }
+        }
+
+    }
+
     sortByResponse(messages) {
         var sorted = [];
         var message_id_roots = [];

zefie_wtvp_minisrv/includes/WTVNewsServer.js

@@ -93,7 +93,6 @@ class WTVNewsServer {
             },
 
             _buildHeaderField: function (session, message, field) {
-                console.log(message,field);
                 if (field.indexOf(':') > 0) field = field.replace(/\:/g, '');
                 var search = self.getHeader(message, field);
                 if (search) return search;
@@ -148,13 +147,9 @@ class WTVNewsServer {
     }
 
     getHeader(message, header) {
-        try {
+        var search = Object.keys(message.headers).find(e => (e.toLowerCase() == header.toLowerCase()));
-            var search = Object.keys(message.headers).find(e => (e.toLowerCase() == header.toLowerCase()));
+        if (search) return message.headers[search];
-            if (search) return message.headers[search];
+        return null;
-            return null;
-        } catch (e) {
-            console.log(e);
-        }
     }
 
     createDataStore() {
@@ -172,6 +167,10 @@ class WTVNewsServer {
         try {
             post_data.articleNumber = articleNumber;
             post_data.messageId = this.getHeader(post_data, "message-id");
+            if (!post_data.messageId) {
+                var messageId = "<" + this.wtvshared.generatePassword(16) + "@" + this.minisrv_config.config.domain_name + ">";
+                post_data.messageId = post_data.headers['Message-ID'] = messageId;
+            }
             //Tue, 11 Oct 2022 17:25:16 -0400
             post_data.headers.Date = this.strftime("%a, %-d %b %Y %H:%M:%S %z", Date.parse(post_data.headers.date))
             post_data.headers['INJECTION-DATE'] = this.strftime("%a, %-d %b %Y %H:%M:%S %z", Date.parse(Date.now()))

zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj

@@ -934,6 +934,9 @@
     <Content Include="ServiceVault\wtv-music\get-playlist.js">
       <SubType>Code</SubType>
     </Content>
+    <Content Include="ServiceVault\wtv-news\get-attachment.js">
+      <SubType>Code</SubType>
+    </Content>
     <Content Include="ServiceVault\wtv-news\lobby.js" />
     <Content Include="ServiceVault\wtv-news\news.js" />
     <Content Include="ServiceVault\wtv-passport\messengerlogin.js" />