update metadata system for usenet

- probably have to wipe existing groups

zefie_wtvp_minisrv/ServiceVault/wtv-news/news.js

@@ -652,9 +652,7 @@ ${wtvshared.htmlEntitize(message_body, true)}
 }
 
 function WebTVSearchGroups(search) {
-    console.log('WebTVSearchGroups init')
     wtvnews.connectUsenet().then(() => {
-        console.log('WebTVSearchGroups connected')
         wtvnews.listGroups(search).then((response) => {
             console.log('WebTVSearchGroups listGroups response', response)
             wtvnews.quitUsenet();

zefie_wtvp_minisrv/includes/WTVNews.js

@@ -99,7 +99,6 @@ class WTVNews {
     listGroups(search = null) {
         // list of groups on the server
         return new Promise((resolve, reject) => {
-            if (search === "*") search = null;
             if (!search) {
                 this.client.list().then((data) => {
                     console.log('listGroups data', data)
@@ -109,11 +108,10 @@ class WTVNews {
                     reject(e);
                 });
             } else {
-                this.client.listNewsgroups('*' + search + '*').then((data) => {
+                this.client.listNewsgroups((search === '*') ? '*' : '*' + search + '*').then((data) => {
-                    console.log('listGroups data', data)
                     resolve(this.processGroupList(data));
                 }).catch((e) => {
-                    console.error(" * WTVNews Error:", "Command: listGroups (all)", e);
+                    console.error(" * WTVNews Error:", "Command: listGroups (search)", search, e);
                     reject(e);
                 });
             }

zefie_wtvp_minisrv/includes/WTVNewsServer.js

@@ -172,12 +172,10 @@ class WTVNewsServer {
     createMetaFile(group, description = null) {
         const g = this.getMetaFilename(group);
         if (this.fs.existsSync(g)) return false;
-        var metadata = {};
+        var metadata = this.selectGroup(group, true, true);
-        metadata.group = group;
-        metadata.last_article_id = this.selectGroup(group).max_index;
         if (description) metadata.description = description;
         this.saveMetadata(group, metadata, true);
-        return metadata;
+        return (!metadata.failed) ? metadata : false
     }
 
     saveMetadata(group, metadata, creating = false) {
@@ -196,12 +194,6 @@ class WTVNewsServer {
         } else return false;
     }
 
-    incrementArticleIdMeta(group) {
-        var metadata = this.getMetadata(group);
-        metadata.last_article_id = metadata.last_article_id + 1;
-        this.saveMetadata(group, metadata)
-    }
-
     findHeaderCaseInsensitive(headers, header) {
         // returns the key with the found case
         var response = null;
@@ -217,7 +209,7 @@ class WTVNewsServer {
     }
 
     postArticle(group, post_data) {
-        var articleNumber = this.getMetadata(group).last_article_id + 1;
+        var articleNumber = this.getMetadata(group).max_index + 1;
         if (!articleNumber) return false;
         try {
             post_data.articleNumber = articleNumber;
@@ -258,7 +250,10 @@ class WTVNewsServer {
         var file = g + this.path.sep + articleNumber + ".newz";
         try {
             this.fs.writeFileSync(file, JSON.stringify(article));
-            this.incrementArticleIdMeta(group);
+            var metadata = this.getMetadata(group);
+            metadata.max_index = metadata.max_index + 1;
+            metadata.total = metadata.total + 1;
+            this.saveMetadata(group, metadata)
             return true;
         } catch (e) {
             console.error(" * WTVNewsServer Error: createArticle: ", e);
@@ -284,8 +279,7 @@ class WTVNewsServer {
         var g = this.getGroupPath(group);
         if (!this.fs.existsSync(g)) {
             this.fs.mkdirSync(g);
-            this.createMetaFile(group, description)
+            return this.createMetaFile(group, description)
-            return this.fs.existsSync(g);
         }
         return false
     }
@@ -307,14 +301,19 @@ class WTVNewsServer {
     }
 
 
-    selectGroup(group) {
+    selectGroup(group, force_update = false, initial_update = false) {
         var g = this.getGroupPath(group);
+        if (initial_update) {
             var out = {
                 total: 0,
-            min_index: null,
+                min_index: 0,
                 max_index: 0,
                 name: group
             }
+        } else {
+            var meta = this.getMetadata(group);
+            var out = { ...meta }
+        } 
         if (this.featuredGroups) {
             Object.keys(this.featuredGroups).forEach((k) => {
                 if (group == this.featuredGroups[k].group) {
@@ -324,6 +323,7 @@ class WTVNewsServer {
             })
         }
         try {
+            if (force_update || this.doesMetaNeedRefreshing(meta)) {
                 this.fs.readdirSync(g).forEach(file => {
                     if (file == "meta.json") return;
                     var articleNumber = parseInt(file.split('.')[0]);
@@ -333,10 +333,18 @@ class WTVNewsServer {
                     if (articleNumber > out.max_index) out.max_index = articleNumber;
                     out.total++;
                 });
+                if (initial_update) {
+                    out.last_scan = Math.floor(Date.now() / 1000);
+                } else {
+                    meta = { ...meta, ...out }
+                    if (meta.wildmat) delete meta.wildmat;
+                    meta.last_scan = Math.floor(Date.now() / 1000);
+                    this.saveMetadata(group, meta);
+                } 
+            }
         } catch (e) {
             out.failed = e;
         }
-        if (out.min_index === null) out.min_index = 0;
         return out;
     }
 
@@ -403,16 +411,33 @@ class WTVNewsServer {
         return res;
     }
 
-    listGroup(group, start, end) {
+    doesMetaNeedRefreshing(meta) {
+        if (!meta) return true;
+        if (!meta.max_index) return true;
+        if (!meta.min_index) return true;
+        if (!meta.total) return true;
+        if (!meta.last_scan) return true;
+        if (meta.last_scan) {
+            if ((Math.floor(Date.now() / 1000) - this.scan_interval) > meta.last_scan) {
+                return true;
+            }
+        } 
+
+        return false;
+    }
+
+    listGroup(group, start, end, force_update = false) {
         var g = this.getGroupPath(group);
         var out = {
             total: 0,
-            min_index: null,
+            min_index: 0,
             max_index: 0,
             name: group
         }
         var articles = [];
         try {
+            var meta = this.getMetadata(group);
+            if (force_update || this.doesMetaNeedRefreshing(meta)) {
                 this.fs.readdirSync(g).forEach(file => {
                     if (file == "meta.json") return;
                     var articleNumber = parseInt(file.split('.')[0]);
@@ -425,6 +450,10 @@ class WTVNewsServer {
                     articles.push(this.getArticle(group, articleNumber));
                     out.total++;
                 });
+                meta = { ...meta, ...out }
+                meta.last_scan = Math.floor(Date.now() / 1000);
+                this.saveMetadata(group, meta);
+            }
         } catch (e) {
             console.error(" * WTVNewsServer Error: listGroup: ", e);
             out.failed = e;
@@ -433,7 +462,7 @@ class WTVNewsServer {
         if (out.min_index === null) out.min_index = 0;
         return {
             articles: articles,
-            group_data: out
+            group_data: meta
         }
     }
 }