zefie/minisrv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update metadata system for usenet

Browse Source 
- probably have to wipe existing groups
This commit is contained in:
zefie
2022-10-14 01:03:33 -04:00
parent a6659b4b16
commit c6bb1a6fee
3 changed files with 75 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
zefie_wtvp_minisrv/ServiceVault/wtv-news/news.js
View File

@@ -652,9 +652,7 @@ ${wtvshared.htmlEntitize(message_body, true)}
} }
function WebTVSearchGroups(search) { function WebTVSearchGroups(search) {
console.log('WebTVSearchGroups init')
wtvnews.connectUsenet().then(() => { wtvnews.connectUsenet().then(() => {
console.log('WebTVSearchGroups connected')
wtvnews.listGroups(search).then((response) => { wtvnews.listGroups(search).then((response) => {
console.log('WebTVSearchGroups listGroups response', response) console.log('WebTVSearchGroups listGroups response', response)
wtvnews.quitUsenet(); wtvnews.quitUsenet();

6
zefie_wtvp_minisrv/includes/WTVNews.js
View File

@@ -99,7 +99,6 @@ class WTVNews {
listGroups(search = null) { listGroups(search = null) {
// list of groups on the server // list of groups on the server
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
if (search === "*") search = null;
if (!search) { if (!search) {
this.client.list().then((data) => { this.client.list().then((data) => {
console.log('listGroups data', data) console.log('listGroups data', data)
@@ -109,11 +108,10 @@ class WTVNews {
reject(e); reject(e);
}); });
} else { } else {
this.client.listNewsgroups('*' + search + '*').then((data) => { this.client.listNewsgroups((search === '*') ? '*' : '*' + search + '*').then((data) => {
console.log('listGroups data', data)
resolve(this.processGroupList(data)); resolve(this.processGroupList(data));
}).catch((e) => { }).catch((e) => {
console.error(" * WTVNews Error:", "Command: listGroups (all)", e); console.error(" * WTVNews Error:", "Command: listGroups (search)", search, e);
reject(e); reject(e);
}); });
} }

117
zefie_wtvp_minisrv/includes/WTVNewsServer.js
View File

@@ -172,12 +172,10 @@ class WTVNewsServer {
createMetaFile(group, description = null) { createMetaFile(group, description = null) {
const g = this.getMetaFilename(group); const g = this.getMetaFilename(group);
if (this.fs.existsSync(g)) return false; if (this.fs.existsSync(g)) return false;
var metadata = {}; var metadata = this.selectGroup(group, true, true);
metadata.group = group;
metadata.last_article_id = this.selectGroup(group).max_index;
if (description) metadata.description = description; if (description) metadata.description = description;
this.saveMetadata(group, metadata, true); this.saveMetadata(group, metadata, true);
return metadata; return (!metadata.failed) ? metadata : false
} }
saveMetadata(group, metadata, creating = false) { saveMetadata(group, metadata, creating = false) {
@@ -196,12 +194,6 @@ class WTVNewsServer {
} else return false; } else return false;
} }
incrementArticleIdMeta(group) {
var metadata = this.getMetadata(group);
metadata.last_article_id = metadata.last_article_id + 1;
this.saveMetadata(group, metadata)
}
findHeaderCaseInsensitive(headers, header) { findHeaderCaseInsensitive(headers, header) {
// returns the key with the found case // returns the key with the found case
var response = null; var response = null;
@@ -217,7 +209,7 @@ class WTVNewsServer {
} }
postArticle(group, post_data) { postArticle(group, post_data) {
var articleNumber = this.getMetadata(group).last_article_id + 1; var articleNumber = this.getMetadata(group).max_index + 1;
if (!articleNumber) return false; if (!articleNumber) return false;
try { try {
post_data.articleNumber = articleNumber; post_data.articleNumber = articleNumber;
@@ -258,7 +250,10 @@ class WTVNewsServer {
var file = g + this.path.sep + articleNumber + ".newz"; var file = g + this.path.sep + articleNumber + ".newz";
try { try {
this.fs.writeFileSync(file, JSON.stringify(article)); this.fs.writeFileSync(file, JSON.stringify(article));
this.incrementArticleIdMeta(group); var metadata = this.getMetadata(group);
metadata.max_index = metadata.max_index + 1;
metadata.total = metadata.total + 1;
this.saveMetadata(group, metadata)
return true; return true;
} catch (e) { } catch (e) {
console.error(" * WTVNewsServer Error: createArticle: ", e); console.error(" * WTVNewsServer Error: createArticle: ", e);
@@ -284,8 +279,7 @@ class WTVNewsServer {
var g = this.getGroupPath(group); var g = this.getGroupPath(group);
if (!this.fs.existsSync(g)) { if (!this.fs.existsSync(g)) {
this.fs.mkdirSync(g); this.fs.mkdirSync(g);
this.createMetaFile(group, description) return this.createMetaFile(group, description)
return this.fs.existsSync(g);
} }
return false return false
} }
@@ -307,14 +301,19 @@ class WTVNewsServer {
} }
selectGroup(group) { selectGroup(group, force_update = false, initial_update = false) {
var g = this.getGroupPath(group); var g = this.getGroupPath(group);
var out = { if (initial_update) {
total: 0, var out = {
min_index: null, total: 0,
max_index: 0, min_index: 0,
name: group max_index: 0,
} name: group
}
} else {
var meta = this.getMetadata(group);
var out = { ...meta }
}
if (this.featuredGroups) { if (this.featuredGroups) {
Object.keys(this.featuredGroups).forEach((k) => { Object.keys(this.featuredGroups).forEach((k) => {
if (group == this.featuredGroups[k].group) { if (group == this.featuredGroups[k].group) {
@@ -324,19 +323,28 @@ class WTVNewsServer {
}) })
} }
try { try {
this.fs.readdirSync(g).forEach(file => { if (force_update || this.doesMetaNeedRefreshing(meta)) {
if (file == "meta.json") return; this.fs.readdirSync(g).forEach(file => {
var articleNumber = parseInt(file.split('.')[0]); if (file == "meta.json") return;
if (out.min_index == null) out.min_index = articleNumber; var articleNumber = parseInt(file.split('.')[0]);
else if (articleNumber < out.min_index) out.min_index = articleNumber; if (out.min_index == null) out.min_index = articleNumber;
else if (articleNumber < out.min_index) out.min_index = articleNumber;
if (articleNumber > out.max_index) out.max_index = articleNumber; if (articleNumber > out.max_index) out.max_index = articleNumber;
out.total++; out.total++;
}); });
if (initial_update) {
out.last_scan = Math.floor(Date.now() / 1000);
} else {
meta = { ...meta, ...out }
if (meta.wildmat) delete meta.wildmat;
meta.last_scan = Math.floor(Date.now() / 1000);
this.saveMetadata(group, meta);
}
}
} catch (e) { } catch (e) {
out.failed = e; out.failed = e;
} }
if (out.min_index === null) out.min_index = 0;
return out; return out;
} }
@@ -403,28 +411,49 @@ class WTVNewsServer {
return res; return res;
} }
listGroup(group, start, end) { doesMetaNeedRefreshing(meta) {
if (!meta) return true;
if (!meta.max_index) return true;
if (!meta.min_index) return true;
if (!meta.total) return true;
if (!meta.last_scan) return true;
if (meta.last_scan) {
if ((Math.floor(Date.now() / 1000) - this.scan_interval) > meta.last_scan) {
return true;
}
}
return false;
}
listGroup(group, start, end, force_update = false) {
var g = this.getGroupPath(group); var g = this.getGroupPath(group);
var out = { var out = {
total: 0, total: 0,
min_index: null, min_index: 0,
max_index: 0, max_index: 0,
name: group name: group
} }
var articles = []; var articles = [];
try { try {
this.fs.readdirSync(g).forEach(file => { var meta = this.getMetadata(group);
if (file == "meta.json") return; if (force_update || this.doesMetaNeedRefreshing(meta)) {
var articleNumber = parseInt(file.split('.')[0]); this.fs.readdirSync(g).forEach(file => {
if (articleNumber < start) return; if (file == "meta.json") return;
if (articleNumber > end) return false; var articleNumber = parseInt(file.split('.')[0]);
if (out.min_index == null) out.min_index = articleNumber; if (articleNumber < start) return;
else if (articleNumber < out.min_index) out.min_index = articleNumber; if (articleNumber > end) return false;
if (out.min_index == null) out.min_index = articleNumber;
else if (articleNumber < out.min_index) out.min_index = articleNumber;
if (articleNumber > out.max_index) out.max_index = articleNumber; if (articleNumber > out.max_index) out.max_index = articleNumber;
articles.push(this.getArticle(group, articleNumber)); articles.push(this.getArticle(group, articleNumber));
out.total++; out.total++;
}); });
meta = { ...meta, ...out }
meta.last_scan = Math.floor(Date.now() / 1000);
this.saveMetadata(group, meta);
}
} catch (e) { } catch (e) {
console.error(" * WTVNewsServer Error: listGroup: ", e); console.error(" * WTVNewsServer Error: listGroup: ", e);
out.failed = e; out.failed = e;
@@ -433,7 +462,7 @@ class WTVNewsServer {
if (out.min_index === null) out.min_index = 0; if (out.min_index === null) out.min_index = 0;
return { return {
articles: articles, articles: articles,
group_data: out group_data: meta
} }
} }
} }