zefie/minisrv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more wtv-disk work

Browse Source
This commit is contained in:
zefie
2025-08-08 11:43:39 -04:00
parent afdade7d9c
commit b3fe082028
2 changed files with 67 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
zefie_wtvp_minisrv/includes/ServiceVault/wtv-disk/sync.js
View File

@@ -32,7 +32,7 @@ if (request_headers['wtv-request-type'] == 'download') {
// delete partition/path if force, and not no_delete // delete partition/path if force, and not no_delete
if (force_update && !no_delete) { if (force_update && !no_delete) {
// don't delete Brower partition, ever, but allow deleting of Browser partition subdirs // don't delete Browser partition, ever, but allow deleting of Browser partition subdirs
if (!diskmap_group_data.base.match(/disk\/browser(\/)?$/i)) { if (!diskmap_group_data.base.match(/disk\/browser(\/)?$/i)) {
if (diskmap_group_data.client_group_data.path.toLowerCase() == diskmap_group_data.base.toLowerCase()) { if (diskmap_group_data.client_group_data.path.toLowerCase() == diskmap_group_data.base.toLowerCase()) {
wtvdl.delete(diskmap_group_data.base, null); wtvdl.delete(diskmap_group_data.base, null);
@@ -82,7 +82,6 @@ if (request_headers['wtv-request-type'] == 'download') {
var get_url = service_name + ":/" + update_list[k].location + "?"; var get_url = service_name + ":/" + update_list[k].location + "?";
if (update_list[k].compress === false) get_url += "dont_compress=true&"; if (update_list[k].compress === false) get_url += "dont_compress=true&";
if (update_list[k].type) get_url += "content_type=" + escape(update_list[k].type) + "&"; if (update_list[k].type) get_url += "content_type=" + escape(update_list[k].type) + "&";
get_url = get_url.slice();
wtvdl.get(update_list[k].file.replace(diskmap_group_data.base, ""), update_list[k].file, get_url, diskmap_group_name, update_list[k].checksum, update_list[k].uncompressed_size || null, update_list[k].original_filename) wtvdl.get(update_list[k].file.replace(diskmap_group_data.base, ""), update_list[k].file, get_url, diskmap_group_name, update_list[k].checksum, update_list[k].uncompressed_size || null, update_list[k].original_filename)
break; break;
} }
@@ -121,8 +120,8 @@ if (request_headers['wtv-request-type'] == 'download') {
function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgroup = null, version = 0) { function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgroup = null, version = 0) {
// parse webtv post // parse webtv post
var output_data = ''; var output_data = '';
var post_data = new Array(); var post_data = [];
var client_group_data = new Array(); var client_group_data = [];
if (request_headers.post_data) { if (request_headers.post_data) {
post_data = request_headers.post_data.toString(CryptoJS.enc.Latin1).split("\n"); post_data = request_headers.post_data.toString(CryptoJS.enc.Latin1).split("\n");
client_group_data = wtvdl.getGroupDataFromClientPost(request_headers.post_data.toString(CryptoJS.enc.Latin1)); client_group_data = wtvdl.getGroupDataFromClientPost(request_headers.post_data.toString(CryptoJS.enc.Latin1));
@@ -133,7 +132,7 @@ if (request_headers['wtv-request-type'] == 'download') {
var post_data_last_modified = false; var post_data_last_modified = false;
var post_data_content_length = false; var post_data_content_length = false;
var post_data_current_group_state = false; var post_data_current_group_state = false;
var post_data_fileinfo = new Array(); var post_data_fileinfo = [];
var entry_type = false; var entry_type = false;
var post_data_current_version = false; var post_data_current_version = false;
var post_data_current_checksum = false; var post_data_current_checksum = false;
@@ -159,7 +158,7 @@ if (request_headers['wtv-request-type'] == 'download') {
switch (post_data_line_name.toLowerCase()) { switch (post_data_line_name.toLowerCase()) {
case "last-modified": case "last-modified":
post_data_last_modified = (new Date(new Date(Date.parse(post_data_line_data)).toUTCString()) / 1000); post_data_last_modified = Math.floor(Date.parse(post_data_line_data) / 1000);
break; break;
case "content-length": case "content-length":
post_data_content_length = parseInt(post_data_line_data); post_data_content_length = parseInt(post_data_line_data);
@@ -177,7 +176,7 @@ if (request_headers['wtv-request-type'] == 'download') {
post_data_current_checksum = post_data_line_data; post_data_current_checksum = post_data_line_data;
break; break;
case "last-checkup-time": case "last-checkup-time":
post_data_last_checkup_time = (new Date(new Date(Date.parse(post_data_line_data)).toUTCString()) / 1000); post_data_last_checkup_time = Math.floor(Date.parse(post_data_line_data) / 1000);
break; break;
} }
} else { } else {
@@ -190,7 +189,7 @@ if (request_headers['wtv-request-type'] == 'download') {
var post_data_current_path = ((entry_type == "file") ? (post_data_current_directory + post_data_current_file) : post_data_current_directory); var post_data_current_path = ((entry_type == "file") ? (post_data_current_directory + post_data_current_file) : post_data_current_directory);
var index = post_data_current_path.replace(/[\:\/]/g, "_").toLowerCase() + "_" + post_data_current_group; var index = post_data_current_path.replace(/[\:\/]/g, "_").toLowerCase() + "_" + post_data_current_group;
if (index.match(/\/$/)) entry_type = "folder"; if (index.match(/\/$/)) entry_type = "folder";
if (!post_data_fileinfo[index]) post_data_fileinfo[index] = new Array(); if (!post_data_fileinfo[index]) post_data_fileinfo[index] = {};
post_data_fileinfo[index].entry_type = entry_type; post_data_fileinfo[index].entry_type = entry_type;
post_data_fileinfo[index].file = post_data_current_path; post_data_fileinfo[index].file = post_data_current_path;
post_data_fileinfo[index].group = post_data_current_group; post_data_fileinfo[index].group = post_data_current_group;
@@ -205,7 +204,7 @@ if (request_headers['wtv-request-type'] == 'download') {
} }
} }
}); });
var wtv_download_list = new Array(); var wtv_download_list = [];
var newest_file_epoch = version; var newest_file_epoch = version;
Object.keys(diskmap_group_data.files).forEach(function (k) { Object.keys(diskmap_group_data.files).forEach(function (k) {
if (!diskmap_group_data.files[k].location) diskmap_group_data.files[k].location = wtvshared.makeSafePath(diskmap_group_data.location,diskmap_group_data.files[k].file.replace(diskmap_group_data.base, ""), true); if (!diskmap_group_data.files[k].location) diskmap_group_data.files[k].location = wtvshared.makeSafePath(diskmap_group_data.location,diskmap_group_data.files[k].file.replace(diskmap_group_data.base, ""), true);
@@ -218,12 +217,12 @@ if (request_headers['wtv-request-type'] == 'download') {
if (diskmap_data_file) { if (diskmap_data_file) {
var diskmap_file_stat = fs.lstatSync(diskmap_data_file); var diskmap_file_stat = fs.lstatSync(diskmap_data_file);
var diskmap_file_data = new Buffer.from(fs.readFileSync(diskmap_data_file, { var diskmap_file_data = Buffer.from(fs.readFileSync(diskmap_data_file, {
encoding: null, encoding: null,
flags: 'r' flags: 'r'
})); }));
diskmap_group_data.files[k].base = diskmap_group_data.base; diskmap_group_data.files[k].base = diskmap_group_data.base;
diskmap_group_data.files[k].last_modified = (new Date(new Date(diskmap_file_stat.mtime).toUTCString()) / 1000); diskmap_group_data.files[k].last_modified = Math.floor(diskmap_file_stat.mtime.getTime() / 1000);
diskmap_group_data.files[k].content_length = diskmap_file_stat.size; diskmap_group_data.files[k].content_length = diskmap_file_stat.size;
diskmap_group_data.files[k].action = (diskmap_group_data.files[k].action) ? diskmap_group_data.files[k].action.toUpperCase() : "GET"; diskmap_group_data.files[k].action = (diskmap_group_data.files[k].action) ? diskmap_group_data.files[k].action.toUpperCase() : "GET";
@@ -245,6 +244,8 @@ if (request_headers['wtv-request-type'] == 'download') {
diskmap_group_data.files[k].invalid = true; diskmap_group_data.files[k].invalid = true;
wtv_download_list.push(diskmap_group_data.files[k]); wtv_download_list.push(diskmap_group_data.files[k]);
// Clean up file data from memory
diskmap_file_data = null;
} }
}); });
// check to see if client says they have this version // check to see if client says they have this version
@@ -252,9 +253,9 @@ if (request_headers['wtv-request-type'] == 'download') {
Object.keys(wtv_download_list).forEach(function (k) { Object.keys(wtv_download_list).forEach(function (k) {
wtv_download_list[k].version = newest_file_epoch; wtv_download_list[k].version = newest_file_epoch;
Object.keys(post_data_fileinfo).forEach(function (g) { Object.keys(post_data_fileinfo).forEach(function (g) {
if (post_data_fileinfo[g].file == wtv_download_list[k] || post_data_fileinfo[g].file == wtv_download_list[k].base) { if (post_data_fileinfo[g].file == wtv_download_list[k].file || post_data_fileinfo[g].file == wtv_download_list[k].base) {
diskmap_group_data.group_exists = true; diskmap_group_data.group_exists = true;
if (diskmap_group_data.files[k].checksum && diskmap_group_data.files[k].checksum.toLowerCase() == post_data_fileinfo[g].checksum) wtv_download_list[k].invalid = false; if (wtv_download_list[k].checksum && wtv_download_list[k].checksum.toLowerCase() == post_data_fileinfo[g].checksum) wtv_download_list[k].invalid = false;
else if (post_data_fileinfo[g].version == wtv_download_list[k].version && post_data_fileinfo[g].state != "invalid") wtv_download_list[k].invalid = false; else if (post_data_fileinfo[g].version == wtv_download_list[k].version && post_data_fileinfo[g].state != "invalid") wtv_download_list[k].invalid = false;
} }
}); });
@@ -274,11 +275,10 @@ if (request_headers['wtv-request-type'] == 'download') {
}); });
if (diskmap_json_file != null) { if (diskmap_json_file != null) {
if (fs.existsSync(diskmap_json_file)) { try {
try { // read diskmap
// read diskmap var json_stats = fs.lstatSync(diskmap_json_file);
var json_stats = fs.lstatSync(diskmap_json_file); var diskmap_data = JSON.parse(fs.readFileSync(diskmap_json_file).toString());
var diskmap_data = JSON.parse(fs.readFileSync(diskmap_json_file).toString());
if (!diskmap_data[request_headers.query.group]) { if (!diskmap_data[request_headers.query.group]) {
throw ("Invalid diskmap data (group does not match)"); throw ("Invalid diskmap data (group does not match)");
} }
@@ -287,12 +287,12 @@ if (request_headers['wtv-request-type'] == 'download') {
if (!diskmap_data.location) { if (!diskmap_data.location) {
Object.keys(diskmap_data).forEach(function (k) { Object.keys(diskmap_data).forEach(function (k) {
if (diskmap_data[k]) { if (diskmap_data[k]) {
diskmap_data[k].version = (new Date(new Date(json_stats.mtime).toUTCString()) / 1000); diskmap_data[k].version = Math.floor(json_stats.mtime.getTime() / 1000);
data += processGroup(request_headers.query.group, diskmap_data[k], k, diskmap_data.version); data += processGroup(request_headers.query.group, diskmap_data[k], k, diskmap_data.version);
} }
}); });
} else { } else {
diskmap_data.version = (new Date(new Date(json_stats.mtime).toUTCString()) / 1000); diskmap_data.version = Math.floor(json_stats.mtime.getTime() / 1000);
data = processGroup(request_headers.query.group, diskmap_data, null, diskmap_data.version); data = processGroup(request_headers.query.group, diskmap_data, null, diskmap_data.version);
} }
@@ -303,7 +303,6 @@ if (request_headers['wtv-request-type'] == 'download') {
data = errpage[1]; data = errpage[1];
console.error(" # " + service_name+":/sync error", e); console.error(" # " + service_name+":/sync error", e);
} }
}
} else { } else {
var errpage = wtvshared.doErrorPage(404, "The requested DiskMap does not exist."); var errpage = wtvshared.doErrorPage(404, "The requested DiskMap does not exist.");
headers = errpage[0]; headers = errpage[0];

96
zefie_wtvp_minisrv/includes/classes/WTVDisk.js
View File

@@ -2,6 +2,9 @@
* wtv/download-list creation helper class * wtv/download-list creation helper class
* By: zefie * By: zefie
*/ */
const { WTVShared, clientShowAlert } = require("./WTVShared.js");
class WTVDownloadList { class WTVDownloadList {
download_list = ""; download_list = "";
@@ -16,11 +19,10 @@ class WTVDownloadList {
* @param {string} service_name Service name to use in wtv-urls * @param {string} service_name Service name to use in wtv-urls
*/ */
constructor(minisrv_config, service_name = "wtv-disk") { constructor(minisrv_config, service_name = "wtv-disk") {
var { WTVShared, clientShowAlert } = require("./WTVShared.js");
this.minisrv_config = minisrv_config; this.minisrv_config = minisrv_config;
this.wtvshared = new WTVShared(minisrv_config); this.wtvshared = new WTVShared(minisrv_config);
this.clientShowAlert = clientShowAlert; this.clientShowAlert = clientShowAlert;
this.service_name = service_name this.service_name = service_name;
this.clear(); this.clear();
} }
@@ -51,7 +53,7 @@ class WTVDownloadList {
* @param {string} message Message to display to the client * @param {string} message Message to display to the client
*/ */
display(message) { display(message) {
this.download_list += "DISPLAY " + message + "\n\n"; this.download_list += `DISPLAY ${message}\n\n`;
} }
/** /**
@@ -59,7 +61,7 @@ class WTVDownloadList {
* @param {string} command client command to execute * @param {string} command client command to execute
*/ */
execute(command) { execute(command) {
this.download_list += "EXECUTE " + command + "\n\n"; this.download_list += `EXECUTE ${command}\n\n`;
} }
/** /**
@@ -68,8 +70,7 @@ class WTVDownloadList {
* @param {string} size Size of the desired partition * @param {string} size Size of the desired partition
*/ */
createPartition(path, size) { createPartition(path, size) {
this.download_list += "CREATE " + path + "\n"; this.download_list += `CREATE ${path}\npartition-size: ${size}\n\n`;
this.download_list += "partition-size: " + size + "\n\n";
} }
/** /**
@@ -80,10 +81,9 @@ class WTVDownloadList {
* @param {boolean|null} service_owned Sets service owned flag. (null = don't set) * @param {boolean|null} service_owned Sets service owned flag. (null = don't set)
*/ */
createGroup(name, path, state = 'invalid', service_owned = null) { createGroup(name, path, state = 'invalid', service_owned = null) {
this.download_list += "CREATE-GROUP " + name + "\n"; this.download_list += `CREATE-GROUP ${name}\nstate: ${state}\n`;
this.download_list += "state: " + state + "\n"; if (service_owned !== null) this.download_list += `service-owned: ${service_owned}\n`;
if (service_owned !== null) this.download_list += "service-owned: " + service_owned + "\n"; this.download_list += `base: ${path}\n\n`;
this.download_list += "base: " + path + "\n\n";
} }
/** /**
@@ -106,9 +106,9 @@ class WTVDownloadList {
*/ */
delete(path, group = null, original_filename = null) { delete(path, group = null, original_filename = null) {
path = this.checkOriginalName(path, original_filename); path = this.checkOriginalName(path, original_filename);
this.download_list += "DELETE " + path + "\n"; this.download_list += `DELETE ${path}\n`;
if (group) this.download_list += "group: " + group + "\n\n"; if (group) this.download_list += `group: ${group}\n\n`;
else (this.download_list) += "\n"; else this.download_list += "\n";
} }
/** /**
@@ -117,8 +117,7 @@ class WTVDownloadList {
* @param {string} destination Destination address (wtv url on service) in which to POST upload the file to * @param {string} destination Destination address (wtv url on service) in which to POST upload the file to
*/ */
put(path, destination) { put(path, destination) {
this.download_list += "PUT " + path + "\n"; this.download_list += `PUT ${path}\nlocation: ${destination}\n\n`;
this.download_list += "location: " + destination + "\n\n";
} }
/** /**
@@ -127,15 +126,15 @@ class WTVDownloadList {
* @param {string} destination Destination file path in the User Store * @param {string} destination Destination file path in the User Store
*/ */
putUserStoreDest(path, destination) { putUserStoreDest(path, destination) {
this.put(path, this.service_name + ":/userstore?partialPath=" + escape(destination)); this.put(path, `${this.service_name}:/userstore?partialPath=${encodeURIComponent(destination)}`);
} }
/** /**
* Alias to putUserStoreDest() that generates the destination * Alias to putUserStoreDest() that generates the destination
* @param {any} path * @param {string} path
*/ */
putUserStore(path) { putUserStore(path) {
var destination = path.replace("file://", ""); const destination = path.replace("file://", "");
this.putUserStoreDest(path, destination); this.putUserStoreDest(path, destination);
} }
/** /**
@@ -152,22 +151,22 @@ class WTVDownloadList {
get(file, path, source, group, checksum = null, uncompressed_size = null, original_filename = null, file_permission = 'r') { get(file, path, source, group, checksum = null, uncompressed_size = null, original_filename = null, file_permission = 'r') {
if (original_filename) { if (original_filename) {
file = file.split('/'); file = file.split('/');
var file_name = file[file.length - 1]; const file_name = file[file.length - 1];
path = path.replace(file_name, original_filename); path = path.replace(file_name, original_filename);
file.pop(); file.pop();
if (file.length > 0) file = file.join('/') + '/' + original_filename; if (file.length > 0) file = file.join('/') + '/' + original_filename;
else file = original_filename; else file = original_filename;
} }
this.download_list += "GET " + file + "\n"; this.download_list += `GET ${file}\n`;
source = source.replace(/\\/g, "/"); source = source.replace(/\\/g, "/");
this.download_list += "group: " + group + "-UPDATE\n"; this.download_list += `group: ${group}-UPDATE\n`;
this.download_list += "location: " + source + "\n"; this.download_list += `location: ${source}\n`;
this.download_list += "file-permission: " + file_permission + "\n"; this.download_list += `file-permission: ${file_permission}\n`;
if (checksum != null) this.download_list += "wtv-checksum: " + checksum + "\n"; if (checksum != null) this.download_list += `wtv-checksum: ${checksum}\n`;
if (uncompressed_size != null) this.download_list += "wtv-uncompressed-filesize: " + uncompressed_size + "\n"; if (uncompressed_size != null) this.download_list += `wtv-uncompressed-filesize: ${uncompressed_size}\n`;
this.download_list += "service-source-location: /webtv/content/" + source.substr(source.indexOf('-') + 1, source.indexOf(':/') - source.indexOf('-') - 1) + "d/" + source.substr(source.indexOf(':/') + 2) + "\n"; this.download_list += `service-source-location: /webtv/content/${source.substring(source.indexOf('-') + 1, source.indexOf(':/'))}d/${source.substring(source.indexOf(':/') + 2)}\n`;
this.download_list += "client-dest-location: " + path + "\n\n"; this.download_list += `client-dest-location: ${path}\n\n`;
} }
/** /**
@@ -178,24 +177,23 @@ class WTVDownloadList {
*/ */
checkOriginalName(path, original_name) { checkOriginalName(path, original_name) {
if (original_name) { if (original_name) {
var tmp = this.wtvshared.getFilePath(path); const tmp = this.wtvshared.getFilePath(path);
if (tmp.length > 0) return tmp + "/" + original_name; if (tmp.length > 0) return `${tmp}/${original_name}`;
return original_name return original_name;
} else return path; } else return path;
} }
getGroupDataFromClientPost(post_data) { getGroupDataFromClientPost(post_data) {
if (typeof post_data == 'string') post_data = post_data.split("\n\n"); if (typeof post_data == 'string') post_data = post_data.split("\n\n");
var group_data = []; const group_data = [];
var i = 0;
post_data.forEach(function (v) { post_data.forEach(function (v) {
if (v.substr(0, 4) == "file") { if (v.substring(0, 4) == "file") {
var block_split = v.split("\n"); const block_split = v.split("\n");
var group_data_entry = {}; const group_data_entry = {};
group_data_entry.path = block_split[0]; group_data_entry.path = block_split[0];
block_split.forEach(function (block_section) { block_split.forEach(function (block_section) {
if (block_section.indexOf(": ") > 0) { if (block_section.indexOf(": ") > 0) {
var block_section_split = block_section.split(": "); const block_section_split = block_section.split(": ");
group_data_entry[block_section_split[0]] = block_section_split[1]; group_data_entry[block_section_split[0]] = block_section_split[1];
} }
}); });
@@ -216,12 +214,12 @@ class WTVDownloadList {
rename(srcfile, destfile, srcgroup, destgroup, original_filename = null) { rename(srcfile, destfile, srcgroup, destgroup, original_filename = null) {
if (original_filename) { if (original_filename) {
srcfile = this.checkOriginalName(srcfile, original_filename); srcfile = this.checkOriginalName(srcfile, original_filename);
destfile = this.checkOriginalName(srcfile, original_filename); destfile = this.checkOriginalName(destfile, original_filename);
} }
this.download_list += "RENAME " + srcfile + "\n"; this.download_list += `RENAME ${srcfile}\n`;
this.download_list += "group: " + srcgroup + "-UPDATE\n"; this.download_list += `group: ${srcgroup}-UPDATE\n`;
this.download_list += "destination-group: " + destgroup + "\n"; this.download_list += `destination-group: ${destgroup}\n`;
this.download_list += "location: " + destfile + "\n\n"; this.download_list += `location: ${destfile}\n\n`;
} }
/** /**
@@ -231,10 +229,10 @@ class WTVDownloadList {
* @param {string} version Version to set group to * @param {string} version Version to set group to
*/ */
setGroup(group, state, version) { setGroup(group, state, version) {
this.download_list += "SET-GROUP " + group + "\n"; this.download_list += `SET-GROUP ${group}\n`;
this.download_list += "state: " + state + "\n"; this.download_list += `state: ${state}\n`;
this.download_list += "version: " + version + "\n"; this.download_list += `version: ${version}\n`;
this.download_list += "last-checkup-time: " + new Date().toUTCString().replace("GMT", "+0000") + "\n\n"; this.download_list += `last-checkup-time: ${new Date().toUTCString().replace("GMT", "+0000")}\n\n`;
} }
/** /**
@@ -242,7 +240,7 @@ class WTVDownloadList {
* @param {string} group Group to delete * @param {string} group Group to delete
*/ */
deleteGroup(group) { deleteGroup(group) {
this.download_list += "DELETE-GROUP " + group + "\n\n"; this.download_list += `DELETE-GROUP ${group}\n\n`;
} }
/** /**
@@ -251,8 +249,8 @@ class WTVDownloadList {
* @param {string} path Group base path * @param {string} path Group base path
*/ */
deleteGroupUpdate(group, path) { deleteGroupUpdate(group, path) {
this.deleteGroup(group + "-UPDATE"); this.deleteGroup(`${group}-UPDATE`);
this.delete(path + ".GROUP-UPDATE/"); this.delete(`${path}.GROUP-UPDATE/`);
} }
} }