diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-disk/delete-group.js b/zefie_wtvp_minisrv/ServiceVault/wtv-disk/delete-group.js
index 3421d09d..cc568a9c 100644
--- a/zefie_wtvp_minisrv/ServiceVault/wtv-disk/delete-group.js
+++ b/zefie_wtvp_minisrv/ServiceVault/wtv-disk/delete-group.js
@@ -4,12 +4,13 @@ if (request_headers.query.group) {
if (request_headers['wtv-request-type']) {
headers = "200 OK\nContent-Type: " + wtvdl.content_type;
wtvdl.deleteGroup(request_headers.query.group);
+ wtvdl.deleteGroupUpdate(request_headers.query.group);
data = wtvdl.getDownloadList();
}
var title = "Deleting group"
var message = title + " " + request_headers.query.group;
headers = "200 OK\nContent-Type: text/html"
- data = wtvdl.getSyncPage(title, request_headers.query.group, "delete", message, message, null, "client:goback", "client:goback", "wtv-disk:/delete-group");
+ data = wtvdl.getSyncPage(title, request_headers.query.group, "delete", message, message, null, null, "client:goback", "client:goback", "wtv-disk:/delete-group");
} else {
headers = "200 OK\nContent-Type: text/html"
data = `
diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-disk/sync.js b/zefie_wtvp_minisrv/ServiceVault/wtv-disk/sync.js
index 6f37edd8..2a138e8c 100644
--- a/zefie_wtvp_minisrv/ServiceVault/wtv-disk/sync.js
+++ b/zefie_wtvp_minisrv/ServiceVault/wtv-disk/sync.js
@@ -2,11 +2,12 @@ const WTVDownloadList = require("./WTVDownloadList.js");
var wtvdl = new WTVDownloadList(minisrv_config, service_name);
var force_update = (request_headers.query.force == "true") ? true : false;
+var no_delete = (request_headers.query.dont_delete_files == "true") ? true : false;
if (request_headers['wtv-request-type'] == 'download') {
var content_dir = "content/"
var diskmap_dir = content_dir + "diskmaps/";
- function generateDownloadList(diskmap_group_data, update_list, diskmap_data) {
+ function generateDownloadList(diskmap_group_name, update_list, diskmap_group_data) {
wtvdl.reset();
var files_to_send = 0;
Object.keys(update_list).forEach(function (k) {
@@ -16,77 +17,98 @@ if (request_headers['wtv-request-type'] == 'download') {
});
// create WebTV Download List
- if (diskmap_data.execute && diskmap_data.execute_when) {
- if (diskmap_data.execute_when.toLowerCase().match(/start/)) {
- wtvdl.execute(diskmap_data.execute);
+ if (diskmap_group_data.execute && diskmap_group_data.execute_when) {
+ if (diskmap_group_data.execute_when.toLowerCase().match(/start/)) {
+ wtvdl.execute(diskmap_group_data.execute);
}
}
- if (diskmap_group_data.display) wtvdl.display(diskmap_group_data.display);
+ console.log(diskmap_group_data.client_group_data);
+
+ // delete groups if force, or group is invalid
+ if (diskmap_group_data.client_group_data) {
+ if (force_update || diskmap_group_data.client_group_data.state == "invalid") {
+ wtvdl.deleteGroupUpdate(diskmap_group_data.client_group_data.group, diskmap_group_data.client_group_data.path);
+ wtvdl.deleteGroup(diskmap_group_data.client_group_data.group, diskmap_group_data.client_group_data.path);
+ }
+
+ // delete partition/path if force, and not no_delete
+ if (force_update && !no_delete) {
+ // don't delete Brower partition, ever, but allow deleting of Browser partition subdirs
+ if (!diskmap_group_data.base.match(/disk\/browser(\/)?$/i)) {
+ if (diskmap_group_data.client_group_data.path.toLowerCase() == diskmap_group_data.base.toLowerCase()) {
+ wtvdl.delete(diskmap_group_data.base, null);
+ }
+ }
+ }
+ }
+
+ if (diskmap_group_name.display) wtvdl.display(diskmap_group_name.display);
if (files_to_send > 0) {
- if (diskmap_data.partition_size) {
- wtvdl.createPartition(diskmap_data.base, diskmap_data.partition_size);
+ if (diskmap_group_data.partition_size) {
+ wtvdl.createPartition(diskmap_group_data.base, diskmap_group_data.partition_size);
}
- if (!diskmap_data.nogroup) {
+ if (!diskmap_group_data.nogroup) {
// only send group commands if group mode is enable
// useful to disable for PUT
- wtvdl.createUpdateGroup(diskmap_group_data, diskmap_data.base, "invalid", (diskmap_data.service_owned || false));
+ wtvdl.createUpdateGroup(diskmap_group_name, diskmap_group_data.base, "invalid", (diskmap_group_data.service_owned || false));
}
Object.keys(update_list).forEach(function (k) {
// file { "action": "delete" }
// Useful to purge files we no longer want on the client
- if (update_list[k].action != "DELETE") {
+ if (update_list[k].action != "DELETE" && update_list[k].action != "DELETEONLY") {
// skip deleting valid files if we aren't specifically requesting their deletion
if (update_list[k].checksum_match && !force_update) return;
if (!update_list[k].invalid && !force_update) return;
}
- wtvdl.delete(update_list[k].file.replace(diskmap_data.base, ""), diskmap_group_data);
+ wtvdl.delete(update_list[k].file.replace(diskmap_group_data.base, ""), diskmap_group_name);
});
Object.keys(update_list).forEach(function (k) {
if (update_list[k].checksum_match && !force_update) return;
if (!update_list[k].invalid && !force_update) return;
+ if (update_list[k].action == "DELETEONLY") return;
if (update_list[k].display) wtvdl.display(update_list[k].display);
switch (update_list[k].action) {
case "PUT":
- wtvdl.put(update_list[k].file.replace(diskmap_data.base, ""), service_name + ":/" + update_list[k].location, update_list[k].display);
+ wtvdl.put(update_list[k].file.replace(diskmap_group_data.base, ""), service_name + ":/" + update_list[k].location, update_list[k].display);
break;
case "GET":
- wtvdl.get(update_list[k].file.replace(diskmap_data.base, ""), update_list[k].file, service_name + ":/" + update_list[k].location, diskmap_group_data, update_list[k].checksum, update_list[k].uncompressed_size || null, update_list[k].original_filename)
+ wtvdl.get(update_list[k].file.replace(diskmap_group_data.base, ""), update_list[k].file, service_name + ":/" + update_list[k].location, diskmap_group_name, update_list[k].checksum, update_list[k].uncompressed_size || null, update_list[k].original_filename)
break;
}
});
- if (!diskmap_data.nogroup) {
- wtvdl.createGroup(diskmap_group_data, diskmap_data.base, "invalid", (diskmap_data.service_owned || false));
-
+ if (!diskmap_group_data.nogroup) {
+ wtvdl.createGroup(diskmap_group_name, diskmap_group_data.base, "invalid", (diskmap_group_data.service_owned || false));
// this rename loop is a part of the group system
Object.keys(update_list).forEach(function (k) {
if (update_list[k].checksum_match && !force_update) return;
if (!update_list[k].invalid && !force_update) return;
- wtvdl.rename(update_list[k].file.replace(diskmap_data.base, ""), update_list[k].file.replace(diskmap_data.base, ""), diskmap_group_data, diskmap_group_data);
+ if (update_list[k].action == "DELETEONLY") return;
+ wtvdl.rename(update_list[k].file.replace(diskmap_group_data.base, ""), update_list[k].file.replace(diskmap_group_data.base, ""), diskmap_group_name, diskmap_group_name, update_list[k].original_filename);
});
- wtvdl.setGroup(diskmap_group_data, 'ok', diskmap_data.version);
+ wtvdl.setGroup(diskmap_group_name, 'ok', diskmap_group_data.version);
}
}
- if (diskmap_data.execute && diskmap_data.execute_when) {
- if (diskmap_data.execute_when.toLowerCase().match(/end/)) {
- wtvdl.execute(diskmap_data.execute);
+ if (diskmap_group_data.execute && diskmap_group_data.execute_when) {
+ if (diskmap_group_data.execute_when.toLowerCase().match(/end/)) {
+ wtvdl.execute(diskmap_group_data.execute);
}
}
if (files_to_send > 0) {
- if (!diskmap_data.nogroup) {
- wtvdl.deleteGroupUpdate(diskmap_group_data, diskmap_data.base);
+ if (!diskmap_group_data.nogroup) {
+ wtvdl.deleteGroupUpdate(diskmap_group_name, diskmap_group_data.base);
}
}
var download_list = wtvdl.getDownloadList();
@@ -94,11 +116,15 @@ if (request_headers['wtv-request-type'] == 'download') {
return download_list;
}
- function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgroup = null) {
+ function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgroup = null, version = 0) {
// parse webtv post
var output_data = '';
var post_data = new Array();
- if (request_headers.post_data) post_data = request_headers.post_data.toString(CryptoJS.enc.Latin1).split("\n");
+ var client_group_data = new Array();
+ if (request_headers.post_data) {
+ post_data = request_headers.post_data.toString(CryptoJS.enc.Latin1).split("\n");
+ client_group_data = wtvdl.getGroupDataFromClientPost(request_headers.post_data.toString(CryptoJS.enc.Latin1));
+ }
var post_data_current_directory = '';
var post_data_current_file = false;
var post_data_current_group = '';
@@ -178,7 +204,7 @@ if (request_headers['wtv-request-type'] == 'download') {
}
});
var wtv_download_list = new Array();
- var newest_file_epoch = 0;
+ var newest_file_epoch = version;
Object.keys(diskmap_group_data.files).forEach(function (k) {
if (!diskmap_group_data.files[k].location) diskmap_group_data.files[k].location = diskmap_group_data.location + diskmap_group_data.files[k].file.replace(diskmap_group_data.base, "");
var post_match_file = null;
@@ -188,9 +214,7 @@ if (request_headers['wtv-request-type'] == 'download') {
if (!fs.existsSync(post_match_file)) post_match_file = null;
});
-
-
- var post_match_file_lstat = fs.lstatSync(post_match_file);
+ var post_match_file_lstat = fs.lstatSync(post_match_file);
var post_match_file_data = new Buffer.from(fs.readFileSync(post_match_file, {
encoding: null,
flags: 'r'
@@ -198,7 +222,7 @@ if (request_headers['wtv-request-type'] == 'download') {
diskmap_group_data.files[k].base = diskmap_group_data.base;
diskmap_group_data.files[k].last_modified = (new Date(new Date(post_match_file_lstat.mtime).toUTCString()) / 1000);
diskmap_group_data.files[k].content_length = post_match_file_lstat.size;
- diskmap_group_data.files[k].action = (diskmap_group_data.files[k].action) ? diskmap_group_data.files[k].action.toUpperCase() : "GET";
+ diskmap_group_data.files[k].action = (diskmap_group_data.files[k].action) ? diskmap_group_data.files[k].action.toUpperCase() : "GET";
if (wtvshared.getFileExt(post_match_file).toLowerCase() == "gz") {
// we need the checksum of the uncompressed data
@@ -208,7 +232,7 @@ if (request_headers['wtv-request-type'] == 'download') {
if (!diskmap_group_data.files[k].dont_extract_filename) {
diskmap_group_data.files[k].original_filename = post_match_file_data.toString('utf8', 10, gzip_fn_end);
}
- //diskmap_group_data.files[k].uncompressed_size = gunzipped.byteLength;
+ diskmap_group_data.files[k].uncompressed_size = gunzipped.byteLength;
gunzipped = null;
} else {
diskmap_group_data.files[k].checksum = CryptoJS.MD5(CryptoJS.lib.WordArray.create(post_match_file_data)).toString(CryptoJS.enc.Hex).toLowerCase();
@@ -232,6 +256,7 @@ if (request_headers['wtv-request-type'] == 'download') {
});
});
var diskmap_group_name = (diskmap_subgroup == null) ? diskmap_primary_group : diskmap_primary_group + "-" + diskmap_subgroup;
+ diskmap_group_data.client_group_data = client_group_data[diskmap_group_name] || null;
output_data = generateDownloadList(diskmap_group_name, wtv_download_list, diskmap_group_data);
return output_data;
}
@@ -245,9 +270,10 @@ if (request_headers['wtv-request-type'] == 'download') {
});
if (diskmap_json_file != null) {
- if (fs.lstatSync(diskmap_json_file)) {
+ if (fs.existsSync(diskmap_json_file)) {
try {
// read diskmap
+ var json_stats = fs.lstatSync(diskmap_json_file);
var diskmap_data = JSON.parse(fs.readFileSync(diskmap_json_file).toString());
if (!diskmap_data[request_headers.query.group]) {
throw ("Invalid diskmap data (group does not match)");
@@ -256,10 +282,15 @@ if (request_headers['wtv-request-type'] == 'download') {
diskmap_data = diskmap_data[request_headers.query.group];
if (!diskmap_data.location) {
Object.keys(diskmap_data).forEach(function (k) {
- if (diskmap_data[k]) data += processGroup(request_headers.query.group, diskmap_data[k], k);
+ console.log(diskmap_data[k]);
+ if (diskmap_data[k]) {
+ diskmap_data[k].version = (new Date(new Date(json_stats.mtime).toUTCString()) / 1000);
+ data += processGroup(request_headers.query.group, diskmap_data[k], k, diskmap_data.version);
+ }
});
} else {
- data = processGroup(request_headers.query.group, diskmap_data);
+ diskmap_data.version = (new Date(new Date(json_stats.mtime).toUTCString()) / 1000);
+ data = processGroup(request_headers.query.group, diskmap_data, null, diskmap_data.version);
}
headers = "200 OK\nContent-Type: wtv/download-list";
@@ -286,5 +317,5 @@ if (request_headers['wtv-request-type'] == 'download') {
var message = request_headers.query.message || "Retrieving files...";
var main_message = request_headers.query.main_message || "Your receiver is downloading files.";
headers = "200 OK\nwtv-connection-close: close\nConnection: close\nContent-Type: text/html";
- data = wtvdl.getSyncPage(message, request_headers.query.group, request_headers.query.diskmap, main_message, message, force_update)
+ data = wtvdl.getSyncPage(message, request_headers.query.group, request_headers.query.diskmap, main_message, message, force_update, no_delete);
}
\ No newline at end of file
diff --git a/zefie_wtvp_minisrv/WTVDownloadList.js b/zefie_wtvp_minisrv/WTVDownloadList.js
index 9daa7e45..9e259546 100644
--- a/zefie_wtvp_minisrv/WTVDownloadList.js
+++ b/zefie_wtvp_minisrv/WTVDownloadList.js
@@ -101,12 +101,14 @@ class WTVDownloadList {
/**
* Adds a DELETE command to the download list
* @param {string} path Non-absolute path of client destination file (relative to group base) if group defined, otherwise absolute file://Disk/ path to delete
- * @param {string} group Group to which it belongs
+ * @param {string|null} group Group to which it belongs
+ * @param {string|null} original_filename Use this filename (useful if WTV GZ)
*/
- delete(path, group = null) {
- path = this.wtvshared.stripGzipFromPath(path);
+ delete(path, group = null, original_filename = null) {
+ path = this.checkOriginalName(path, original_filename);
this.download_list += "DELETE " + path + "\n";
- if (group !== null) this.download_list += "group: " + group + "\n\n";
+ if (group) this.download_list += "group: " + group + "\n\n";
+ else (this.download_list) += "\n";
}
/**
@@ -142,8 +144,9 @@ class WTVDownloadList {
* @param {string} path Absolute file://Disk/ path of destination
* @param {string} source wtv-url to fetch file from
* @param {string} group Group this file belongs to
- * @param {string} display Message to display while working on this file
- * @param {string} checksum md5sum of the file
+ * @param {string|null} checksum md5sum of the file
+ * @param {string|null} uncompressed_size Uncompressed size of gzip file
+ * @param {string|null} original_filename Use this filename (useful if WTV GZ)
* @param {string} file_permission File permissions
*/
get(file, path, source, group, checksum = null, uncompressed_size = null, original_filename = null, file_permission = 'r') {
@@ -166,16 +169,54 @@ class WTVDownloadList {
this.download_list += "client-dest-location: " + path + "\n\n";
}
+ /**
+ * Helper function for WTV GZIP, if original_name is set, use this filename instead of the name in the path
+ * @param {string} path
+ * @param {string} original_name
+ * @returns {string} Path, with filename replaces by original_name, or just path if original_name = null
+ */
+ checkOriginalName(path, original_name) {
+ if (original_name) {
+ var tmp = this.wtvshared.getFilePath(path);
+ if (tmp.length > 0) return tmp + "/" + original_name;
+ return original_name
+ } else return path;
+ }
+
+ getGroupDataFromClientPost(post_data) {
+ if (typeof post_data == 'string') post_data = post_data.split("\n\n");
+ var group_data = [];
+ var i = 0;
+ post_data.forEach(function (v) {
+ if (v.substr(0, 4) == "file") {
+ var block_split = v.split("\n");
+ var group_data_entry = {};
+ group_data_entry.path = block_split[0];
+ block_split.forEach(function (block_section) {
+ if (block_section.indexOf(": ") > 0) {
+ var block_section_split = block_section.split(": ");
+ group_data_entry[block_section_split[0]] = block_section_split[1];
+ }
+ });
+ group_data[group_data_entry.group] = group_data_entry;
+ }
+ });
+ return group_data;
+ }
+
/**
* Adds a RENAME command to the download list
* @param {string} srcfile Non-absolute path of client source file (relative to source group base)
* @param {string} destfile Non-absolute path of client destination file (relative to destination group base)
* @param {string} srcgroup Source Group
* @param {string} destgroup Destination Group
+ * @param {string} original_filename Use this filename (useful if WTV GZ)
*/
- rename(srcfile, destfile, srcgroup, destgroup) {
- srcfile = this.wtvshared.stripGzipFromPath(srcfile);
- destfile = this.wtvshared.stripGzipFromPath(destfile);
+ rename(srcfile, destfile, srcgroup, destgroup, original_filename = null) {
+ if (original_filename) {
+ srcfile = this.checkOriginalName(srcfile, original_filename);
+ destfile = this.checkOriginalName(srcfile, original_filename);
+ }
this.download_list += "RENAME " + srcfile + "\n";
this.download_list += "group: " + srcgroup + "-UPDATE\n";
this.download_list += "destination-group: " + destgroup + "\n";
@@ -227,7 +268,7 @@ class WTVDownloadList {
* @param {string|null} url Use your own URL for client:fetch?source= instead of our generated one
* @returns {string} HTML Download Page
*/
- getSyncPage(title, group, diskmap = null, main_message = null, message = null, force_update = null, success_url = null, fail_url = null, url = null) {
+ getSyncPage(title, group, diskmap = null, main_message = null, message = null, force_update = null, dont_delete_files = null, success_url = null, fail_url = null, url = null) {
// Begin Set defaults
if (main_message === null) main_message = "Your receiver is downloading files.";
@@ -235,7 +276,10 @@ class WTVDownloadList {
if (force_update === null) force_update = false;
- if (url === null) url = this.service_name + ":/sync?diskmap=" + escape(diskmap) + "&force=" + force_update;
+ if (url === null) url = this.service_name + ":/sync?diskmap=" + escape(diskmap);
+
+ if (force_update) url += "&force=" + force_update;
+ if (dont_delete_files) url += "&dont_delete_files=" + dont_delete_files;
if (success_url === null) success_url = new this.clientShowAlert({
'image': this.minisrv_config.config.service_logo,
diff --git a/zefie_wtvp_minisrv/package-lock.json b/zefie_wtvp_minisrv/package-lock.json
index bd758ac4..b87f18c9 100644
--- a/zefie_wtvp_minisrv/package-lock.json
+++ b/zefie_wtvp_minisrv/package-lock.json
@@ -1,6 +1,6 @@
{
"name": "zefie_wtvp_minisrv",
- "version": "0.9.11",
+ "version": "0.9.18",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
diff --git a/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj b/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj
index 8bb0702b..5924d734 100644
--- a/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj
+++ b/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj
@@ -32,6 +32,9 @@
+
+ Code
+