diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-disk/delete-group.js b/zefie_wtvp_minisrv/ServiceVault/wtv-disk/delete-group.js index 3421d09d..cc568a9c 100644 --- a/zefie_wtvp_minisrv/ServiceVault/wtv-disk/delete-group.js +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-disk/delete-group.js @@ -4,12 +4,13 @@ if (request_headers.query.group) { if (request_headers['wtv-request-type']) { headers = "200 OK\nContent-Type: " + wtvdl.content_type; wtvdl.deleteGroup(request_headers.query.group); + wtvdl.deleteGroupUpdate(request_headers.query.group); data = wtvdl.getDownloadList(); } var title = "Deleting group" var message = title + " " + request_headers.query.group; headers = "200 OK\nContent-Type: text/html" - data = wtvdl.getSyncPage(title, request_headers.query.group, "delete", message, message, null, "client:goback", "client:goback", "wtv-disk:/delete-group"); + data = wtvdl.getSyncPage(title, request_headers.query.group, "delete", message, message, null, null, "client:goback", "client:goback", "wtv-disk:/delete-group"); } else { headers = "200 OK\nContent-Type: text/html" data = ` diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-disk/sync.js b/zefie_wtvp_minisrv/ServiceVault/wtv-disk/sync.js index 6f37edd8..2a138e8c 100644 --- a/zefie_wtvp_minisrv/ServiceVault/wtv-disk/sync.js +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-disk/sync.js @@ -2,11 +2,12 @@ const WTVDownloadList = require("./WTVDownloadList.js"); var wtvdl = new WTVDownloadList(minisrv_config, service_name); var force_update = (request_headers.query.force == "true") ? true : false; +var no_delete = (request_headers.query.dont_delete_files == "true") ? true : false; if (request_headers['wtv-request-type'] == 'download') { var content_dir = "content/" var diskmap_dir = content_dir + "diskmaps/"; - function generateDownloadList(diskmap_group_data, update_list, diskmap_data) { + function generateDownloadList(diskmap_group_name, update_list, diskmap_group_data) { wtvdl.reset(); var files_to_send = 0; Object.keys(update_list).forEach(function (k) { @@ -16,77 +17,98 @@ if (request_headers['wtv-request-type'] == 'download') { }); // create WebTV Download List - if (diskmap_data.execute && diskmap_data.execute_when) { - if (diskmap_data.execute_when.toLowerCase().match(/start/)) { - wtvdl.execute(diskmap_data.execute); + if (diskmap_group_data.execute && diskmap_group_data.execute_when) { + if (diskmap_group_data.execute_when.toLowerCase().match(/start/)) { + wtvdl.execute(diskmap_group_data.execute); } } - if (diskmap_group_data.display) wtvdl.display(diskmap_group_data.display); + console.log(diskmap_group_data.client_group_data); + + // delete groups if force, or group is invalid + if (diskmap_group_data.client_group_data) { + if (force_update || diskmap_group_data.client_group_data.state == "invalid") { + wtvdl.deleteGroupUpdate(diskmap_group_data.client_group_data.group, diskmap_group_data.client_group_data.path); + wtvdl.deleteGroup(diskmap_group_data.client_group_data.group, diskmap_group_data.client_group_data.path); + } + + // delete partition/path if force, and not no_delete + if (force_update && !no_delete) { + // don't delete Brower partition, ever, but allow deleting of Browser partition subdirs + if (!diskmap_group_data.base.match(/disk\/browser(\/)?$/i)) { + if (diskmap_group_data.client_group_data.path.toLowerCase() == diskmap_group_data.base.toLowerCase()) { + wtvdl.delete(diskmap_group_data.base, null); + } + } + } + } + + if (diskmap_group_name.display) wtvdl.display(diskmap_group_name.display); if (files_to_send > 0) { - if (diskmap_data.partition_size) { - wtvdl.createPartition(diskmap_data.base, diskmap_data.partition_size); + if (diskmap_group_data.partition_size) { + wtvdl.createPartition(diskmap_group_data.base, diskmap_group_data.partition_size); } - if (!diskmap_data.nogroup) { + if (!diskmap_group_data.nogroup) { // only send group commands if group mode is enable // useful to disable for PUT - wtvdl.createUpdateGroup(diskmap_group_data, diskmap_data.base, "invalid", (diskmap_data.service_owned || false)); + wtvdl.createUpdateGroup(diskmap_group_name, diskmap_group_data.base, "invalid", (diskmap_group_data.service_owned || false)); } Object.keys(update_list).forEach(function (k) { // file { "action": "delete" } // Useful to purge files we no longer want on the client - if (update_list[k].action != "DELETE") { + if (update_list[k].action != "DELETE" && update_list[k].action != "DELETEONLY") { // skip deleting valid files if we aren't specifically requesting their deletion if (update_list[k].checksum_match && !force_update) return; if (!update_list[k].invalid && !force_update) return; } - wtvdl.delete(update_list[k].file.replace(diskmap_data.base, ""), diskmap_group_data); + wtvdl.delete(update_list[k].file.replace(diskmap_group_data.base, ""), diskmap_group_name); }); Object.keys(update_list).forEach(function (k) { if (update_list[k].checksum_match && !force_update) return; if (!update_list[k].invalid && !force_update) return; + if (update_list[k].action == "DELETEONLY") return; if (update_list[k].display) wtvdl.display(update_list[k].display); switch (update_list[k].action) { case "PUT": - wtvdl.put(update_list[k].file.replace(diskmap_data.base, ""), service_name + ":/" + update_list[k].location, update_list[k].display); + wtvdl.put(update_list[k].file.replace(diskmap_group_data.base, ""), service_name + ":/" + update_list[k].location, update_list[k].display); break; case "GET": - wtvdl.get(update_list[k].file.replace(diskmap_data.base, ""), update_list[k].file, service_name + ":/" + update_list[k].location, diskmap_group_data, update_list[k].checksum, update_list[k].uncompressed_size || null, update_list[k].original_filename) + wtvdl.get(update_list[k].file.replace(diskmap_group_data.base, ""), update_list[k].file, service_name + ":/" + update_list[k].location, diskmap_group_name, update_list[k].checksum, update_list[k].uncompressed_size || null, update_list[k].original_filename) break; } }); - if (!diskmap_data.nogroup) { - wtvdl.createGroup(diskmap_group_data, diskmap_data.base, "invalid", (diskmap_data.service_owned || false)); - + if (!diskmap_group_data.nogroup) { + wtvdl.createGroup(diskmap_group_name, diskmap_group_data.base, "invalid", (diskmap_group_data.service_owned || false)); // this rename loop is a part of the group system Object.keys(update_list).forEach(function (k) { if (update_list[k].checksum_match && !force_update) return; if (!update_list[k].invalid && !force_update) return; - wtvdl.rename(update_list[k].file.replace(diskmap_data.base, ""), update_list[k].file.replace(diskmap_data.base, ""), diskmap_group_data, diskmap_group_data); + if (update_list[k].action == "DELETEONLY") return; + wtvdl.rename(update_list[k].file.replace(diskmap_group_data.base, ""), update_list[k].file.replace(diskmap_group_data.base, ""), diskmap_group_name, diskmap_group_name, update_list[k].original_filename); }); - wtvdl.setGroup(diskmap_group_data, 'ok', diskmap_data.version); + wtvdl.setGroup(diskmap_group_name, 'ok', diskmap_group_data.version); } } - if (diskmap_data.execute && diskmap_data.execute_when) { - if (diskmap_data.execute_when.toLowerCase().match(/end/)) { - wtvdl.execute(diskmap_data.execute); + if (diskmap_group_data.execute && diskmap_group_data.execute_when) { + if (diskmap_group_data.execute_when.toLowerCase().match(/end/)) { + wtvdl.execute(diskmap_group_data.execute); } } if (files_to_send > 0) { - if (!diskmap_data.nogroup) { - wtvdl.deleteGroupUpdate(diskmap_group_data, diskmap_data.base); + if (!diskmap_group_data.nogroup) { + wtvdl.deleteGroupUpdate(diskmap_group_name, diskmap_group_data.base); } } var download_list = wtvdl.getDownloadList(); @@ -94,11 +116,15 @@ if (request_headers['wtv-request-type'] == 'download') { return download_list; } - function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgroup = null) { + function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgroup = null, version = 0) { // parse webtv post var output_data = ''; var post_data = new Array(); - if (request_headers.post_data) post_data = request_headers.post_data.toString(CryptoJS.enc.Latin1).split("\n"); + var client_group_data = new Array(); + if (request_headers.post_data) { + post_data = request_headers.post_data.toString(CryptoJS.enc.Latin1).split("\n"); + client_group_data = wtvdl.getGroupDataFromClientPost(request_headers.post_data.toString(CryptoJS.enc.Latin1)); + } var post_data_current_directory = ''; var post_data_current_file = false; var post_data_current_group = ''; @@ -178,7 +204,7 @@ if (request_headers['wtv-request-type'] == 'download') { } }); var wtv_download_list = new Array(); - var newest_file_epoch = 0; + var newest_file_epoch = version; Object.keys(diskmap_group_data.files).forEach(function (k) { if (!diskmap_group_data.files[k].location) diskmap_group_data.files[k].location = diskmap_group_data.location + diskmap_group_data.files[k].file.replace(diskmap_group_data.base, ""); var post_match_file = null; @@ -188,9 +214,7 @@ if (request_headers['wtv-request-type'] == 'download') { if (!fs.existsSync(post_match_file)) post_match_file = null; }); - - - var post_match_file_lstat = fs.lstatSync(post_match_file); + var post_match_file_lstat = fs.lstatSync(post_match_file); var post_match_file_data = new Buffer.from(fs.readFileSync(post_match_file, { encoding: null, flags: 'r' @@ -198,7 +222,7 @@ if (request_headers['wtv-request-type'] == 'download') { diskmap_group_data.files[k].base = diskmap_group_data.base; diskmap_group_data.files[k].last_modified = (new Date(new Date(post_match_file_lstat.mtime).toUTCString()) / 1000); diskmap_group_data.files[k].content_length = post_match_file_lstat.size; - diskmap_group_data.files[k].action = (diskmap_group_data.files[k].action) ? diskmap_group_data.files[k].action.toUpperCase() : "GET"; + diskmap_group_data.files[k].action = (diskmap_group_data.files[k].action) ? diskmap_group_data.files[k].action.toUpperCase() : "GET"; if (wtvshared.getFileExt(post_match_file).toLowerCase() == "gz") { // we need the checksum of the uncompressed data @@ -208,7 +232,7 @@ if (request_headers['wtv-request-type'] == 'download') { if (!diskmap_group_data.files[k].dont_extract_filename) { diskmap_group_data.files[k].original_filename = post_match_file_data.toString('utf8', 10, gzip_fn_end); } - //diskmap_group_data.files[k].uncompressed_size = gunzipped.byteLength; + diskmap_group_data.files[k].uncompressed_size = gunzipped.byteLength; gunzipped = null; } else { diskmap_group_data.files[k].checksum = CryptoJS.MD5(CryptoJS.lib.WordArray.create(post_match_file_data)).toString(CryptoJS.enc.Hex).toLowerCase(); @@ -232,6 +256,7 @@ if (request_headers['wtv-request-type'] == 'download') { }); }); var diskmap_group_name = (diskmap_subgroup == null) ? diskmap_primary_group : diskmap_primary_group + "-" + diskmap_subgroup; + diskmap_group_data.client_group_data = client_group_data[diskmap_group_name] || null; output_data = generateDownloadList(diskmap_group_name, wtv_download_list, diskmap_group_data); return output_data; } @@ -245,9 +270,10 @@ if (request_headers['wtv-request-type'] == 'download') { }); if (diskmap_json_file != null) { - if (fs.lstatSync(diskmap_json_file)) { + if (fs.existsSync(diskmap_json_file)) { try { // read diskmap + var json_stats = fs.lstatSync(diskmap_json_file); var diskmap_data = JSON.parse(fs.readFileSync(diskmap_json_file).toString()); if (!diskmap_data[request_headers.query.group]) { throw ("Invalid diskmap data (group does not match)"); @@ -256,10 +282,15 @@ if (request_headers['wtv-request-type'] == 'download') { diskmap_data = diskmap_data[request_headers.query.group]; if (!diskmap_data.location) { Object.keys(diskmap_data).forEach(function (k) { - if (diskmap_data[k]) data += processGroup(request_headers.query.group, diskmap_data[k], k); + console.log(diskmap_data[k]); + if (diskmap_data[k]) { + diskmap_data[k].version = (new Date(new Date(json_stats.mtime).toUTCString()) / 1000); + data += processGroup(request_headers.query.group, diskmap_data[k], k, diskmap_data.version); + } }); } else { - data = processGroup(request_headers.query.group, diskmap_data); + diskmap_data.version = (new Date(new Date(json_stats.mtime).toUTCString()) / 1000); + data = processGroup(request_headers.query.group, diskmap_data, null, diskmap_data.version); } headers = "200 OK\nContent-Type: wtv/download-list"; @@ -286,5 +317,5 @@ if (request_headers['wtv-request-type'] == 'download') { var message = request_headers.query.message || "Retrieving files..."; var main_message = request_headers.query.main_message || "Your receiver is downloading files."; headers = "200 OK\nwtv-connection-close: close\nConnection: close\nContent-Type: text/html"; - data = wtvdl.getSyncPage(message, request_headers.query.group, request_headers.query.diskmap, main_message, message, force_update) + data = wtvdl.getSyncPage(message, request_headers.query.group, request_headers.query.diskmap, main_message, message, force_update, no_delete); } \ No newline at end of file diff --git a/zefie_wtvp_minisrv/WTVDownloadList.js b/zefie_wtvp_minisrv/WTVDownloadList.js index 9daa7e45..9e259546 100644 --- a/zefie_wtvp_minisrv/WTVDownloadList.js +++ b/zefie_wtvp_minisrv/WTVDownloadList.js @@ -101,12 +101,14 @@ class WTVDownloadList { /** * Adds a DELETE command to the download list * @param {string} path Non-absolute path of client destination file (relative to group base) if group defined, otherwise absolute file://Disk/ path to delete - * @param {string} group Group to which it belongs + * @param {string|null} group Group to which it belongs + * @param {string|null} original_filename Use this filename (useful if WTV GZ) */ - delete(path, group = null) { - path = this.wtvshared.stripGzipFromPath(path); + delete(path, group = null, original_filename = null) { + path = this.checkOriginalName(path, original_filename); this.download_list += "DELETE " + path + "\n"; - if (group !== null) this.download_list += "group: " + group + "\n\n"; + if (group) this.download_list += "group: " + group + "\n\n"; + else (this.download_list) += "\n"; } /** @@ -142,8 +144,9 @@ class WTVDownloadList { * @param {string} path Absolute file://Disk/ path of destination * @param {string} source wtv-url to fetch file from * @param {string} group Group this file belongs to - * @param {string} display Message to display while working on this file - * @param {string} checksum md5sum of the file + * @param {string|null} checksum md5sum of the file + * @param {string|null} uncompressed_size Uncompressed size of gzip file + * @param {string|null} original_filename Use this filename (useful if WTV GZ) * @param {string} file_permission File permissions */ get(file, path, source, group, checksum = null, uncompressed_size = null, original_filename = null, file_permission = 'r') { @@ -166,16 +169,54 @@ class WTVDownloadList { this.download_list += "client-dest-location: " + path + "\n\n"; } + /** + * Helper function for WTV GZIP, if original_name is set, use this filename instead of the name in the path + * @param {string} path + * @param {string} original_name + * @returns {string} Path, with filename replaces by original_name, or just path if original_name = null + */ + checkOriginalName(path, original_name) { + if (original_name) { + var tmp = this.wtvshared.getFilePath(path); + if (tmp.length > 0) return tmp + "/" + original_name; + return original_name + } else return path; + } + + getGroupDataFromClientPost(post_data) { + if (typeof post_data == 'string') post_data = post_data.split("\n\n"); + var group_data = []; + var i = 0; + post_data.forEach(function (v) { + if (v.substr(0, 4) == "file") { + var block_split = v.split("\n"); + var group_data_entry = {}; + group_data_entry.path = block_split[0]; + block_split.forEach(function (block_section) { + if (block_section.indexOf(": ") > 0) { + var block_section_split = block_section.split(": "); + group_data_entry[block_section_split[0]] = block_section_split[1]; + } + }); + group_data[group_data_entry.group] = group_data_entry; + } + }); + return group_data; + } + /** * Adds a RENAME command to the download list * @param {string} srcfile Non-absolute path of client source file (relative to source group base) * @param {string} destfile Non-absolute path of client destination file (relative to destination group base) * @param {string} srcgroup Source Group * @param {string} destgroup Destination Group + * @param {string} original_filename Use this filename (useful if WTV GZ) */ - rename(srcfile, destfile, srcgroup, destgroup) { - srcfile = this.wtvshared.stripGzipFromPath(srcfile); - destfile = this.wtvshared.stripGzipFromPath(destfile); + rename(srcfile, destfile, srcgroup, destgroup, original_filename = null) { + if (original_filename) { + srcfile = this.checkOriginalName(srcfile, original_filename); + destfile = this.checkOriginalName(srcfile, original_filename); + } this.download_list += "RENAME " + srcfile + "\n"; this.download_list += "group: " + srcgroup + "-UPDATE\n"; this.download_list += "destination-group: " + destgroup + "\n"; @@ -227,7 +268,7 @@ class WTVDownloadList { * @param {string|null} url Use your own URL for client:fetch?source= instead of our generated one * @returns {string} HTML Download Page */ - getSyncPage(title, group, diskmap = null, main_message = null, message = null, force_update = null, success_url = null, fail_url = null, url = null) { + getSyncPage(title, group, diskmap = null, main_message = null, message = null, force_update = null, dont_delete_files = null, success_url = null, fail_url = null, url = null) { // Begin Set defaults if (main_message === null) main_message = "Your receiver is downloading files."; @@ -235,7 +276,10 @@ class WTVDownloadList { if (force_update === null) force_update = false; - if (url === null) url = this.service_name + ":/sync?diskmap=" + escape(diskmap) + "&force=" + force_update; + if (url === null) url = this.service_name + ":/sync?diskmap=" + escape(diskmap); + + if (force_update) url += "&force=" + force_update; + if (dont_delete_files) url += "&dont_delete_files=" + dont_delete_files; if (success_url === null) success_url = new this.clientShowAlert({ 'image': this.minisrv_config.config.service_logo, diff --git a/zefie_wtvp_minisrv/package-lock.json b/zefie_wtvp_minisrv/package-lock.json index bd758ac4..b87f18c9 100644 --- a/zefie_wtvp_minisrv/package-lock.json +++ b/zefie_wtvp_minisrv/package-lock.json @@ -1,6 +1,6 @@ { "name": "zefie_wtvp_minisrv", - "version": "0.9.11", + "version": "0.9.18", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj b/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj index 8bb0702b..5924d734 100644 --- a/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj +++ b/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj @@ -32,6 +32,9 @@ + + Code +