diff --git a/zefie_wtvp_minisrv/ServiceVault/wtv-disk/sync.js b/zefie_wtvp_minisrv/ServiceVault/wtv-disk/sync.js index 091fcd5d..24ec9e75 100644 --- a/zefie_wtvp_minisrv/ServiceVault/wtv-disk/sync.js +++ b/zefie_wtvp_minisrv/ServiceVault/wtv-disk/sync.js @@ -2,87 +2,105 @@ var force_update = (request_headers.query.force == "true") ? true : false; if (request_headers['wtv-request-type'] == 'download') { - var path = require("path"); + const WTVDownloadList = require("./WTVDownloadList.js"); + var wtvdl = new WTVDownloadList(); var content_dir = "content/" var diskmap_dir = content_dir + "diskmaps/"; function generateDownloadList(diskmap_group_data, update_list, diskmap_data) { + wtvdl.reset(); + var files_to_send = 0; + Object.keys(update_list).forEach(function (k) { + if (update_list[k].checksum_match && !force_update) return; + if (!update_list[k].invalid && !force_update) return; + files_to_send++; + }); + // create WebTV Download List - var newest_file_epoch = 0; - var download_list = ''; - - if (diskmap_data.execute && diskmap_data.execute_when == "atStart") { - download_list += "EXECUTE " + diskmap_data.execute + "\n\n"; + if (diskmap_data.execute && diskmap_data.execute_when) { + if (diskmap_data.execute_when.toLowerCase().match(/start/)) { + wtvdl.execute(diskmap_data.execute); + } } - if (diskmap_data.partition_size) { - download_list += "CREATE " + diskmap_data.base + "\n"; - download_list += "partition-size: " + diskmap_data.partition_size + "\n\n"; + if (diskmap_group_data.display) wtvdl.display(diskmap_group_data.display); + + if (files_to_send > 0) { + + if (diskmap_data.partition_size) { + wtvdl.createPartition(diskmap_data.base, diskmap_data.partition_size); + } + + if (!diskmap_data.nogroup) { + // only send group commands if group mode is enable + // useful to disable for PUT + wtvdl.createUpdateGroup(diskmap_group_data, diskmap_data.base, "invalid", (diskmap_data.service_owned || false)); + } + + Object.keys(update_list).forEach(function (k) { + // file { "action": "delete" } + // Useful to purge files we no longer want on the client + if (update_list[k].action != "DELETE") { + // skip deleting valid files if we aren't specifically requesting their deletion + if (update_list[k].checksum_match && !force_update) return; + if (!update_list[k].invalid && !force_update) return; + } + wtvdl.delete(update_list[k].file.replace(diskmap_data.base, ""), diskmap_group_data); + }); + + Object.keys(update_list).forEach(function (k) { + if (update_list[k].checksum_match && !force_update) return; + if (!update_list[k].invalid && !force_update) return; + if (update_list[k].display) wtvdl.display(update_list[k].display); + switch (update_list[k].action) { + case "PUT": + wtvdl.put(update_list[k].file.replace(diskmap_data.base, ""), service_name + ":/" + update_list[k].location, update_list[k].display); + break; + + case "GET": + wtvdl.get(update_list[k].file.replace(diskmap_data.base, ""), update_list[k].file, service_name + ":/" + update_list[k].location, diskmap_group_data, update_list[k].checksum) + break; + } + }); + + if (!diskmap_data.nogroup) { + wtvdl.createGroup(diskmap_group_data, diskmap_data.base, "invalid", (diskmap_data.service_owned || false)); + + + // this rename loop is a part of the group system + Object.keys(update_list).forEach(function (k) { + if (update_list[k].checksum_match && !force_update) return; + if (!update_list[k].invalid && !force_update) return; + wtvdl.rename(update_list[k].file.replace(diskmap_data.base, ""), update_list[k].file.replace(diskmap_data.base, ""), diskmap_group_data, diskmap_group_data); + }); + + wtvdl.setGroup(diskmap_group_data, 'ok', diskmap_data.version); + } + } - download_list += "CREATE-GROUP " + diskmap_group_data + "-UPDATE\n"; - download_list += "state: invalid\n"; - download_list += "base: " + diskmap_data.base + ".GROUP-UPDATE/\n\n"; - - download_list += "CREATE-GROUP " + diskmap_group_data + "\n"; - download_list += "state: invalid\n"; - download_list += "service-owned: " + (diskmap_data.service_owned || false) + "\n"; - download_list += "base: " + diskmap_data.base + "\n\n"; - - Object.keys(update_list).forEach(function (k) { - if (update_list[k].checksum_match && !force_update) return; - if (!update_list[k].invalid && !force_update) return; - download_list += "DELETE " + update_list[k].file.replace(diskmap_data.base, "") + "\n"; - download_list += "group: " + diskmap_group_data + "\n\n"; - }); - - Object.keys(update_list).forEach(function (k) { - if (update_list[k].checksum_match && !force_update) return; - if (!update_list[k].invalid && !force_update) return; - download_list += "DISPLAY " + update_list[k].display + "\n\n"; - download_list += "GET " + update_list[k].file.replace(diskmap_data.base, "") + "\n"; - download_list += "group: " + diskmap_group_data + "-UPDATE\n"; - download_list += "location: " + service_name + ":/" + update_list[k].location + "\n"; - download_list += "file-permission: r\n" - download_list += "wtv-checksum: " + update_list[k].checksum + "\n"; - download_list += "service-source-location: /webtv/content/" + service_name.replace("wtv-", "") + "d/" + update_list[k].location + "\n"; - download_list += "client-dest-location: " + update_list[k].file + "\n\n"; - }); - - download_list += "CREATE-GROUP " + diskmap_group_data + "\n"; - download_list += "state: invalid\n"; - download_list += "service-owned: " + (diskmap_data.service_owned || false) + "\n"; - download_list += "base: " + diskmap_data.base + "\n\n"; - - - Object.keys(update_list).forEach(function (k) { - if (update_list[k].checksum_match && !force_update) return; - if (!update_list[k].invalid && !force_update) return; - download_list += "RENAME " + update_list[k].file.replace(diskmap_data.base, "") + "\n"; - download_list += "group: " + diskmap_group_data + "-UPDATE\n"; - download_list += "destination-group: " + diskmap_group_data + "\n"; - download_list += "location: " + update_list[k].file.replace(diskmap_data.base, "") + "\n\n"; - }); - - download_list += "SET-GROUP " + diskmap_group_data + "\n"; - download_list += "state: ok\n"; - download_list += "version: " + diskmap_data.version + "\n"; - download_list += "last-checkup-time: " + new Date().toUTCString().replace("GMT", "+0000") + "\n\n"; - - if (diskmap_data.execute && diskmap_data.execute_when == "atEnd") { - download_list += "EXECUTE " + diskmap_data.execute + "\n\n"; + if (diskmap_data.execute && diskmap_data.execute_when) { + if (diskmap_data.execute_when.toLowerCase().match(/end/)) { + wtvdl.execute(diskmap_data.execute); + } } - download_list += "DELETE-GROUP " + diskmap_group_data + "-UPDATE\n\n"; - download_list += "DELETE " + diskmap_data.base + ".GROUP-UPDATE/\n\n"; + if (files_to_send > 0) { + if (!diskmap_data.nogroup) { + wtvdl.deleteGroupUpdate(diskmap_group_data, diskmap_data.base); + } + } + var download_list = wtvdl.getDownloadList(); + console.log(download_list); return download_list; } function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgroup = null) { // parse webtv post var output_data = ''; - var post_data = request_headers.post_data.toString(CryptoJS.enc.Latin1).split("\n"); + var post_data = new Array(); + if (request_headers.post_data) post_data = request_headers.post_data.toString(CryptoJS.enc.Latin1).split("\n"); var post_data_current_directory = ''; var post_data_current_file = false; var post_data_current_group = ''; @@ -181,9 +199,10 @@ if (request_headers['wtv-request-type'] == 'download') { diskmap_group_data.files[k].last_modified = (new Date(new Date(post_match_file_lstat.mtime).toUTCString()) / 1000); diskmap_group_data.files[k].content_length = post_match_file_lstat.size; diskmap_group_data.files[k].checksum = CryptoJS.MD5(CryptoJS.lib.WordArray.create(post_match_file_data)).toString(CryptoJS.enc.Hex).toLowerCase(); + diskmap_group_data.files[k].action = (diskmap_group_data.files[k].action) ? diskmap_group_data.files[k].action.toUpperCase() : "GET"; if (parseInt(diskmap_group_data.files[k].last_modified) > newest_file_epoch) newest_file_epoch = parseInt(diskmap_group_data.files[k].last_modified); - if (!diskmap_group_data.files[k].display) diskmap_group_data.files[k].display = diskmap_group_data.display; + //if (!diskmap_group_data.files[k].display) diskmap_group_data.files[k].display = diskmap_group_data.display; diskmap_group_data.files[k].invalid = true; wtv_download_list.push(diskmap_group_data.files[k]); @@ -205,7 +224,7 @@ if (request_headers['wtv-request-type'] == 'download') { return output_data; } - if (request_headers.query.diskmap && request_headers.query.group && request_headers.post_data) { + if (request_headers.query.diskmap && request_headers.query.group) { var diskmap_json_file = null; Object.keys(service_vaults).forEach(function (g) { if (diskmap_json_file != null) return; diff --git a/zefie_wtvp_minisrv/WTVDownloadList.js b/zefie_wtvp_minisrv/WTVDownloadList.js new file mode 100644 index 00000000..99af3336 --- /dev/null +++ b/zefie_wtvp_minisrv/WTVDownloadList.js @@ -0,0 +1,170 @@ +class WTVDownloadList { + + download_list = ""; + content_type = "wtv/download-list"; + + constructor() { + this.clear(); + } + + /** + * Clears the download list + */ + clear() { + this.download_list = ""; + } + + /** + * Alias to clear() (clears the download list) + */ + reset() { + this.clear(); + } + + /** + * Returns the download list. + * @returns {string} Download list for client; + */ + getDownloadList() { + return this.download_list; + } + + /** + * Adds a DISPLAY command to the download list + * @param {string} message Message to display to the client + */ + display(message) { + this.download_list += "DISPLAY " + message + "\n\n"; + } + + /** + * Adds an EXECUTE command to the download list + * @param {string} command client command to execute + */ + execute(command) { + this.download_list += "EXECUTE " + command + "\n\n"; + } + + /** + * Adds a CREATE partition command to the download list + * @param {string} path file://Disk/ path to desired partition + * @param {string} size Size of the desired partition + */ + createPartition(path, size) { + this.download_list += "CREATE " + path + "\n"; + this.download_list += "partition-size: " + size + "\n\n"; + } + + /** + * Adds a CREATE-GROUP command to the download list + * @param {string} name Group name + * @param {string} path file://Disk/ path of desired group + * @param {string} state Group state + * @param {boolean|null} service_owned Sets service owned flag. (null = don't set) + */ + createGroup(name, path, state = 'invalid', service_owned = null) { + this.download_list += "CREATE-GROUP " + name + "\n"; + this.download_list += "state: " + state + "\n"; + if (service_owned !== null) this.download_list += "service-owned: " + service_owned + "\n"; + this.download_list += "base: " + path + "\n\n"; + } + + /** + * An alias for createGroup() that handles creating the '-UPDATE' group for you + * @param {string} name Group name + * @param {string} path file://Disk/ path of desired group + * @param {string} state Group state + * @param {boolean} service_owned Sets service owned flag. + */ + createUpdateGroup(name, path, state = 'invalid', service_owned = false) { + this.createGroup(name + "-UPDATE", path, state); + this.createGroup(name, path, state, service_owned); + } + + /** + * Adds a DELETE command to the download list + * @param {string} path Non-absolute path of client destination file (relative to group base) if group defined, otherwise absolute file://Disk/ path to delete + * @param {string} group Group to which it belongs + */ + delete(path, group = null) { + this.download_list += "DELETE " + path + "\n"; + if (group !== null) this.download_list += "group: " + group + "\n\n"; + } + + /** + * Adds a PUT command to the download list + * @param {string} path Absolute file://Disk/ path of a file to upload to the service + * @param {string} destination Destination address (wtv url on service) in which to POST upload the file to + * @param {string} display Message to display while working on this file + */ + put(path, destination) { + this.download_list += "PUT " + path + "\n"; + this.download_list += "location: " + destination + "\n"; + } + + /** + * Adds a GET command to the download list + * @param {string} file Non-absolute path of client destination file (relative to group base) + * @param {string} path Absolute file://Disk/ path of destination + * @param {string} source wtv-url to fetch file from + * @param {string} group Group this file belongs to + * @param {string} display Message to display while working on this file + * @param {string} checksum md5sum of the file + * @param {string} file_permission File permissions + */ + get(file, path, source, group, checksum = null, file_permission = 'r') { + this.download_list += "GET " + file + "\n"; + this.download_list += "group: " + group + "-UPDATE\n"; + this.download_list += "location: " + source + "\n"; + this.download_list += "file-permission: " + file_permission + "\n"; + if (checksum != null) this.download_list += "wtv-checksum: " + checksum + "\n"; + this.download_list += "service-source-location: /webtv/content/" + source.substr(source.indexOf('-') + 1, source.indexOf(':/') - source.indexOf('-') - 1) + "d/" + source.substr(source.indexOf(':/') + 2) + "\n"; + this.download_list += "client-dest-location: " + path + "\n\n"; + } + + /** + * Adds a RENAME command to the download list + * @param {string} srcfile Non-absolute path of client source file (relative to source group base) + * @param {string} destfile Non-absolute path of client destination file (relative to destination group base) + * @param {string} srcgroup Source Group + * @param {string} destgroup Destination Group + */ + rename(srcfile, destfile, srcgroup, destgroup) { + this.download_list += "RENAME " + srcfile + "\n"; + this.download_list += "group: " + srcgroup + "-UPDATE\n"; + this.download_list += "destination-group: " + destgroup + "\n"; + this.download_list += "location: " + destfile + "\n\n"; + } + + /** + * Adds a SET-GROUP command to the download list + * @param {any} group Group to set state of + * @param {any} state State to set group to + * @param {any} version Version to set group to + */ + setGroup(group, state, version) { + this.download_list += "SET-GROUP " + group + "\n"; + this.download_list += "state: " + state + "\n"; + this.download_list += "version: " + version + "\n"; + this.download_list += "last-checkup-time: " + new Date().toUTCString().replace("GMT", "+0000") + "\n\n"; + } + /** + * Adds a DELETE-GROUP command to the download list + * @param {any} group Group to delete + */ + deleteGroup(group) { + this.download_list += "DELETE-GROUP " + group + "\n\n"; + } + + /** + * An alias for deleteGroup() that handles deleting the '-UPDATE' group files for you + * @param {any} group + * @param {any} path + */ + deleteGroupUpdate(group, path) { + this.deleteGroup(group + "-UPDATE"); + this.delete(path + ".GROUP-UPDATE/"); + } +} + +module.exports = WTVDownloadList; \ No newline at end of file diff --git a/zefie_wtvp_minisrv/app.js b/zefie_wtvp_minisrv/app.js index ac1624bb..039445ae 100644 --- a/zefie_wtvp_minisrv/app.js +++ b/zefie_wtvp_minisrv/app.js @@ -720,8 +720,7 @@ function shouldWeCompress(ssid, headers_obj) { // gzip only if (content_type.match(/^audio\/(x-)?[s3m|mod|xm]$/)) compress_data = true; // s3m, mod, xm if (content_type.match(/^audio\/(x-)?[midi|wav|wave]$/)) compress_data = true; // midi & wav - if (content_type.match(/^binary\/x-wtv-approm$/)) compress_data = true; // midi & wav - + if (content_type.match(/^binary\/x-wtv-approm$/)) compress_data = true; // approms } } } diff --git a/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj b/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj index 24b592f9..eea66531 100644 --- a/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj +++ b/zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj @@ -274,6 +274,9 @@ Code + + Code + Code