Move download list stuff to WTVDownloadList class

- easier to create your own download list in JS
- possible future client upload ability via PUT

zefie_wtvp_minisrv/ServiceVault/wtv-disk/sync.js

@@ -2,87 +2,105 @@
 
 var force_update = (request_headers.query.force == "true") ? true : false;
 if (request_headers['wtv-request-type'] == 'download') {
-    var path = require("path");
+    const WTVDownloadList = require("./WTVDownloadList.js");
+    var wtvdl = new WTVDownloadList();
 
     var content_dir = "content/"
     var diskmap_dir = content_dir + "diskmaps/";
 
     function generateDownloadList(diskmap_group_data, update_list, diskmap_data) {
+        wtvdl.reset();
+        var files_to_send = 0;
+        Object.keys(update_list).forEach(function (k) {
+            if (update_list[k].checksum_match && !force_update) return;
+            if (!update_list[k].invalid && !force_update) return;
+            files_to_send++;
+        });
+
         // create WebTV Download List
-        var newest_file_epoch = 0;
+        if (diskmap_data.execute && diskmap_data.execute_when) {
-        var download_list = '';
+            if (diskmap_data.execute_when.toLowerCase().match(/start/)) {
-
+                wtvdl.execute(diskmap_data.execute);
-        if (diskmap_data.execute && diskmap_data.execute_when == "atStart") {
+            }
-            download_list += "EXECUTE " + diskmap_data.execute + "\n\n";
         }
 
-        if (diskmap_data.partition_size) {
+        if (diskmap_group_data.display) wtvdl.display(diskmap_group_data.display);
-            download_list += "CREATE " + diskmap_data.base + "\n";
+
-            download_list += "partition-size: " + diskmap_data.partition_size + "\n\n";
+        if (files_to_send > 0) {
+
+            if (diskmap_data.partition_size) {
+                wtvdl.createPartition(diskmap_data.base, diskmap_data.partition_size);
+            }
+
+            if (!diskmap_data.nogroup) {
+                // only send group commands if group mode is enable
+                // useful to disable for PUT
+                wtvdl.createUpdateGroup(diskmap_group_data, diskmap_data.base, "invalid", (diskmap_data.service_owned || false));
+            }
+
+            Object.keys(update_list).forEach(function (k) {
+                // file { "action": "delete" }
+                // Useful to purge files we no longer want on the client
+                if (update_list[k].action != "DELETE") {
+                    // skip deleting valid files if we aren't specifically requesting their deletion
+                    if (update_list[k].checksum_match && !force_update) return;
+                    if (!update_list[k].invalid && !force_update) return;
+                }
+                wtvdl.delete(update_list[k].file.replace(diskmap_data.base, ""), diskmap_group_data);
+            });
+
+            Object.keys(update_list).forEach(function (k) {
+                if (update_list[k].checksum_match && !force_update) return;
+                if (!update_list[k].invalid && !force_update) return;
+                if (update_list[k].display) wtvdl.display(update_list[k].display);
+                switch (update_list[k].action) {
+                    case "PUT":
+                        wtvdl.put(update_list[k].file.replace(diskmap_data.base, ""), service_name + ":/" + update_list[k].location, update_list[k].display);
+                        break;
+
+                    case "GET":
+                        wtvdl.get(update_list[k].file.replace(diskmap_data.base, ""), update_list[k].file, service_name + ":/" + update_list[k].location, diskmap_group_data, update_list[k].checksum)
+                        break;
+                }
+            });
+
+            if (!diskmap_data.nogroup) {
+                wtvdl.createGroup(diskmap_group_data, diskmap_data.base, "invalid", (diskmap_data.service_owned || false));
+
+
+                // this rename loop is a part of the group system
+                Object.keys(update_list).forEach(function (k) {
+                    if (update_list[k].checksum_match && !force_update) return;
+                    if (!update_list[k].invalid && !force_update) return;
+                    wtvdl.rename(update_list[k].file.replace(diskmap_data.base, ""), update_list[k].file.replace(diskmap_data.base, ""), diskmap_group_data, diskmap_group_data);
+                });
+
+                wtvdl.setGroup(diskmap_group_data, 'ok', diskmap_data.version);
+            }
+
         }
 
-        download_list += "CREATE-GROUP " + diskmap_group_data + "-UPDATE\n";
+        if (diskmap_data.execute && diskmap_data.execute_when) {
-        download_list += "state: invalid\n";
+            if (diskmap_data.execute_when.toLowerCase().match(/end/)) {
-        download_list += "base: " + diskmap_data.base + ".GROUP-UPDATE/\n\n";
+                wtvdl.execute(diskmap_data.execute);
-
+            }
-        download_list += "CREATE-GROUP " + diskmap_group_data + "\n";
-        download_list += "state: invalid\n";
-        download_list += "service-owned: " + (diskmap_data.service_owned || false) + "\n";
-        download_list += "base: " + diskmap_data.base + "\n\n";
-
-        Object.keys(update_list).forEach(function (k) {
-            if (update_list[k].checksum_match && !force_update) return;
-            if (!update_list[k].invalid && !force_update) return;
-            download_list += "DELETE " + update_list[k].file.replace(diskmap_data.base, "") + "\n";
-            download_list += "group: " + diskmap_group_data + "\n\n";
-        });
-
-        Object.keys(update_list).forEach(function (k) {
-            if (update_list[k].checksum_match && !force_update) return;
-            if (!update_list[k].invalid && !force_update) return;
-            download_list += "DISPLAY " + update_list[k].display + "\n\n";
-            download_list += "GET " + update_list[k].file.replace(diskmap_data.base, "") + "\n";
-            download_list += "group: " + diskmap_group_data + "-UPDATE\n";
-            download_list += "location: " + service_name + ":/" + update_list[k].location + "\n";
-            download_list += "file-permission: r\n"
-            download_list += "wtv-checksum: " + update_list[k].checksum + "\n";
-            download_list += "service-source-location: /webtv/content/" + service_name.replace("wtv-", "") + "d/" + update_list[k].location + "\n";
-            download_list += "client-dest-location: " + update_list[k].file + "\n\n";
-        });
-
-        download_list += "CREATE-GROUP " + diskmap_group_data + "\n";
-        download_list += "state: invalid\n";
-        download_list += "service-owned: " + (diskmap_data.service_owned || false) + "\n";
-        download_list += "base: " + diskmap_data.base + "\n\n";
-
-
-        Object.keys(update_list).forEach(function (k) {
-            if (update_list[k].checksum_match && !force_update) return;
-            if (!update_list[k].invalid && !force_update) return;
-            download_list += "RENAME " + update_list[k].file.replace(diskmap_data.base, "") + "\n";
-            download_list += "group: " + diskmap_group_data + "-UPDATE\n";
-            download_list += "destination-group: " + diskmap_group_data + "\n";
-            download_list += "location: " + update_list[k].file.replace(diskmap_data.base, "") + "\n\n";
-        });
-
-        download_list += "SET-GROUP " + diskmap_group_data + "\n";
-        download_list += "state: ok\n";
-        download_list += "version: " + diskmap_data.version + "\n";
-        download_list += "last-checkup-time: " + new Date().toUTCString().replace("GMT", "+0000") + "\n\n";
-
-        if (diskmap_data.execute && diskmap_data.execute_when == "atEnd") {
-            download_list += "EXECUTE " + diskmap_data.execute + "\n\n";
         }
 
-        download_list += "DELETE-GROUP " + diskmap_group_data + "-UPDATE\n\n";
+        if (files_to_send > 0) {
-        download_list += "DELETE " + diskmap_data.base + ".GROUP-UPDATE/\n\n";
+            if (!diskmap_data.nogroup) {
+                wtvdl.deleteGroupUpdate(diskmap_group_data, diskmap_data.base);
+            }
+        }
+        var download_list = wtvdl.getDownloadList();
+        console.log(download_list);
         return download_list;
     }
 
     function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgroup = null) {
         // parse webtv post
         var output_data = '';
-        var post_data = request_headers.post_data.toString(CryptoJS.enc.Latin1).split("\n");
+        var post_data = new Array();
+        if (request_headers.post_data) post_data = request_headers.post_data.toString(CryptoJS.enc.Latin1).split("\n");
         var post_data_current_directory = '';
         var post_data_current_file = false;
         var post_data_current_group = '';
@@ -181,9 +199,10 @@ if (request_headers['wtv-request-type'] == 'download') {
             diskmap_group_data.files[k].last_modified = (new Date(new Date(post_match_file_lstat.mtime).toUTCString()) / 1000);
             diskmap_group_data.files[k].content_length = post_match_file_lstat.size;
             diskmap_group_data.files[k].checksum = CryptoJS.MD5(CryptoJS.lib.WordArray.create(post_match_file_data)).toString(CryptoJS.enc.Hex).toLowerCase();
+            diskmap_group_data.files[k].action = (diskmap_group_data.files[k].action) ? diskmap_group_data.files[k].action.toUpperCase() : "GET";
 
             if (parseInt(diskmap_group_data.files[k].last_modified) > newest_file_epoch) newest_file_epoch = parseInt(diskmap_group_data.files[k].last_modified);
-            if (!diskmap_group_data.files[k].display) diskmap_group_data.files[k].display = diskmap_group_data.display;
+            //if (!diskmap_group_data.files[k].display) diskmap_group_data.files[k].display = diskmap_group_data.display;
 
             diskmap_group_data.files[k].invalid = true;
             wtv_download_list.push(diskmap_group_data.files[k]);
@@ -205,7 +224,7 @@ if (request_headers['wtv-request-type'] == 'download') {
         return output_data;
     }
 
-    if (request_headers.query.diskmap && request_headers.query.group && request_headers.post_data) {
+    if (request_headers.query.diskmap && request_headers.query.group) {
         var diskmap_json_file = null;
         Object.keys(service_vaults).forEach(function (g) {
             if (diskmap_json_file != null) return;

zefie_wtvp_minisrv/WTVDownloadList.js

@@ -0,0 +1,170 @@
+class WTVDownloadList {
+
+    download_list = "";
+    content_type = "wtv/download-list";
+
+    constructor() {
+        this.clear();
+    }
+
+    /**
+     * Clears the download list
+     */
+    clear() {
+        this.download_list = "";
+    }
+
+    /**
+     * Alias to clear() (clears the download list)
+     */
+    reset() {
+        this.clear();
+    }
+
+    /**
+     * Returns the download list.
+     * @returns {string} Download list for client;
+     */
+    getDownloadList() {
+        return this.download_list;
+    }
+
+    /**
+     * Adds a DISPLAY command to the download list
+     * @param {string} message Message to display to the client
+     */
+    display(message) {
+        this.download_list += "DISPLAY " + message + "\n\n";
+    }
+
+    /**
+     * Adds an EXECUTE command to the download list
+     * @param {string} command client command to execute
+     */
+    execute(command) {
+        this.download_list += "EXECUTE " + command + "\n\n";
+    }
+
+    /**
+     * Adds a CREATE partition command to the download list
+     * @param {string} path file://Disk/ path to desired partition
+     * @param {string} size Size of the desired partition
+     */
+    createPartition(path, size) {
+        this.download_list += "CREATE " + path + "\n";
+        this.download_list += "partition-size: " + size + "\n\n";
+    }
+
+    /**
+     * Adds a CREATE-GROUP command to the download list
+     * @param {string} name Group name
+     * @param {string} path file://Disk/ path of desired group
+     * @param {string} state Group state
+     * @param {boolean|null} service_owned Sets service owned flag. (null = don't set)
+     */
+    createGroup(name, path, state = 'invalid', service_owned = null) {
+        this.download_list += "CREATE-GROUP " + name + "\n";
+        this.download_list += "state: " + state + "\n";
+        if (service_owned !== null) this.download_list += "service-owned: " + service_owned + "\n";
+        this.download_list += "base: " + path + "\n\n";
+    }
+
+    /**
+     * An alias for createGroup() that handles creating the '-UPDATE' group for you
+     * @param {string} name Group name
+     * @param {string} path file://Disk/ path of desired group
+     * @param {string} state Group state
+     * @param {boolean} service_owned Sets service owned flag.
+     */
+    createUpdateGroup(name, path, state = 'invalid', service_owned = false) {
+        this.createGroup(name + "-UPDATE", path, state);
+        this.createGroup(name, path, state, service_owned);
+    }
+
+    /**
+     * Adds a DELETE command to the download list
+     * @param {string} path Non-absolute path of client destination file (relative to group base) if group defined, otherwise absolute file://Disk/ path to delete
+     * @param {string} group Group to which it belongs
+     */
+    delete(path, group = null) {
+        this.download_list += "DELETE " + path + "\n";
+        if (group !== null) this.download_list += "group: " + group + "\n\n";
+    }
+
+    /**
+     * Adds a PUT command to the download list
+     * @param {string} path Absolute file://Disk/ path of a file to upload to the service
+     * @param {string} destination Destination address (wtv url on service) in which to POST upload the file to
+     * @param {string} display Message to display while working on this file
+     */
+    put(path, destination) {
+        this.download_list += "PUT " + path + "\n";
+        this.download_list += "location: " + destination + "\n";
+    }
+
+    /**
+     * Adds a GET command to the download list
+     * @param {string} file Non-absolute path of client destination file (relative to group base)
+     * @param {string} path Absolute file://Disk/ path of destination
+     * @param {string} source wtv-url to fetch file from
+     * @param {string} group Group this file belongs to
+     * @param {string} display Message to display while working on this file
+     * @param {string} checksum md5sum of the file
+     * @param {string} file_permission File permissions
+     */
+    get(file, path, source, group, checksum = null, file_permission = 'r') {
+        this.download_list += "GET " + file + "\n";
+        this.download_list += "group: " + group + "-UPDATE\n";
+        this.download_list += "location: " + source + "\n";
+        this.download_list += "file-permission: " + file_permission + "\n";
+        if (checksum != null) this.download_list += "wtv-checksum: " + checksum + "\n";
+        this.download_list += "service-source-location: /webtv/content/" + source.substr(source.indexOf('-') + 1, source.indexOf(':/') - source.indexOf('-') - 1) + "d/" + source.substr(source.indexOf(':/') + 2) + "\n";
+        this.download_list += "client-dest-location: " + path + "\n\n";
+    }
+
+    /**
+     * Adds a RENAME command to the download list
+     * @param {string} srcfile Non-absolute path of client source file (relative to source group base)
+     * @param {string} destfile Non-absolute path of client destination file (relative to destination group base)
+     * @param {string} srcgroup Source Group
+     * @param {string} destgroup Destination Group
+     */
+    rename(srcfile, destfile, srcgroup, destgroup) {
+        this.download_list += "RENAME " + srcfile + "\n";
+        this.download_list += "group: " + srcgroup + "-UPDATE\n";
+        this.download_list += "destination-group: " + destgroup + "\n";
+        this.download_list += "location: " + destfile + "\n\n";
+    }
+
+    /**
+     * Adds a SET-GROUP command to the download list
+     * @param {any} group Group to set state of
+     * @param {any} state State to set group to
+     * @param {any} version Version to set group to
+     */
+    setGroup(group, state, version) {
+        this.download_list += "SET-GROUP " + group + "\n";
+        this.download_list += "state: " + state + "\n";
+        this.download_list += "version: " + version + "\n";
+        this.download_list += "last-checkup-time: " + new Date().toUTCString().replace("GMT", "+0000") + "\n\n";
+    }
+    /**
+     * Adds a DELETE-GROUP command to the download list
+     * @param {any} group Group to delete
+     */
+    deleteGroup(group) {
+        this.download_list += "DELETE-GROUP " + group + "\n\n";
+    }
+
+    /**
+     * An alias for deleteGroup() that handles deleting the '-UPDATE' group files for you
+     * @param {any} group
+     * @param {any} path
+     */
+    deleteGroupUpdate(group, path) {
+        this.deleteGroup(group + "-UPDATE");
+        this.delete(path + ".GROUP-UPDATE/");
+    }
+}
+
+module.exports = WTVDownloadList;

zefie_wtvp_minisrv/app.js

@@ -720,8 +720,7 @@ function shouldWeCompress(ssid, headers_obj) {
                         // gzip only
                         if (content_type.match(/^audio\/(x-)?[s3m|mod|xm]$/)) compress_data = true; // s3m, mod, xm
                         if (content_type.match(/^audio\/(x-)?[midi|wav|wave]$/)) compress_data = true; // midi & wav
-                        if (content_type.match(/^binary\/x-wtv-approm$/)) compress_data = true; // midi & wav
+                        if (content_type.match(/^binary\/x-wtv-approm$/)) compress_data = true; // approms                        
-                        
                     }
                 }
             }

zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj

@@ -274,6 +274,9 @@
     <Content Include="WTVClientCapabilities.js">
       <SubType>Code</SubType>
     </Content>
+    <Content Include="WTVDownloadList.js">
+      <SubType>Code</SubType>
+    </Content>
     <Content Include="WTVFlashrom.js">
       <SubType>Code</SubType>
     </Content>