updates to disk download system

zefie_wtvp_minisrv/ServiceVault/wtv-disk/delete-group.js

@@ -4,12 +4,13 @@ if (request_headers.query.group) {
     if (request_headers['wtv-request-type']) {
         headers = "200 OK\nContent-Type: " + wtvdl.content_type;
         wtvdl.deleteGroup(request_headers.query.group);
+        wtvdl.deleteGroupUpdate(request_headers.query.group);
         data = wtvdl.getDownloadList();
     }
     var title = "Deleting group"
     var message = title + " " + request_headers.query.group;
     headers = "200 OK\nContent-Type: text/html"
-    data = wtvdl.getSyncPage(title, request_headers.query.group, "delete", message, message, null, "client:goback", "client:goback", "wtv-disk:/delete-group");
+    data = wtvdl.getSyncPage(title, request_headers.query.group, "delete", message, message, null, null, "client:goback", "client:goback", "wtv-disk:/delete-group");
 } else {
     headers = "200 OK\nContent-Type: text/html"
     data = `

zefie_wtvp_minisrv/ServiceVault/wtv-disk/sync.js

@@ -2,11 +2,12 @@ const WTVDownloadList = require("./WTVDownloadList.js");
 var wtvdl = new WTVDownloadList(minisrv_config, service_name);
 
 var force_update = (request_headers.query.force == "true") ? true : false;
+var no_delete = (request_headers.query.dont_delete_files == "true") ? true : false;
 if (request_headers['wtv-request-type'] == 'download') {
     var content_dir = "content/"
     var diskmap_dir = content_dir + "diskmaps/";
 
-    function generateDownloadList(diskmap_group_data, update_list, diskmap_data) {
+    function generateDownloadList(diskmap_group_name, update_list, diskmap_group_data) {
         wtvdl.reset();
         var files_to_send = 0;
         Object.keys(update_list).forEach(function (k) {
@@ -16,77 +17,98 @@ if (request_headers['wtv-request-type'] == 'download') {
         });
 
         // create WebTV Download List
-        if (diskmap_data.execute && diskmap_data.execute_when) {
+        if (diskmap_group_data.execute && diskmap_group_data.execute_when) {
-            if (diskmap_data.execute_when.toLowerCase().match(/start/)) {
+            if (diskmap_group_data.execute_when.toLowerCase().match(/start/)) {
-                wtvdl.execute(diskmap_data.execute);
+                wtvdl.execute(diskmap_group_data.execute);
             }
         }
 
-        if (diskmap_group_data.display) wtvdl.display(diskmap_group_data.display);
+        console.log(diskmap_group_data.client_group_data);
+
+        // delete groups if force, or group is invalid
+        if (diskmap_group_data.client_group_data) {
+            if (force_update || diskmap_group_data.client_group_data.state == "invalid") {
+                wtvdl.deleteGroupUpdate(diskmap_group_data.client_group_data.group, diskmap_group_data.client_group_data.path);
+                wtvdl.deleteGroup(diskmap_group_data.client_group_data.group, diskmap_group_data.client_group_data.path);
+            }
+
+            // delete partition/path if force, and not no_delete
+            if (force_update && !no_delete) {
+                // don't delete Brower partition, ever, but allow deleting of Browser partition subdirs
+                if (!diskmap_group_data.base.match(/disk\/browser(\/)?$/i)) {
+                    if (diskmap_group_data.client_group_data.path.toLowerCase() == diskmap_group_data.base.toLowerCase()) {
+                        wtvdl.delete(diskmap_group_data.base, null);
+                    }
+                }
+            }
+        }
+
+        if (diskmap_group_name.display) wtvdl.display(diskmap_group_name.display);
 
         if (files_to_send > 0) {
 
-            if (diskmap_data.partition_size) {
+            if (diskmap_group_data.partition_size) {
-                wtvdl.createPartition(diskmap_data.base, diskmap_data.partition_size);
+                wtvdl.createPartition(diskmap_group_data.base, diskmap_group_data.partition_size);
             }
 
-            if (!diskmap_data.nogroup) {
+            if (!diskmap_group_data.nogroup) {
                 // only send group commands if group mode is enable
                 // useful to disable for PUT
-                wtvdl.createUpdateGroup(diskmap_group_data, diskmap_data.base, "invalid", (diskmap_data.service_owned || false));
+                wtvdl.createUpdateGroup(diskmap_group_name, diskmap_group_data.base, "invalid", (diskmap_group_data.service_owned || false));
             }
 
             Object.keys(update_list).forEach(function (k) {
                 // file { "action": "delete" }
                 // Useful to purge files we no longer want on the client
-                if (update_list[k].action != "DELETE") {
+                if (update_list[k].action != "DELETE" && update_list[k].action != "DELETEONLY") {
                     // skip deleting valid files if we aren't specifically requesting their deletion
                     if (update_list[k].checksum_match && !force_update) return;
                     if (!update_list[k].invalid && !force_update) return;
                 }
-                wtvdl.delete(update_list[k].file.replace(diskmap_data.base, ""), diskmap_group_data);
+                wtvdl.delete(update_list[k].file.replace(diskmap_group_data.base, ""), diskmap_group_name);
             });
 
             Object.keys(update_list).forEach(function (k) {
                 if (update_list[k].checksum_match && !force_update) return;
                 if (!update_list[k].invalid && !force_update) return;
+                if (update_list[k].action == "DELETEONLY") return;
                 if (update_list[k].display) wtvdl.display(update_list[k].display);
                 switch (update_list[k].action) {
                     case "PUT":
-                        wtvdl.put(update_list[k].file.replace(diskmap_data.base, ""), service_name + ":/" + update_list[k].location, update_list[k].display);
+                        wtvdl.put(update_list[k].file.replace(diskmap_group_data.base, ""), service_name + ":/" + update_list[k].location, update_list[k].display);
                         break;
 
                     case "GET":
-                        wtvdl.get(update_list[k].file.replace(diskmap_data.base, ""), update_list[k].file, service_name + ":/" + update_list[k].location, diskmap_group_data, update_list[k].checksum, update_list[k].uncompressed_size || null, update_list[k].original_filename)
+                        wtvdl.get(update_list[k].file.replace(diskmap_group_data.base, ""), update_list[k].file, service_name + ":/" + update_list[k].location, diskmap_group_name, update_list[k].checksum, update_list[k].uncompressed_size || null, update_list[k].original_filename)
                         break;
                 }
             });
 
-            if (!diskmap_data.nogroup) {
+            if (!diskmap_group_data.nogroup) {
-                wtvdl.createGroup(diskmap_group_data, diskmap_data.base, "invalid", (diskmap_data.service_owned || false));
+                wtvdl.createGroup(diskmap_group_name, diskmap_group_data.base, "invalid", (diskmap_group_data.service_owned || false));
-
 
                 // this rename loop is a part of the group system
                 Object.keys(update_list).forEach(function (k) {
                     if (update_list[k].checksum_match && !force_update) return;
                     if (!update_list[k].invalid && !force_update) return;
-                    wtvdl.rename(update_list[k].file.replace(diskmap_data.base, ""), update_list[k].file.replace(diskmap_data.base, ""), diskmap_group_data, diskmap_group_data);
+                    if (update_list[k].action == "DELETEONLY") return;
+                    wtvdl.rename(update_list[k].file.replace(diskmap_group_data.base, ""), update_list[k].file.replace(diskmap_group_data.base, ""), diskmap_group_name, diskmap_group_name, update_list[k].original_filename);
                 });
 
-                wtvdl.setGroup(diskmap_group_data, 'ok', diskmap_data.version);
+                wtvdl.setGroup(diskmap_group_name, 'ok', diskmap_group_data.version);
             }
 
         }
 
-        if (diskmap_data.execute && diskmap_data.execute_when) {
+        if (diskmap_group_data.execute && diskmap_group_data.execute_when) {
-            if (diskmap_data.execute_when.toLowerCase().match(/end/)) {
+            if (diskmap_group_data.execute_when.toLowerCase().match(/end/)) {
-                wtvdl.execute(diskmap_data.execute);
+                wtvdl.execute(diskmap_group_data.execute);
             }
         }
 
         if (files_to_send > 0) {
-            if (!diskmap_data.nogroup) {
+            if (!diskmap_group_data.nogroup) {
-                wtvdl.deleteGroupUpdate(diskmap_group_data, diskmap_data.base);
+                wtvdl.deleteGroupUpdate(diskmap_group_name, diskmap_group_data.base);
             }
         }
         var download_list = wtvdl.getDownloadList();
@@ -94,11 +116,15 @@ if (request_headers['wtv-request-type'] == 'download') {
         return download_list;
     }
 
-    function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgroup = null) {
+    function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgroup = null, version = 0) {
         // parse webtv post
         var output_data = '';
         var post_data = new Array();
-        if (request_headers.post_data) post_data = request_headers.post_data.toString(CryptoJS.enc.Latin1).split("\n");
+        var client_group_data = new Array();
+        if (request_headers.post_data) {
+            post_data = request_headers.post_data.toString(CryptoJS.enc.Latin1).split("\n");
+            client_group_data = wtvdl.getGroupDataFromClientPost(request_headers.post_data.toString(CryptoJS.enc.Latin1));
+        }
         var post_data_current_directory = '';
         var post_data_current_file = false;
         var post_data_current_group = '';
@@ -178,7 +204,7 @@ if (request_headers['wtv-request-type'] == 'download') {
             }
         });
         var wtv_download_list = new Array();
-        var newest_file_epoch = 0;
+        var newest_file_epoch = version;
         Object.keys(diskmap_group_data.files).forEach(function (k) {
             if (!diskmap_group_data.files[k].location) diskmap_group_data.files[k].location = diskmap_group_data.location + diskmap_group_data.files[k].file.replace(diskmap_group_data.base, "");
             var post_match_file = null;
@@ -188,9 +214,7 @@ if (request_headers['wtv-request-type'] == 'download') {
                 if (!fs.existsSync(post_match_file)) post_match_file = null;
             });
 
-            
+                    var post_match_file_lstat = fs.lstatSync(post_match_file);
-
-            var post_match_file_lstat = fs.lstatSync(post_match_file);
             var post_match_file_data = new Buffer.from(fs.readFileSync(post_match_file, {
                 encoding: null,
                 flags: 'r'
@@ -208,7 +232,7 @@ if (request_headers['wtv-request-type'] == 'download') {
                 if (!diskmap_group_data.files[k].dont_extract_filename) {
                     diskmap_group_data.files[k].original_filename = post_match_file_data.toString('utf8', 10, gzip_fn_end);
                 }
-                //diskmap_group_data.files[k].uncompressed_size = gunzipped.byteLength;
+                diskmap_group_data.files[k].uncompressed_size = gunzipped.byteLength;
                 gunzipped = null;
             } else {
                 diskmap_group_data.files[k].checksum = CryptoJS.MD5(CryptoJS.lib.WordArray.create(post_match_file_data)).toString(CryptoJS.enc.Hex).toLowerCase();
@@ -232,6 +256,7 @@ if (request_headers['wtv-request-type'] == 'download') {
             });
         });
         var diskmap_group_name = (diskmap_subgroup == null) ? diskmap_primary_group : diskmap_primary_group + "-" + diskmap_subgroup;
+        diskmap_group_data.client_group_data = client_group_data[diskmap_group_name] || null;
         output_data = generateDownloadList(diskmap_group_name, wtv_download_list, diskmap_group_data);
         return output_data;
     }
@@ -245,9 +270,10 @@ if (request_headers['wtv-request-type'] == 'download') {
         });
 
         if (diskmap_json_file != null) {
-            if (fs.lstatSync(diskmap_json_file)) {
+            if (fs.existsSync(diskmap_json_file)) {
                 try {
                     // read diskmap
+                    var json_stats = fs.lstatSync(diskmap_json_file);
                     var diskmap_data = JSON.parse(fs.readFileSync(diskmap_json_file).toString());
                     if (!diskmap_data[request_headers.query.group]) {
                         throw ("Invalid diskmap data (group does not match)");
@@ -256,10 +282,15 @@ if (request_headers['wtv-request-type'] == 'download') {
                     diskmap_data = diskmap_data[request_headers.query.group];
                     if (!diskmap_data.location) {
                         Object.keys(diskmap_data).forEach(function (k) {
-                            if (diskmap_data[k]) data += processGroup(request_headers.query.group, diskmap_data[k], k);
+                            console.log(diskmap_data[k]);
+                            if (diskmap_data[k]) {
+                                diskmap_data[k].version = (new Date(new Date(json_stats.mtime).toUTCString()) / 1000);
+                                data += processGroup(request_headers.query.group, diskmap_data[k], k, diskmap_data.version);
+                            }
                         });
                     } else {
-                        data = processGroup(request_headers.query.group, diskmap_data);
+                        diskmap_data.version = (new Date(new Date(json_stats.mtime).toUTCString()) / 1000);
+                        data = processGroup(request_headers.query.group, diskmap_data, null, diskmap_data.version);
                     }
 
                     headers = "200 OK\nContent-Type: wtv/download-list";
@@ -286,5 +317,5 @@ if (request_headers['wtv-request-type'] == 'download') {
     var message = request_headers.query.message || "Retrieving files...";
     var main_message = request_headers.query.main_message || "Your receiver is downloading files.";
     headers = "200 OK\nwtv-connection-close: close\nConnection: close\nContent-Type: text/html";
-    data = wtvdl.getSyncPage(message, request_headers.query.group, request_headers.query.diskmap, main_message, message, force_update)
+    data = wtvdl.getSyncPage(message, request_headers.query.group, request_headers.query.diskmap, main_message, message, force_update, no_delete);
 }

zefie_wtvp_minisrv/WTVDownloadList.js

@@ -101,12 +101,14 @@ class WTVDownloadList {
     /**
      * Adds a DELETE command to the download list
      * @param {string} path Non-absolute path of client destination file (relative to group base) if group defined, otherwise absolute file://Disk/ path to delete
-     * @param {string} group Group to which it belongs
+     * @param {string|null} group Group to which it belongs
+     * @param {string|null} original_filename Use this filename (useful if WTV GZ)
      */
-    delete(path, group = null) {
+    delete(path, group = null, original_filename = null) {
-        path = this.wtvshared.stripGzipFromPath(path);
+        path = this.checkOriginalName(path, original_filename);
         this.download_list += "DELETE " + path + "\n";
-        if (group !== null) this.download_list += "group: " + group + "\n\n";
+        if (group) this.download_list += "group: " + group + "\n\n";
+        else (this.download_list) += "\n";
     }
 
     /**
@@ -142,8 +144,9 @@ class WTVDownloadList {
      * @param {string} path Absolute file://Disk/ path of destination
      * @param {string} source wtv-url to fetch file from
      * @param {string} group Group this file belongs to
-     * @param {string} display Message to display while working on this file
+     * @param {string|null} checksum md5sum of the file
-     * @param {string} checksum md5sum of the file
+     * @param {string|null} uncompressed_size Uncompressed size of gzip file
+     * @param {string|null} original_filename Use this filename (useful if WTV GZ)
      * @param {string} file_permission File permissions
      */
     get(file, path, source, group, checksum = null, uncompressed_size = null, original_filename = null, file_permission = 'r') {
@@ -166,16 +169,54 @@ class WTVDownloadList {
         this.download_list += "client-dest-location: " + path + "\n\n";
     }
 
+    /**
+     * Helper function for WTV GZIP, if original_name is set, use this filename instead of the name in the path
+     * @param {string} path
+     * @param {string} original_name
+     * @returns {string} Path, with filename replaces by original_name, or just path if original_name = null
+     */
+    checkOriginalName(path, original_name) {
+        if (original_name) {
+            var tmp = this.wtvshared.getFilePath(path);
+            if (tmp.length > 0) return tmp + "/" + original_name;
+            return original_name
+        } else return path;
+    }
+
+    getGroupDataFromClientPost(post_data) {
+        if (typeof post_data == 'string') post_data = post_data.split("\n\n");
+        var group_data = [];
+        var i = 0;
+        post_data.forEach(function (v) {
+            if (v.substr(0, 4) == "file") {
+                var block_split = v.split("\n");
+                var group_data_entry = {};
+                group_data_entry.path = block_split[0];
+                block_split.forEach(function (block_section) {
+                    if (block_section.indexOf(": ") > 0) {
+                        var block_section_split = block_section.split(": ");
+                        group_data_entry[block_section_split[0]] = block_section_split[1];
+                    }
+                });
+                group_data[group_data_entry.group] = group_data_entry;
+            }
+        });
+        return group_data;
+    }
+
     /**
      * Adds a RENAME command to the download list
      * @param {string} srcfile Non-absolute path of client source file (relative to source group base)
      * @param {string} destfile Non-absolute path of client destination file (relative to destination group base)
      * @param {string} srcgroup Source Group
      * @param {string} destgroup Destination Group
+     * @param {string} original_filename Use this filename (useful if WTV GZ)
      */
-    rename(srcfile, destfile, srcgroup, destgroup) {
+    rename(srcfile, destfile, srcgroup, destgroup, original_filename = null) {
-        srcfile = this.wtvshared.stripGzipFromPath(srcfile);
+        if (original_filename) {
-        destfile = this.wtvshared.stripGzipFromPath(destfile);
+            srcfile = this.checkOriginalName(srcfile, original_filename);
+            destfile = this.checkOriginalName(srcfile, original_filename);
+        }
         this.download_list += "RENAME " + srcfile + "\n";
         this.download_list += "group: " + srcgroup + "-UPDATE\n";
         this.download_list += "destination-group: " + destgroup + "\n";
@@ -227,7 +268,7 @@ class WTVDownloadList {
     * @param {string|null} url Use your own URL for client:fetch?source= instead of our generated one
     * @returns {string} HTML Download Page
     */
-    getSyncPage(title, group, diskmap = null, main_message = null, message = null, force_update = null, success_url = null, fail_url = null, url = null) {
+    getSyncPage(title, group, diskmap = null, main_message = null, message = null, force_update = null, dont_delete_files = null, success_url = null, fail_url = null, url = null) {
         // Begin Set defaults
         if (main_message === null) main_message = "Your receiver is downloading files.";
 
@@ -235,7 +276,10 @@ class WTVDownloadList {
 
         if (force_update === null) force_update = false;
 
-        if (url === null) url = this.service_name + ":/sync?diskmap=" + escape(diskmap) + "&force=" + force_update;
+        if (url === null) url = this.service_name + ":/sync?diskmap=" + escape(diskmap);
+
+        if (force_update) url += "&force=" + force_update;
+        if (dont_delete_files) url += "&dont_delete_files=" + dont_delete_files;
 
         if (success_url === null) success_url = new this.clientShowAlert({
             'image': this.minisrv_config.config.service_logo,

zefie_wtvp_minisrv/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "zefie_wtvp_minisrv",
-  "version": "0.9.11",
+  "version": "0.9.18",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj

@@ -32,6 +32,9 @@
     <Content Include=".gitignore" />
     <Content Include="app.js" />
     <Content Include="config.json" />
+    <Content Include="diskmap_gen.js">
+      <SubType>Code</SubType>
+    </Content>
     <Content Include="ServiceVault\http_pc\get.js" />
     <Content Include="ServiceVault\http_pc\index.js" />
     <Content Include="ServiceVault\wtv-1800\noflash.js" />