zefie/minisrv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updates to disk download system

Browse Source
This commit is contained in:
zefie
2021-08-12 16:16:03 -04:00
parent 3133a83197
commit 368eb58838
5 changed files with 127 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
zefie_wtvp_minisrv/ServiceVault/wtv-disk/delete-group.js
View File

@@ -4,12 +4,13 @@ if (request_headers.query.group) {
if (request_headers['wtv-request-type']) {
headers = "200 OK\nContent-Type: " + wtvdl.content_type;
wtvdl.deleteGroup(request_headers.query.group);
wtvdl.deleteGroupUpdate(request_headers.query.group);
data = wtvdl.getDownloadList();
}
var title = "Deleting group"
var message = title + " " + request_headers.query.group;
headers = "200 OK\nContent-Type: text/html"
data = wtvdl.getSyncPage(title, request_headers.query.group, "delete", message, message, null, "client:goback", "client:goback", "wtv-disk:/delete-group");
data = wtvdl.getSyncPage(title, request_headers.query.group, "delete", message, message, null, null, "client:goback", "client:goback", "wtv-disk:/delete-group");
} else {
headers = "200 OK\nContent-Type: text/html"
data = `

99
zefie_wtvp_minisrv/ServiceVault/wtv-disk/sync.js
View File

@@ -2,11 +2,12 @@ const WTVDownloadList = require("./WTVDownloadList.js");
var wtvdl = new WTVDownloadList(minisrv_config, service_name);
var force_update = (request_headers.query.force == "true") ? true : false;
var no_delete = (request_headers.query.dont_delete_files == "true") ? true : false;
if (request_headers['wtv-request-type'] == 'download') {
var content_dir = "content/"
var diskmap_dir = content_dir + "diskmaps/";
function generateDownloadList(diskmap_group_data, update_list, diskmap_data) {
function generateDownloadList(diskmap_group_name, update_list, diskmap_group_data) {
wtvdl.reset();
var files_to_send = 0;
Object.keys(update_list).forEach(function (k) {
@@ -16,77 +17,98 @@ if (request_headers['wtv-request-type'] == 'download') {
});
// create WebTV Download List
if (diskmap_data.execute && diskmap_data.execute_when) {
if (diskmap_data.execute_when.toLowerCase().match(/start/)) {
wtvdl.execute(diskmap_data.execute);
if (diskmap_group_data.execute && diskmap_group_data.execute_when) {
if (diskmap_group_data.execute_when.toLowerCase().match(/start/)) {
wtvdl.execute(diskmap_group_data.execute);
}
}
if (diskmap_group_data.display) wtvdl.display(diskmap_group_data.display);
console.log(diskmap_group_data.client_group_data);
// delete groups if force, or group is invalid
if (diskmap_group_data.client_group_data) {
if (force_update || diskmap_group_data.client_group_data.state == "invalid") {
wtvdl.deleteGroupUpdate(diskmap_group_data.client_group_data.group, diskmap_group_data.client_group_data.path);
wtvdl.deleteGroup(diskmap_group_data.client_group_data.group, diskmap_group_data.client_group_data.path);
}
// delete partition/path if force, and not no_delete
if (force_update && !no_delete) {
// don't delete Brower partition, ever, but allow deleting of Browser partition subdirs
if (!diskmap_group_data.base.match(/disk\/browser(\/)?$/i)) {
if (diskmap_group_data.client_group_data.path.toLowerCase() == diskmap_group_data.base.toLowerCase()) {
wtvdl.delete(diskmap_group_data.base, null);
}
}
}
}
if (diskmap_group_name.display) wtvdl.display(diskmap_group_name.display);
if (files_to_send > 0) {
if (diskmap_data.partition_size) {
wtvdl.createPartition(diskmap_data.base, diskmap_data.partition_size);
if (diskmap_group_data.partition_size) {
wtvdl.createPartition(diskmap_group_data.base, diskmap_group_data.partition_size);
}
if (!diskmap_data.nogroup) {
if (!diskmap_group_data.nogroup) {
// only send group commands if group mode is enable
// useful to disable for PUT
wtvdl.createUpdateGroup(diskmap_group_data, diskmap_data.base, "invalid", (diskmap_data.service_owned || false));
wtvdl.createUpdateGroup(diskmap_group_name, diskmap_group_data.base, "invalid", (diskmap_group_data.service_owned || false));
}
Object.keys(update_list).forEach(function (k) {
// file { "action": "delete" }
// Useful to purge files we no longer want on the client
if (update_list[k].action != "DELETE") {
if (update_list[k].action != "DELETE" && update_list[k].action != "DELETEONLY") {
// skip deleting valid files if we aren't specifically requesting their deletion
if (update_list[k].checksum_match && !force_update) return;
if (!update_list[k].invalid && !force_update) return;
}
wtvdl.delete(update_list[k].file.replace(diskmap_data.base, ""), diskmap_group_data);
wtvdl.delete(update_list[k].file.replace(diskmap_group_data.base, ""), diskmap_group_name);
});
Object.keys(update_list).forEach(function (k) {
if (update_list[k].checksum_match && !force_update) return;
if (!update_list[k].invalid && !force_update) return;
if (update_list[k].action == "DELETEONLY") return;
if (update_list[k].display) wtvdl.display(update_list[k].display);
switch (update_list[k].action) {
case "PUT":
wtvdl.put(update_list[k].file.replace(diskmap_data.base, ""), service_name + ":/" + update_list[k].location, update_list[k].display);
wtvdl.put(update_list[k].file.replace(diskmap_group_data.base, ""), service_name + ":/" + update_list[k].location, update_list[k].display);
break;
case "GET":
wtvdl.get(update_list[k].file.replace(diskmap_data.base, ""), update_list[k].file, service_name + ":/" + update_list[k].location, diskmap_group_data, update_list[k].checksum, update_list[k].uncompressed_size || null, update_list[k].original_filename)
wtvdl.get(update_list[k].file.replace(diskmap_group_data.base, ""), update_list[k].file, service_name + ":/" + update_list[k].location, diskmap_group_name, update_list[k].checksum, update_list[k].uncompressed_size || null, update_list[k].original_filename)
break;
}
});
if (!diskmap_data.nogroup) {
wtvdl.createGroup(diskmap_group_data, diskmap_data.base, "invalid", (diskmap_data.service_owned || false));
if (!diskmap_group_data.nogroup) {
wtvdl.createGroup(diskmap_group_name, diskmap_group_data.base, "invalid", (diskmap_group_data.service_owned || false));
// this rename loop is a part of the group system
Object.keys(update_list).forEach(function (k) {
if (update_list[k].checksum_match && !force_update) return;
if (!update_list[k].invalid && !force_update) return;
wtvdl.rename(update_list[k].file.replace(diskmap_data.base, ""), update_list[k].file.replace(diskmap_data.base, ""), diskmap_group_data, diskmap_group_data);
if (update_list[k].action == "DELETEONLY") return;
wtvdl.rename(update_list[k].file.replace(diskmap_group_data.base, ""), update_list[k].file.replace(diskmap_group_data.base, ""), diskmap_group_name, diskmap_group_name, update_list[k].original_filename);
});
wtvdl.setGroup(diskmap_group_data, 'ok', diskmap_data.version);
wtvdl.setGroup(diskmap_group_name, 'ok', diskmap_group_data.version);
}
}
if (diskmap_data.execute && diskmap_data.execute_when) {
if (diskmap_data.execute_when.toLowerCase().match(/end/)) {
wtvdl.execute(diskmap_data.execute);
if (diskmap_group_data.execute && diskmap_group_data.execute_when) {
if (diskmap_group_data.execute_when.toLowerCase().match(/end/)) {
wtvdl.execute(diskmap_group_data.execute);
}
}
if (files_to_send > 0) {
if (!diskmap_data.nogroup) {
wtvdl.deleteGroupUpdate(diskmap_group_data, diskmap_data.base);
if (!diskmap_group_data.nogroup) {
wtvdl.deleteGroupUpdate(diskmap_group_name, diskmap_group_data.base);
}
}
var download_list = wtvdl.getDownloadList();
@@ -94,11 +116,15 @@ if (request_headers['wtv-request-type'] == 'download') {
return download_list;
}
function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgroup = null) {
function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgroup = null, version = 0) {
// parse webtv post
var output_data = '';
var post_data = new Array();
if (request_headers.post_data) post_data = request_headers.post_data.toString(CryptoJS.enc.Latin1).split("\n");
var client_group_data = new Array();
if (request_headers.post_data) {
post_data = request_headers.post_data.toString(CryptoJS.enc.Latin1).split("\n");
client_group_data = wtvdl.getGroupDataFromClientPost(request_headers.post_data.toString(CryptoJS.enc.Latin1));
}
var post_data_current_directory = '';
var post_data_current_file = false;
var post_data_current_group = '';
@@ -178,7 +204,7 @@ if (request_headers['wtv-request-type'] == 'download') {
}
});
var wtv_download_list = new Array();
var newest_file_epoch = 0;
var newest_file_epoch = version;
Object.keys(diskmap_group_data.files).forEach(function (k) {
if (!diskmap_group_data.files[k].location) diskmap_group_data.files[k].location = diskmap_group_data.location + diskmap_group_data.files[k].file.replace(diskmap_group_data.base, "");
var post_match_file = null;
@@ -188,9 +214,7 @@ if (request_headers['wtv-request-type'] == 'download') {
if (!fs.existsSync(post_match_file)) post_match_file = null;
});
var post_match_file_lstat = fs.lstatSync(post_match_file);
var post_match_file_lstat = fs.lstatSync(post_match_file);
var post_match_file_data = new Buffer.from(fs.readFileSync(post_match_file, {
encoding: null,
flags: 'r'
@@ -208,7 +232,7 @@ if (request_headers['wtv-request-type'] == 'download') {
if (!diskmap_group_data.files[k].dont_extract_filename) {
diskmap_group_data.files[k].original_filename = post_match_file_data.toString('utf8', 10, gzip_fn_end);
}
//diskmap_group_data.files[k].uncompressed_size = gunzipped.byteLength;
diskmap_group_data.files[k].uncompressed_size = gunzipped.byteLength;
gunzipped = null;
} else {
diskmap_group_data.files[k].checksum = CryptoJS.MD5(CryptoJS.lib.WordArray.create(post_match_file_data)).toString(CryptoJS.enc.Hex).toLowerCase();
@@ -232,6 +256,7 @@ if (request_headers['wtv-request-type'] == 'download') {
});
});
var diskmap_group_name = (diskmap_subgroup == null) ? diskmap_primary_group : diskmap_primary_group + "-" + diskmap_subgroup;
diskmap_group_data.client_group_data = client_group_data[diskmap_group_name] || null;
output_data = generateDownloadList(diskmap_group_name, wtv_download_list, diskmap_group_data);
return output_data;
}
@@ -245,9 +270,10 @@ if (request_headers['wtv-request-type'] == 'download') {
});
if (diskmap_json_file != null) {
if (fs.lstatSync(diskmap_json_file)) {
if (fs.existsSync(diskmap_json_file)) {
try {
// read diskmap
var json_stats = fs.lstatSync(diskmap_json_file);
var diskmap_data = JSON.parse(fs.readFileSync(diskmap_json_file).toString());
if (!diskmap_data[request_headers.query.group]) {
throw ("Invalid diskmap data (group does not match)");
@@ -256,10 +282,15 @@ if (request_headers['wtv-request-type'] == 'download') {
diskmap_data = diskmap_data[request_headers.query.group];
if (!diskmap_data.location) {
Object.keys(diskmap_data).forEach(function (k) {
if (diskmap_data[k]) data += processGroup(request_headers.query.group, diskmap_data[k], k);
console.log(diskmap_data[k]);
if (diskmap_data[k]) {
diskmap_data[k].version = (new Date(new Date(json_stats.mtime).toUTCString()) / 1000);
data += processGroup(request_headers.query.group, diskmap_data[k], k, diskmap_data.version);
}
});
} else {
data = processGroup(request_headers.query.group, diskmap_data);
diskmap_data.version = (new Date(new Date(json_stats.mtime).toUTCString()) / 1000);
data = processGroup(request_headers.query.group, diskmap_data, null, diskmap_data.version);
}
headers = "200 OK\nContent-Type: wtv/download-list";
@@ -286,5 +317,5 @@ if (request_headers['wtv-request-type'] == 'download') {
var message = request_headers.query.message || "Retrieving files...";
var main_message = request_headers.query.main_message || "Your receiver is downloading files.";
headers = "200 OK\nwtv-connection-close: close\nConnection: close\nContent-Type: text/html";
data = wtvdl.getSyncPage(message, request_headers.query.group, request_headers.query.diskmap, main_message, message, force_update)
data = wtvdl.getSyncPage(message, request_headers.query.group, request_headers.query.diskmap, main_message, message, force_update, no_delete);
}

66
zefie_wtvp_minisrv/WTVDownloadList.js
View File

@@ -101,12 +101,14 @@ class WTVDownloadList {
/**
* Adds a DELETE command to the download list
* @param {string} path Non-absolute path of client destination file (relative to group base) if group defined, otherwise absolute file://Disk/ path to delete
* @param {string} group Group to which it belongs
* @param {string|null} group Group to which it belongs
* @param {string|null} original_filename Use this filename (useful if WTV GZ)
*/
delete(path, group = null) {
path = this.wtvshared.stripGzipFromPath(path);
delete(path, group = null, original_filename = null) {
path = this.checkOriginalName(path, original_filename);
this.download_list += "DELETE " + path + "\n";
if (group !== null) this.download_list += "group: " + group + "\n\n";
if (group) this.download_list += "group: " + group + "\n\n";
else (this.download_list) += "\n";
}
/**
@@ -142,8 +144,9 @@ class WTVDownloadList {
* @param {string} path Absolute file://Disk/ path of destination
* @param {string} source wtv-url to fetch file from
* @param {string} group Group this file belongs to
* @param {string} display Message to display while working on this file
* @param {string} checksum md5sum of the file
* @param {string|null} checksum md5sum of the file
* @param {string|null} uncompressed_size Uncompressed size of gzip file
* @param {string|null} original_filename Use this filename (useful if WTV GZ)
* @param {string} file_permission File permissions
*/
get(file, path, source, group, checksum = null, uncompressed_size = null, original_filename = null, file_permission = 'r') {
@@ -166,16 +169,54 @@ class WTVDownloadList {
this.download_list += "client-dest-location: " + path + "\n\n";
}
/**
* Helper function for WTV GZIP, if original_name is set, use this filename instead of the name in the path
* @param {string} path
* @param {string} original_name
* @returns {string} Path, with filename replaces by original_name, or just path if original_name = null
*/
checkOriginalName(path, original_name) {
if (original_name) {
var tmp = this.wtvshared.getFilePath(path);
if (tmp.length > 0) return tmp + "/" + original_name;
return original_name
} else return path;
}
getGroupDataFromClientPost(post_data) {
if (typeof post_data == 'string') post_data = post_data.split("\n\n");
var group_data = [];
var i = 0;
post_data.forEach(function (v) {
if (v.substr(0, 4) == "file") {
var block_split = v.split("\n");
var group_data_entry = {};
group_data_entry.path = block_split[0];
block_split.forEach(function (block_section) {
if (block_section.indexOf(": ") > 0) {
var block_section_split = block_section.split(": ");
group_data_entry[block_section_split[0]] = block_section_split[1];
}
});
group_data[group_data_entry.group] = group_data_entry;
}
});
return group_data;
}
/**
* Adds a RENAME command to the download list
* @param {string} srcfile Non-absolute path of client source file (relative to source group base)
* @param {string} destfile Non-absolute path of client destination file (relative to destination group base)
* @param {string} srcgroup Source Group
* @param {string} destgroup Destination Group
* @param {string} original_filename Use this filename (useful if WTV GZ)
*/
rename(srcfile, destfile, srcgroup, destgroup) {
srcfile = this.wtvshared.stripGzipFromPath(srcfile);
destfile = this.wtvshared.stripGzipFromPath(destfile);
rename(srcfile, destfile, srcgroup, destgroup, original_filename = null) {
if (original_filename) {
srcfile = this.checkOriginalName(srcfile, original_filename);
destfile = this.checkOriginalName(srcfile, original_filename);
}
this.download_list += "RENAME " + srcfile + "\n";
this.download_list += "group: " + srcgroup + "-UPDATE\n";
this.download_list += "destination-group: " + destgroup + "\n";
@@ -227,7 +268,7 @@ class WTVDownloadList {
* @param {string|null} url Use your own URL for client:fetch?source= instead of our generated one
* @returns {string} HTML Download Page
*/
getSyncPage(title, group, diskmap = null, main_message = null, message = null, force_update = null, success_url = null, fail_url = null, url = null) {
getSyncPage(title, group, diskmap = null, main_message = null, message = null, force_update = null, dont_delete_files = null, success_url = null, fail_url = null, url = null) {
// Begin Set defaults
if (main_message === null) main_message = "Your receiver is downloading files.";
@@ -235,7 +276,10 @@ class WTVDownloadList {
if (force_update === null) force_update = false;
if (url === null) url = this.service_name + ":/sync?diskmap=" + escape(diskmap) + "&force=" + force_update;
if (url === null) url = this.service_name + ":/sync?diskmap=" + escape(diskmap);
if (force_update) url += "&force=" + force_update;
if (dont_delete_files) url += "&dont_delete_files=" + dont_delete_files;
if (success_url === null) success_url = new this.clientShowAlert({
'image': this.minisrv_config.config.service_logo,

2
zefie_wtvp_minisrv/package-lock.json generated
View File

@@ -1,6 +1,6 @@
{
"name": "zefie_wtvp_minisrv",
"version": "0.9.11",
"version": "0.9.18",
"lockfileVersion": 1,
"requires": true,
"dependencies": {

3
zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj
View File

@@ -32,6 +32,9 @@
<Content Include=".gitignore" />
<Content Include="app.js" />
<Content Include="config.json" />
<Content Include="diskmap_gen.js">
<SubType>Code</SubType>
</Content>
<Content Include="ServiceVault\http_pc\get.js" />
<Content Include="ServiceVault\http_pc\index.js" />
<Content Include="ServiceVault\wtv-1800\noflash.js" />