zefie/minisrv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move download list stuff to WTVDownloadList class

Browse Source 
- easier to create your own download list in JS
- possible future client upload ability via PUT
This commit is contained in:
zefie
2021-08-10 13:31:29 -04:00
parent e53de05d76
commit 83f44d7867
4 changed files with 258 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
zefie_wtvp_minisrv/ServiceVault/wtv-disk/sync.js
View File

@@ -2,87 +2,105 @@
var force_update = (request_headers.query.force == "true") ? true : false;
if (request_headers['wtv-request-type'] == 'download') {
var path = require("path");
const WTVDownloadList = require("./WTVDownloadList.js");
var wtvdl = new WTVDownloadList();
var content_dir = "content/"
var diskmap_dir = content_dir + "diskmaps/";
function generateDownloadList(diskmap_group_data, update_list, diskmap_data) {
wtvdl.reset();
var files_to_send = 0;
Object.keys(update_list).forEach(function (k) {
if (update_list[k].checksum_match && !force_update) return;
if (!update_list[k].invalid && !force_update) return;
files_to_send++;
});
// create WebTV Download List
var newest_file_epoch = 0;
var download_list = '';
if (diskmap_data.execute && diskmap_data.execute_when == "atStart") {
download_list += "EXECUTE " + diskmap_data.execute + "\n\n";
if (diskmap_data.execute && diskmap_data.execute_when) {
if (diskmap_data.execute_when.toLowerCase().match(/start/)) {
wtvdl.execute(diskmap_data.execute);
}
}
if (diskmap_data.partition_size) {
download_list += "CREATE " + diskmap_data.base + "\n";
download_list += "partition-size: " + diskmap_data.partition_size + "\n\n";
if (diskmap_group_data.display) wtvdl.display(diskmap_group_data.display);
if (files_to_send > 0) {
if (diskmap_data.partition_size) {
wtvdl.createPartition(diskmap_data.base, diskmap_data.partition_size);
}
if (!diskmap_data.nogroup) {
// only send group commands if group mode is enable
// useful to disable for PUT
wtvdl.createUpdateGroup(diskmap_group_data, diskmap_data.base, "invalid", (diskmap_data.service_owned || false));
}
Object.keys(update_list).forEach(function (k) {
// file { "action": "delete" }
// Useful to purge files we no longer want on the client
if (update_list[k].action != "DELETE") {
// skip deleting valid files if we aren't specifically requesting their deletion
if (update_list[k].checksum_match && !force_update) return;
if (!update_list[k].invalid && !force_update) return;
}
wtvdl.delete(update_list[k].file.replace(diskmap_data.base, ""), diskmap_group_data);
});
Object.keys(update_list).forEach(function (k) {
if (update_list[k].checksum_match && !force_update) return;
if (!update_list[k].invalid && !force_update) return;
if (update_list[k].display) wtvdl.display(update_list[k].display);
switch (update_list[k].action) {
case "PUT":
wtvdl.put(update_list[k].file.replace(diskmap_data.base, ""), service_name + ":/" + update_list[k].location, update_list[k].display);
break;
case "GET":
wtvdl.get(update_list[k].file.replace(diskmap_data.base, ""), update_list[k].file, service_name + ":/" + update_list[k].location, diskmap_group_data, update_list[k].checksum)
break;
}
});
if (!diskmap_data.nogroup) {
wtvdl.createGroup(diskmap_group_data, diskmap_data.base, "invalid", (diskmap_data.service_owned || false));
// this rename loop is a part of the group system
Object.keys(update_list).forEach(function (k) {
if (update_list[k].checksum_match && !force_update) return;
if (!update_list[k].invalid && !force_update) return;
wtvdl.rename(update_list[k].file.replace(diskmap_data.base, ""), update_list[k].file.replace(diskmap_data.base, ""), diskmap_group_data, diskmap_group_data);
});
wtvdl.setGroup(diskmap_group_data, 'ok', diskmap_data.version);
}
}
download_list += "CREATE-GROUP " + diskmap_group_data + "-UPDATE\n";
download_list += "state: invalid\n";
download_list += "base: " + diskmap_data.base + ".GROUP-UPDATE/\n\n";
download_list += "CREATE-GROUP " + diskmap_group_data + "\n";
download_list += "state: invalid\n";
download_list += "service-owned: " + (diskmap_data.service_owned || false) + "\n";
download_list += "base: " + diskmap_data.base + "\n\n";
Object.keys(update_list).forEach(function (k) {
if (update_list[k].checksum_match && !force_update) return;
if (!update_list[k].invalid && !force_update) return;
download_list += "DELETE " + update_list[k].file.replace(diskmap_data.base, "") + "\n";
download_list += "group: " + diskmap_group_data + "\n\n";
});
Object.keys(update_list).forEach(function (k) {
if (update_list[k].checksum_match && !force_update) return;
if (!update_list[k].invalid && !force_update) return;
download_list += "DISPLAY " + update_list[k].display + "\n\n";
download_list += "GET " + update_list[k].file.replace(diskmap_data.base, "") + "\n";
download_list += "group: " + diskmap_group_data + "-UPDATE\n";
download_list += "location: " + service_name + ":/" + update_list[k].location + "\n";
download_list += "file-permission: r\n"
download_list += "wtv-checksum: " + update_list[k].checksum + "\n";
download_list += "service-source-location: /webtv/content/" + service_name.replace("wtv-", "") + "d/" + update_list[k].location + "\n";
download_list += "client-dest-location: " + update_list[k].file + "\n\n";
});
download_list += "CREATE-GROUP " + diskmap_group_data + "\n";
download_list += "state: invalid\n";
download_list += "service-owned: " + (diskmap_data.service_owned || false) + "\n";
download_list += "base: " + diskmap_data.base + "\n\n";
Object.keys(update_list).forEach(function (k) {
if (update_list[k].checksum_match && !force_update) return;
if (!update_list[k].invalid && !force_update) return;
download_list += "RENAME " + update_list[k].file.replace(diskmap_data.base, "") + "\n";
download_list += "group: " + diskmap_group_data + "-UPDATE\n";
download_list += "destination-group: " + diskmap_group_data + "\n";
download_list += "location: " + update_list[k].file.replace(diskmap_data.base, "") + "\n\n";
});
download_list += "SET-GROUP " + diskmap_group_data + "\n";
download_list += "state: ok\n";
download_list += "version: " + diskmap_data.version + "\n";
download_list += "last-checkup-time: " + new Date().toUTCString().replace("GMT", "+0000") + "\n\n";
if (diskmap_data.execute && diskmap_data.execute_when == "atEnd") {
download_list += "EXECUTE " + diskmap_data.execute + "\n\n";
if (diskmap_data.execute && diskmap_data.execute_when) {
if (diskmap_data.execute_when.toLowerCase().match(/end/)) {
wtvdl.execute(diskmap_data.execute);
}
}
download_list += "DELETE-GROUP " + diskmap_group_data + "-UPDATE\n\n";
download_list += "DELETE " + diskmap_data.base + ".GROUP-UPDATE/\n\n";
if (files_to_send > 0) {
if (!diskmap_data.nogroup) {
wtvdl.deleteGroupUpdate(diskmap_group_data, diskmap_data.base);
}
}
var download_list = wtvdl.getDownloadList();
console.log(download_list);
return download_list;
}
function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgroup = null) {
// parse webtv post
var output_data = '';
var post_data = request_headers.post_data.toString(CryptoJS.enc.Latin1).split("\n");
var post_data = new Array();
if (request_headers.post_data) post_data = request_headers.post_data.toString(CryptoJS.enc.Latin1).split("\n");
var post_data_current_directory = '';
var post_data_current_file = false;
var post_data_current_group = '';
@@ -181,9 +199,10 @@ if (request_headers['wtv-request-type'] == 'download') {
diskmap_group_data.files[k].last_modified = (new Date(new Date(post_match_file_lstat.mtime).toUTCString()) / 1000);
diskmap_group_data.files[k].content_length = post_match_file_lstat.size;
diskmap_group_data.files[k].checksum = CryptoJS.MD5(CryptoJS.lib.WordArray.create(post_match_file_data)).toString(CryptoJS.enc.Hex).toLowerCase();
diskmap_group_data.files[k].action = (diskmap_group_data.files[k].action) ? diskmap_group_data.files[k].action.toUpperCase() : "GET";
if (parseInt(diskmap_group_data.files[k].last_modified) > newest_file_epoch) newest_file_epoch = parseInt(diskmap_group_data.files[k].last_modified);
if (!diskmap_group_data.files[k].display) diskmap_group_data.files[k].display = diskmap_group_data.display;
//if (!diskmap_group_data.files[k].display) diskmap_group_data.files[k].display = diskmap_group_data.display;
diskmap_group_data.files[k].invalid = true;
wtv_download_list.push(diskmap_group_data.files[k]);
@@ -205,7 +224,7 @@ if (request_headers['wtv-request-type'] == 'download') {
return output_data;
}
if (request_headers.query.diskmap && request_headers.query.group && request_headers.post_data) {
if (request_headers.query.diskmap && request_headers.query.group) {
var diskmap_json_file = null;
Object.keys(service_vaults).forEach(function (g) {
if (diskmap_json_file != null) return;

170
zefie_wtvp_minisrv/WTVDownloadList.js Normal file
View File

@@ -0,0 +1,170 @@
class WTVDownloadList {
download_list = "";
content_type = "wtv/download-list";
constructor() {
this.clear();
}
/**
* Clears the download list
*/
clear() {
this.download_list = "";
}
/**
* Alias to clear() (clears the download list)
*/
reset() {
this.clear();
}
/**
* Returns the download list.
* @returns {string} Download list for client;
*/
getDownloadList() {
return this.download_list;
}
/**
* Adds a DISPLAY command to the download list
* @param {string} message Message to display to the client
*/
display(message) {
this.download_list += "DISPLAY " + message + "\n\n";
}
/**
* Adds an EXECUTE command to the download list
* @param {string} command client command to execute
*/
execute(command) {
this.download_list += "EXECUTE " + command + "\n\n";
}
/**
* Adds a CREATE partition command to the download list
* @param {string} path file://Disk/ path to desired partition
* @param {string} size Size of the desired partition
*/
createPartition(path, size) {
this.download_list += "CREATE " + path + "\n";
this.download_list += "partition-size: " + size + "\n\n";
}
/**
* Adds a CREATE-GROUP command to the download list
* @param {string} name Group name
* @param {string} path file://Disk/ path of desired group
* @param {string} state Group state
* @param {boolean|null} service_owned Sets service owned flag. (null = don't set)
*/
createGroup(name, path, state = 'invalid', service_owned = null) {
this.download_list += "CREATE-GROUP " + name + "\n";
this.download_list += "state: " + state + "\n";
if (service_owned !== null) this.download_list += "service-owned: " + service_owned + "\n";
this.download_list += "base: " + path + "\n\n";
}
/**
* An alias for createGroup() that handles creating the '-UPDATE' group for you
* @param {string} name Group name
* @param {string} path file://Disk/ path of desired group
* @param {string} state Group state
* @param {boolean} service_owned Sets service owned flag.
*/
createUpdateGroup(name, path, state = 'invalid', service_owned = false) {
this.createGroup(name + "-UPDATE", path, state);
this.createGroup(name, path, state, service_owned);
}
/**
* Adds a DELETE command to the download list
* @param {string} path Non-absolute path of client destination file (relative to group base) if group defined, otherwise absolute file://Disk/ path to delete
* @param {string} group Group to which it belongs
*/
delete(path, group = null) {
this.download_list += "DELETE " + path + "\n";
if (group !== null) this.download_list += "group: " + group + "\n\n";
}
/**
* Adds a PUT command to the download list
* @param {string} path Absolute file://Disk/ path of a file to upload to the service
* @param {string} destination Destination address (wtv url on service) in which to POST upload the file to
* @param {string} display Message to display while working on this file
*/
put(path, destination) {
this.download_list += "PUT " + path + "\n";
this.download_list += "location: " + destination + "\n";
}
/**
* Adds a GET command to the download list
* @param {string} file Non-absolute path of client destination file (relative to group base)
* @param {string} path Absolute file://Disk/ path of destination
* @param {string} source wtv-url to fetch file from
* @param {string} group Group this file belongs to
* @param {string} display Message to display while working on this file
* @param {string} checksum md5sum of the file
* @param {string} file_permission File permissions
*/
get(file, path, source, group, checksum = null, file_permission = 'r') {
this.download_list += "GET " + file + "\n";
this.download_list += "group: " + group + "-UPDATE\n";
this.download_list += "location: " + source + "\n";
this.download_list += "file-permission: " + file_permission + "\n";
if (checksum != null) this.download_list += "wtv-checksum: " + checksum + "\n";
this.download_list += "service-source-location: /webtv/content/" + source.substr(source.indexOf('-') + 1, source.indexOf(':/') - source.indexOf('-') - 1) + "d/" + source.substr(source.indexOf(':/') + 2) + "\n";
this.download_list += "client-dest-location: " + path + "\n\n";
}
/**
* Adds a RENAME command to the download list
* @param {string} srcfile Non-absolute path of client source file (relative to source group base)
* @param {string} destfile Non-absolute path of client destination file (relative to destination group base)
* @param {string} srcgroup Source Group
* @param {string} destgroup Destination Group
*/
rename(srcfile, destfile, srcgroup, destgroup) {
this.download_list += "RENAME " + srcfile + "\n";
this.download_list += "group: " + srcgroup + "-UPDATE\n";
this.download_list += "destination-group: " + destgroup + "\n";
this.download_list += "location: " + destfile + "\n\n";
}
/**
* Adds a SET-GROUP command to the download list
* @param {any} group Group to set state of
* @param {any} state State to set group to
* @param {any} version Version to set group to
*/
setGroup(group, state, version) {
this.download_list += "SET-GROUP " + group + "\n";
this.download_list += "state: " + state + "\n";
this.download_list += "version: " + version + "\n";
this.download_list += "last-checkup-time: " + new Date().toUTCString().replace("GMT", "+0000") + "\n\n";
}
/**
* Adds a DELETE-GROUP command to the download list
* @param {any} group Group to delete
*/
deleteGroup(group) {
this.download_list += "DELETE-GROUP " + group + "\n\n";
}
/**
* An alias for deleteGroup() that handles deleting the '-UPDATE' group files for you
* @param {any} group
* @param {any} path
*/
deleteGroupUpdate(group, path) {
this.deleteGroup(group + "-UPDATE");
this.delete(path + ".GROUP-UPDATE/");
}
}
module.exports = WTVDownloadList;

3
zefie_wtvp_minisrv/app.js
View File

@@ -720,8 +720,7 @@ function shouldWeCompress(ssid, headers_obj) {
// gzip only
if (content_type.match(/^audio\/(x-)?[s3m|mod|xm]$/)) compress_data = true; // s3m, mod, xm
if (content_type.match(/^audio\/(x-)?[midi|wav|wave]$/)) compress_data = true; // midi & wav
if (content_type.match(/^binary\/x-wtv-approm$/)) compress_data = true; // midi & wav
if (content_type.match(/^binary\/x-wtv-approm$/)) compress_data = true; // approms
}
}
}

3
zefie_wtvp_minisrv/zefie_wtvp_minisrv.njsproj
View File

@@ -274,6 +274,9 @@
<Content Include="WTVClientCapabilities.js">
<SubType>Code</SubType>
</Content>
<Content Include="WTVDownloadList.js">
<SubType>Code</SubType>
</Content>
<Content Include="WTVFlashrom.js">
<SubType>Code</SubType>
</Content>