zefie/minisrv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.9.19

Browse Source 
- numerous bug fixes
 - add `npm start` support
 - minibrowser theme thanks to @GraspYonOx
 - remove TOS coz no one cares anyway
This commit is contained in:
zefie
2021-08-12 01:07:04 -04:00
parent b96718555b
commit 72c15305fa
333 changed files with 1588 additions and 1300 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
zefie_wtvp_minisrv/ServiceVault/wtv-disk/sync.js
View File

@@ -2,11 +2,12 @@ const WTVDownloadList = require("./WTVDownloadList.js");
var wtvdl = new WTVDownloadList(minisrv_config, service_name);
var force_update = (request_headers.query.force == "true") ? true : false;
var no_delete = (request_headers.query.dont_delete_files == "true") ? true : false;
if (request_headers['wtv-request-type'] == 'download') {
var content_dir = "content/"
var diskmap_dir = content_dir + "diskmaps/";
function generateDownloadList(diskmap_group_data, update_list, diskmap_data) {
function generateDownloadList(diskmap_group_name, update_list, diskmap_group_data) {
wtvdl.reset();
var files_to_send = 0;
Object.keys(update_list).forEach(function (k) {
@@ -16,77 +17,99 @@ if (request_headers['wtv-request-type'] == 'download') {
});
// create WebTV Download List
if (diskmap_data.execute && diskmap_data.execute_when) {
if (diskmap_data.execute_when.toLowerCase().match(/start/)) {
wtvdl.execute(diskmap_data.execute);
if (diskmap_group_data.execute_start) {
wtvdl.execute(diskmap_group_data.execute_start);
}
console.log(diskmap_group_data.client_group_data);
// delete groups if force, or group is invalid
if (diskmap_group_data.client_group_data) {
if (force_update || diskmap_group_data.client_group_data.state == "invalid") {
wtvdl.deleteGroupUpdate(diskmap_group_data.client_group_data.group, diskmap_group_data.client_group_data.path);
}
// delete partition/path if force, and not no_delete
if (force_update && !no_delete) {
// don't delete Brower partition, ever, but allow deleting of Browser partition subdirs
if (!diskmap_group_data.base.match(/disk\/browser(\/)?$/i)) {
if (diskmap_group_data.client_group_data.path.toLowerCase() == diskmap_group_data.base.toLowerCase()) {
wtvdl.delete(diskmap_group_data.base, null);
} else {
wtvdl.delete(diskmap_group_data.base, diskmap_group_data.client_group_data.group);
}
}
}
}
if (diskmap_group_data.display) wtvdl.display(diskmap_group_data.display);
if (diskmap_group_name.display) wtvdl.display(diskmap_group_name.display);
if (files_to_send > 0) {
if (diskmap_data.partition_size) {
wtvdl.createPartition(diskmap_data.base, diskmap_data.partition_size);
if (diskmap_group_data.partition_size) {
wtvdl.createPartition(diskmap_group_data.base, diskmap_group_data.partition_size);
}
if (!diskmap_data.nogroup) {
if (!diskmap_group_data.nogroup) {
// only send group commands if group mode is enable
// useful to disable for PUT
wtvdl.createUpdateGroup(diskmap_group_data, diskmap_data.base, "invalid", (diskmap_data.service_owned || false));
wtvdl.createUpdateGroup(diskmap_group_name, diskmap_group_data.base, "invalid", (diskmap_group_data.service_owned || false));
}
Object.keys(update_list).forEach(function (k) {
// file { "action": "delete" }
// Useful to purge files we no longer want on the client
if (update_list[k].action != "DELETE") {
if (update_list[k].action != "DELETE" && update_list[k].action != "DELETEONLY") {
// skip deleting valid files if we aren't specifically requesting their deletion
if (update_list[k].checksum_match && !force_update) return;
if (!update_list[k].invalid && !force_update) return;
}
wtvdl.delete(update_list[k].file.replace(diskmap_data.base, ""), diskmap_group_data);
wtvdl.delete(update_list[k].file.replace(diskmap_group_data.base, ""), diskmap_group_name);
});
Object.keys(update_list).forEach(function (k) {
if (update_list[k].checksum_match && !force_update) return;
if (!update_list[k].invalid && !force_update) return;
if (update_list[k].action == "DELETEONLY") return;
if (update_list[k].display) wtvdl.display(update_list[k].display);
switch (update_list[k].action) {
case "PUT":
wtvdl.put(update_list[k].file.replace(diskmap_data.base, ""), service_name + ":/" + update_list[k].location, update_list[k].display);
wtvdl.put(update_list[k].file.replace(diskmap_group_data.base, ""), service_name + ":/" + update_list[k].location, update_list[k].display);
break;
case "GET":
wtvdl.get(update_list[k].file.replace(diskmap_data.base, ""), update_list[k].file, service_name + ":/" + update_list[k].location, diskmap_group_data, update_list[k].checksum, update_list[k].uncompressed_size || null, update_list[k].original_filename)
var get_url = service_name + ":/" + update_list[k].location + "?";
if (update_list[k].compress === false) get_url += "dont_compress=true&";
if (update_list[k].type) get_url += "content_type=" + escape(update_list[k].type) + "&";
get_url = get_url.slice();
wtvdl.get(update_list[k].file.replace(diskmap_group_data.base, ""), update_list[k].file, get_url, diskmap_group_name, update_list[k].checksum, update_list[k].uncompressed_size || null, update_list[k].original_filename)
break;
}
});
if (!diskmap_data.nogroup) {
wtvdl.createGroup(diskmap_group_data, diskmap_data.base, "invalid", (diskmap_data.service_owned || false));
if (!diskmap_group_data.nogroup) {
wtvdl.createGroup(diskmap_group_name, diskmap_group_data.base, "invalid", (diskmap_group_data.service_owned || false));
// this rename loop is a part of the group system
Object.keys(update_list).forEach(function (k) {
if (update_list[k].checksum_match && !force_update) return;
if (!update_list[k].invalid && !force_update) return;
wtvdl.rename(update_list[k].file.replace(diskmap_data.base, ""), update_list[k].file.replace(diskmap_data.base, ""), diskmap_group_data, diskmap_group_data);
if (update_list[k].action == "DELETEONLY") return;
wtvdl.rename(update_list[k].file.replace(diskmap_group_data.base, ""), update_list[k].file.replace(diskmap_group_data.base, ""), diskmap_group_name, diskmap_group_name, update_list[k].rename || update_list[k].original_filename || null);
});
wtvdl.setGroup(diskmap_group_data, 'ok', diskmap_data.version);
wtvdl.setGroup(diskmap_group_name, 'ok', diskmap_group_data.version);
}
}
if (diskmap_data.execute && diskmap_data.execute_when) {
if (diskmap_data.execute_when.toLowerCase().match(/end/)) {
wtvdl.execute(diskmap_data.execute);
}
if (diskmap_group_data.execute_end) {
wtvdl.execute(diskmap_group_data.execute_end);
}
if (files_to_send > 0) {
if (!diskmap_data.nogroup) {
wtvdl.deleteGroupUpdate(diskmap_group_data, diskmap_data.base);
if (!diskmap_group_data.nogroup) {
wtvdl.deleteGroupUpdate(diskmap_group_name, diskmap_group_data.base);
}
}
var download_list = wtvdl.getDownloadList();
@@ -94,11 +117,15 @@ if (request_headers['wtv-request-type'] == 'download') {
return download_list;
}
function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgroup = null) {
function processGroup(diskmap_primary_group, diskmap_group_data, diskmap_subgroup = null, version = 0) {
// parse webtv post
var output_data = '';
var post_data = new Array();
if (request_headers.post_data) post_data = request_headers.post_data.toString(CryptoJS.enc.Latin1).split("\n");
var client_group_data = new Array();
if (request_headers.post_data) {
post_data = request_headers.post_data.toString(CryptoJS.enc.Latin1).split("\n");
client_group_data = wtvdl.getGroupDataFromClientPost(request_headers.post_data.toString(CryptoJS.enc.Latin1));
}
var post_data_current_directory = '';
var post_data_current_file = false;
var post_data_current_group = '';
@@ -178,40 +205,40 @@ if (request_headers['wtv-request-type'] == 'download') {
}
});
var wtv_download_list = new Array();
var newest_file_epoch = 0;
var newest_file_epoch = version;
Object.keys(diskmap_group_data.files).forEach(function (k) {
if (!diskmap_group_data.files[k].location) diskmap_group_data.files[k].location = diskmap_group_data.location + diskmap_group_data.files[k].file.replace(diskmap_group_data.base, "");
var post_match_file = null;
if (!diskmap_group_data.files[k].location) diskmap_group_data.files[k].location = wtvshared.makeSafePath(diskmap_group_data.location,diskmap_group_data.files[k].file.replace(diskmap_group_data.base, ""));
var diskmap_data_file = null;
Object.keys(service_vaults).forEach(function (g) {
if (post_match_file != null) return;
post_match_file = service_vaults[g] + "/" + service_name + "/" + diskmap_group_data.files[k].location;
if (!fs.existsSync(post_match_file)) post_match_file = null;
if (diskmap_data_file != null) return;
diskmap_data_file = service_vaults[g] + "/" + service_name + "/" + diskmap_group_data.files[k].location;
if (!fs.existsSync(diskmap_data_file)) {
console.error("Could not find a file for", diskmap_group_data.files[k].location, "(Last tried SV:", diskmap_data_file, ")");
}
});
var post_match_file_lstat = fs.lstatSync(post_match_file);
var post_match_file_data = new Buffer.from(fs.readFileSync(post_match_file, {
var diskmap_file_stat = fs.lstatSync(diskmap_data_file);
var diskmap_file_data = new Buffer.from(fs.readFileSync(diskmap_data_file, {
encoding: null,
flags: 'r'
}));
diskmap_group_data.files[k].base = diskmap_group_data.base;
diskmap_group_data.files[k].last_modified = (new Date(new Date(post_match_file_lstat.mtime).toUTCString()) / 1000);
diskmap_group_data.files[k].content_length = post_match_file_lstat.size;
diskmap_group_data.files[k].action = (diskmap_group_data.files[k].action) ? diskmap_group_data.files[k].action.toUpperCase() : "GET";
diskmap_group_data.files[k].last_modified = (new Date(new Date(diskmap_file_stat.mtime).toUTCString()) / 1000);
diskmap_group_data.files[k].content_length = diskmap_file_stat.size;
diskmap_group_data.files[k].action = (diskmap_group_data.files[k].action) ? diskmap_group_data.files[k].action.toUpperCase() : "GET";
if (wtvshared.getFileExt(post_match_file).toLowerCase() == "gz") {
// we need the checksum of the uncompressed data
var gunzipped = zlib.gunzipSync(post_match_file_data);
// we need the checksum of the uncompressed data
if (wtvshared.getFileExt(diskmap_data_file).toLowerCase() == "gz") {
var gunzipped = zlib.gunzipSync(diskmap_file_data);
diskmap_group_data.files[k].checksum = CryptoJS.MD5(CryptoJS.lib.WordArray.create(gunzipped)).toString(CryptoJS.enc.Hex).toLowerCase();
var gzip_fn_end = post_match_file_data.indexOf("\0", 10);
var gzip_fn_end = diskmap_file_data.indexOf("\0", 10);
if (!diskmap_group_data.files[k].dont_extract_filename) {
diskmap_group_data.files[k].original_filename = post_match_file_data.toString('utf8', 10, gzip_fn_end);
diskmap_group_data.files[k].original_filename = diskmap_file_data.toString('utf8', 10, gzip_fn_end);
}
//diskmap_group_data.files[k].uncompressed_size = gunzipped.byteLength;
diskmap_group_data.files[k].uncompressed_size = gunzipped.byteLength;
gunzipped = null;
} else {
diskmap_group_data.files[k].checksum = CryptoJS.MD5(CryptoJS.lib.WordArray.create(post_match_file_data)).toString(CryptoJS.enc.Hex).toLowerCase();
diskmap_group_data.files[k].checksum = CryptoJS.MD5(CryptoJS.lib.WordArray.create(diskmap_file_data)).toString(CryptoJS.enc.Hex).toLowerCase();
}
if (parseInt(diskmap_group_data.files[k].last_modified) > newest_file_epoch) newest_file_epoch = parseInt(diskmap_group_data.files[k].last_modified);
@@ -232,6 +259,7 @@ if (request_headers['wtv-request-type'] == 'download') {
});
});
var diskmap_group_name = (diskmap_subgroup == null) ? diskmap_primary_group : diskmap_primary_group + "-" + diskmap_subgroup;
diskmap_group_data.client_group_data = client_group_data[diskmap_group_name] || null;
output_data = generateDownloadList(diskmap_group_name, wtv_download_list, diskmap_group_data);
return output_data;
}
@@ -245,9 +273,10 @@ if (request_headers['wtv-request-type'] == 'download') {
});
if (diskmap_json_file != null) {
if (fs.lstatSync(diskmap_json_file)) {
if (fs.existsSync(diskmap_json_file)) {
try {
// read diskmap
var json_stats = fs.lstatSync(diskmap_json_file);
var diskmap_data = JSON.parse(fs.readFileSync(diskmap_json_file).toString());
if (!diskmap_data[request_headers.query.group]) {
throw ("Invalid diskmap data (group does not match)");
@@ -256,10 +285,15 @@ if (request_headers['wtv-request-type'] == 'download') {
diskmap_data = diskmap_data[request_headers.query.group];
if (!diskmap_data.location) {
Object.keys(diskmap_data).forEach(function (k) {
if (diskmap_data[k]) data += processGroup(request_headers.query.group, diskmap_data[k], k);
console.log(diskmap_data[k]);
if (diskmap_data[k]) {
diskmap_data[k].version = (new Date(new Date(json_stats.mtime).toUTCString()) / 1000);
data += processGroup(request_headers.query.group, diskmap_data[k], k, diskmap_data.version);
}
});
} else {
data = processGroup(request_headers.query.group, diskmap_data);
diskmap_data.version = (new Date(new Date(json_stats.mtime).toUTCString()) / 1000);
data = processGroup(request_headers.query.group, diskmap_data, null, diskmap_data.version);
}
headers = "200 OK\nContent-Type: wtv/download-list";
@@ -286,5 +320,5 @@ if (request_headers['wtv-request-type'] == 'download') {
var message = request_headers.query.message || "Retrieving files...";
var main_message = request_headers.query.main_message || "Your receiver is downloading files.";
headers = "200 OK\nwtv-connection-close: close\nConnection: close\nContent-Type: text/html";
data = wtvdl.getSyncPage(message, request_headers.query.group, request_headers.query.diskmap, main_message, message, force_update)
data = wtvdl.getSyncPage(message, request_headers.query.group, request_headers.query.diskmap, main_message, message, force_update, no_delete);
}