read-bundles.js

const assert = require('assert');
const fs = require('fs');
const globule = require('globule');
const mkdirp = require('mkdirp');
const murmurhash = require('murmurhash-native');
const path = require('path');
const process = require('process');
const util = require("util");
const zlib = require('zlib');

// ==================================================================

// Change this value to choose extension
const WANTED_EXT = 'package'; // set to 'none' to use all known extensions
const KNOWN_EXTENSIONS = [ 'package', 'unit', 'lua', 'material', 'particles', 'shading_environment', 'vector_field', 'wwise_dep', 'font', 'mouse_cursor', 'texture', 'physics_properties', 'animation', 'bones', 'level', 'state_machine', 'bsi', 'animation_set', 'editor', 'entity', 'mod', 'physx_metadata', 'scene_cache', 'shader_import', 'shader_node', 'shader_source', 'statemachine_editor', 'texture_category', 'wwise_bank', 'wwise_bank_metadata', 'script_flow_nodes'];

// Change this value to choose a specific bundle name
const DESIRED_BUNDLE = 'none'; // (names are 16 characters or 'none')

// Change this value to choose a specific patch file number
const DESIRED_PATCH = 'none'; // (in format 'patch_xxx', 'patch_x' (VR only), 'base', or 'none')

// Change this value to true to ignore or focus on extracted files that do not have a dictionary match
const IGNORE_UNDEFINED = false; // (accepted values are 'all', 'only', and false)

// Change this value to true to sort extracted files by their bundle of origin
const SORT_BY_BUNDLE = false;

// Change this value to true to sort extracted files by their patch file number of origin
const SORT_BY_PATCH = false;

// Change this value to choose between extracting from Vermintide 1, Vermintide 2, and Hero Trials VR
const EXTRACTION_MODE = 'vt1'; // (options are 'vt1', 'vt2', and 'vr')

// ==================================================================

// Tracks successful matches and appends them to an output file
var successfulDictionary = [];
var undefinedCounter = 0

function FileReader(filename) {
	this.fd = fs.openSync(filename, 'r');
	this.path = filename;
}
FileReader.prototype = {

	readBytes: function readBytes(count) {
		var result = Buffer.allocUnsafe(count);
		var bytesRead = fs.readSync(this.fd, result, 0, count, null);
		assert.strictEqual(bytesRead, count, 'bytesRead');
		return result;
	},

	readUint: function readUint() {
		return this.readBytes(4).readUInt32LE(0);
	},

	close: function close() {
		fs.closeSync(this.fd);
		this.fd = null;
	}
};
FileReader.prototype.constructor = FileReader;

function InflatingReader(fileReader) {
	this.fileReader = fileReader;
	this.pos = 0;
	this.buf = Buffer.allocUnsafe(0);
}
InflatingReader.prototype = {

	readBytes: function readBytes(count) {
		var result = Buffer.allocUnsafe(count);
		var self = this;
		while (count !== 0) {
			if (self.pos == self.buf.length) {
				var chunkSize = this.fileReader.readUint();
				var chunkZipped = this.fileReader.readBytes(chunkSize);
				while (chunkSize > 0xFFFF) {
					// This is bizarre.
					count = Math.max(0, count - chunkSize);

					chunkSize = this.fileReader.readUint();
					chunkZipped = Buffer.concat([ this.fileReader.readBytes(chunkSize), chunkZipped ]);
				}
				self.buf = zlib.inflateSync(chunkZipped);
				self.pos = 0;
			}
			var copySize = Math.min(self.buf.length - self.pos, count);
			self.buf.copy(result, result.length - count, self.pos, self.pos + copySize);
			self.pos += copySize;
			count -= copySize;
		}
		return result;
	},

	readUint: function readUint() {
		return this.readBytes(4).readUInt32LE(0);
	}
};
InflatingReader.prototype.constructor = InflatingReader;

function Dictionary() {
	var lines = fs.readFileSync(path.join(__dirname, 'dictionary.txt')).toString().split("\n");
	var hasher = murmurhash.LE.murmurHash64;
	for (var i in lines) {
		var filename = lines[i];
		var hash = '' + hasher(filename);
		//console.log('adding hash '+ hash);
		this[hash] = filename;
	}
}

const dictionary = new Dictionary();
const fileVersions = {};

function processBundlefile(bundlefile, bundlefilePath, chosenExtensionIn, extensionHashIn) {
	var magic = bundlefile.readUint();
	//console.log('magic: ' + magic);

	// A new bundle file format was added in 1.6 beta.
	var isNewFormat = (magic === 0xF0000005);
	assert(isNewFormat || (magic === 0xF0000004), 'magic');

	var unzippedSize = bundlefile.readUint();
	//console.log('unzippedSize: ' + unzippedSize);

	var padding = bundlefile.readUint();
	//console.log('padding: ' + padding);
	assert.strictEqual(padding, 0x0, 'padding');

	var inflater = new InflatingReader(bundlefile);
	if (EXTRACTION_MODE === 'vr') {
		inflater.readBytes(260); // skip the initial header early, as well as an extra 4 bytes
	}

	var entryCount = inflater.readUint();
	//console.log('entryCount: ' + entryCount);

	// skip header
	if (EXTRACTION_MODE === 'vr') {
		inflater.readBytes(4); // skip the header remainder
	} else {
		inflater.readBytes(256);
	}

	var containsScripts = false;
	for (var i = 0; i < entryCount; ++i) {
		var extensionHash = inflater.readBytes(8);
		var nameHash = inflater.readBytes(8);
		// New format has an extra 4-byte field here, I've seen values of 1 (seems to mean
		// luajit v1 bytecode) and 2 (not sure, might mean file no longer in use).
		// VR bundles do not seem to use this field.
		var flags = 0
		if (EXTRACTION_MODE != 'vr') {
			flags = isNewFormat ? inflater.readUint() : 0;
		}
		//console.log('entry hashes ' + i + ': ' + extensionHash.toString('hex') + '  ' + nameHash.toString('hex'));

		// Print filename to console
		if (false) {
			var nameHashString = nameHash.toString('hex');
			var name = dictionary[nameHashString];
			var extensionHashString = extensionHash.toString('hex');
			var extension = dictionary[extensionHashString];
			var filePath = (name || nameHashString) + '.' + (extension || extensionHashString);
			console.log('  ' + filePath);
		}

		containsScripts = containsScripts || (extensionHashIn === extensionHash.toString('hex'));
	}

	if (containsScripts) {
		for (var i = 0; i < entryCount; ++i) {
			var extensionHash = inflater.readBytes(8);
			var nameHash = inflater.readBytes(8);
			//console.log('  entry hashes ' + i + ': ' + extensionHash.toString('hex') + '  ' + nameHash.toString('hex'));

			var headerCount = inflater.readUint();
			var headerUnknown = inflater.readUint();
			var headers = [];
			for (var j = 0; j < headerCount; ++j) {
				var languageId = inflater.readUint();
				var size = inflater.readUint();
				var unknown = inflater.readUint();
				headers.push({ languageId: languageId, size: size, unknown: unknown });
			}

			for (var j = 0; j < headerCount; ++j) {
				var entryData = inflater.readBytes(headers[j].size);

				if ((extensionHashIn === extensionHash.toString('hex')) && (headers[j].languageId === 0)) {
					var nameHashString = nameHash.toString('hex');
					var name = dictionary[nameHashString];
					if (name != null) {
						successfulDictionary.push(name);
					} else {
						undefinedCounter += 1
					}

					if ((name && (IGNORE_UNDEFINED != 'only')) || (!name && (IGNORE_UNDEFINED != 'all'))) {
						var filePath = (name || nameHashString) + '.' + chosenExtensionIn;

						if (SORT_BY_PATCH) {
							if (EXTRACTION_MODE != 'vr') {
								if (bundlefilePath.includes('patch_')) {
									filePath = bundlefilePath.substring(bundlefilePath.length-9) + '/' + filePath
								} else {
									filePath = 'base_bundle/' + filePath
								}
							} else {
								if (bundlefilePath.includes('patch_')) {
									filePath = bundlefilePath.substring(bundlefilePath.length-7) + '/' + filePath
								} else {
									filePath = 'base_bundle/' + filePath
								}
							}
						}

						if (SORT_BY_BUNDLE) {
							if (EXTRACTION_MODE != 'vr') {
								if (bundlefilePath.includes('patch_')) {
									filePath = bundlefilePath.substring(bundlefilePath.length-26, bundlefilePath.length-10) + '/' + filePath
								} else {
									filePath = bundlefilePath.substring(bundlefilePath.length-16) + '/' + filePath
								}
							} else { //
								if (bundlefilePath.includes('patch_')) {
									filePath = bundlefilePath.substring(bundlefilePath.length-24, bundlefilePath.length-8) + '/' + filePath
								} else {
									filePath = bundlefilePath.substring(bundlefilePath.length-16) + '/' + filePath
								}
							}
						}

						console.log('extracting ' + filePath);
						//console.log('  size=' + headers[j].size + ' unknown=' + headers[j].unknown + ' @0=' +
						//        entryData.readUInt32LE(0) + ' @4=' + entryData.readUInt32LE(4) + ' @8=' + entryData.readUInt32LE(8));

						var localpath = path.join(process.argv[3] || '', filePath);
						mkdirp.sync(path.dirname(localpath));

						// New format adds another 4-byte field, seems to always be 0.
						var headerByteCount = isNewFormat ? 12 : 8;
						var scriptEnd = headerByteCount + entryData.readUInt32LE(0);
						var scriptData = entryData.slice(headerByteCount, scriptEnd);
						fs.writeFileSync(localpath, scriptData);
					}
				}
			}
		}
	}
}

function processDirectory(dirpath) {
	var bundleFiles = null;

	// No desired bundle or patch to focus on
	if ((DESIRED_BUNDLE === 'none') && (DESIRED_PATCH === 'none')) {
		bundlefiles = globule.find(dirpath + '/*', '!' + dirpath + '/*.stream*', '!' + dirpath + '/*.ini', '!' + dirpath + '/*.data');
	// Desired bundle but no patch to focus on
	} else if ((DESIRED_BUNDLE != 'none') && (DESIRED_PATCH === 'none')) {
		bundlefiles = globule.find(dirpath + '/' + DESIRED_BUNDLE + '*', '!' + dirpath + '/*.stream*', '!' + dirpath + '/*.ini', '!' + dirpath + '/*.data');
	// No desired bundle but desired patch to focus on
	} else if ((DESIRED_BUNDLE === 'none') && (DESIRED_PATCH != 'none')) {
		if (DESIRED_PATCH != 'base') {
			bundlefiles = globule.find(dirpath + '/*.' + DESIRED_PATCH, '!' + dirpath + '/*.stream*', '!' + dirpath + '/*.ini', '!' + dirpath + '/*.data');
		} else {
			bundlefiles = globule.find(dirpath + '/*', '!' + dirpath + '/*.stream*', '!' + dirpath + '/*.ini', '!' + dirpath + '/*.data', '!' + dirpath + '/*.patch_*');
		}
	// Desired bundle and patch to focus on
	} else {
		if (DESIRED_PATCH != 'base') {
			bundlefiles = globule.find(dirpath + '/' + DESIRED_BUNDLE + '.' + DESIRED_PATCH, '!' + dirpath + '/*.stream*', '!' + dirpath + '/*.ini', '!' + dirpath + '/*.data');
		} else {
			bundlefiles = globule.find(dirpath + '/' + DESIRED_BUNDLE + '*', '!' + dirpath + '/*.stream*', '!' + dirpath + '/*.ini', '!' + dirpath + '/*.data', '!' + dirpath + '/*.patch_*');
		}
	}
	bundlefiles.sort(function(a, b) { return path.extname(a).localeCompare(path.extname(b)) });

	for (var i in bundlefiles) {
		var bundlefilePath = bundlefiles[i];
		if (!fs.lstatSync(bundlefilePath).isDirectory()) {
			console.log('___ processing file: ' + bundlefilePath);
			try {

				if (WANTED_EXT === 'none') {
					for (var j in KNOWN_EXTENSIONS) {
						var bundlefile = new FileReader(bundlefilePath);
						var chosenExtension = KNOWN_EXTENSIONS[j]
						var extensionHash = murmurhash.LE.murmurHash64(chosenExtension);
						processBundlefile(bundlefile, bundlefilePath, chosenExtension, extensionHash);
						if (bundlefile) {
							bundlefile.close();
						}
					}
				} else {
					var bundlefile = new FileReader(bundlefilePath);
					var chosenExtension = WANTED_EXT
					var extensionHash = murmurhash.LE.murmurHash64(chosenExtension);
					processBundlefile(bundlefile, bundlefilePath, chosenExtension, extensionHash);
				}
			}
			catch (excn) {
				console.log('error processing file: ' + bundlefilePath + ": " + excn);
			}
			finally {
				if (bundlefile && WANTED_EXT != 'none') {
					bundlefile.close();
				}
			}
		}
	}
}

function printHash(str) {
	console.log(str + '=' + murmurhash.LE.murmurHash64(str));
}

// Output successful match dictionary
processDirectory(process.argv[2]);
var outputPath = process.cwd();
for (var s = 0, len = successfulDictionary.length; s < len; s++) {
	fs.appendFileSync(outputPath+"\\dictionary_matches.txt", successfulDictionary[s]+"\n");
}
if ((IGNORE_UNDEFINED != 'all') && (IGNORE_UNDEFINED != 'only')) {
	console.log("Matched " + successfulDictionary.length + " files, found " + undefinedCounter + " undefined files successfully")
} else if (IGNORE_UNDEFINED === 'all') {
	console.log("Matched " + successfulDictionary.length + " files successfully")
} else {
	console.log("Found " + undefinedCounter + " undefined files")
}