Skip to content

Use async/await for LZ4.loadPackage #27034

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sbc100 wants to merge 1 commit into
base: main
Choose a base branch
from
+16 −35
Open

Use async/await for LZ4.loadPackage #27034

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 3 additions & 6 deletions src/lib/liblz4.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

#if LZ4
addToLibrary({
$LZ4__deps: ['$FS', '$preloadPlugins', '$getUniqueRunDependency', '$addRunDependency', '$removeRunDependency'],
$LZ4__deps: ['$FS', '$preloadPlugins'],
$LZ4: {
DIR_MODE: {{{ cDefs.S_IFDIR | 0o777 }}},
FILE_MODE: {{{ cDefs.S_IFREG | 0o777 }}},
Expand All @@ -20,7 +20,7 @@ addToLibrary({
})();
LZ4.CHUNK_SIZE = LZ4.codec.CHUNK_SIZE;
},
loadPackage(pack, preloadPlugin) {
async loadPackage(pack, preloadPlugin) {
LZ4.init();
var compressedData = pack['compressedData'] || LZ4.codec.compressPackage(pack['data']);
assert(compressedData['cachedIndexes'].length === compressedData['cachedChunks'].length);
Expand Down Expand Up @@ -52,14 +52,11 @@ addToLibrary({
var fullname = file.filename;
for (var plugin of preloadPlugins) {
if (plugin['canHandle'](fullname)) {
var dep = getUniqueRunDependency('fp ' + fullname);
addRunDependency(dep);
var finish = () => removeRunDependency(dep);
var byteArray = FS.readFile(fullname);
#if ASSERTIONS
assert(plugin['handle'].constructor.name === 'AsyncFunction', 'Filesystem plugin handlers must be async functions (See #24914)')
#endif
plugin['handle'](byteArray, fullname).then(finish).catch(finish);
await plugin['handle'](byteArray, fullname);
break;
}
}
Expand Down
7 changes: 1 addition & 6 deletions test/codesize/test_codesize_file_preload.expected.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,10 +112,6 @@ Module["expectedDataFileDownloads"]++;
function assert(check, msg) {
if (!check) throw new Error(msg);
}
for (var file of metadata["files"]) {
var name = file["filename"];
Module["addRunDependency"](`fp ${name}`);
}
async function processPackageData(arrayBuffer) {
assert(arrayBuffer, "Loading data file failed.");
assert(arrayBuffer.constructor.name === ArrayBuffer.name, "bad input to processPackageData " + arrayBuffer.constructor.name);
Expand All @@ -126,7 +122,6 @@ Module["expectedDataFileDownloads"]++;
var data = byteArray.subarray(file["start"], file["end"]);
// canOwn this data in the filesystem, it is a slice into the heap that will never change
Module["FS_createDataFile"](name, null, data, true, true, true);
Module["removeRunDependency"](`fp ${name}`);
}
Module["removeRunDependency"]("datafile_a.out.data");
}
Expand All @@ -138,7 +133,7 @@ Module["expectedDataFileDownloads"]++;
if (!fetched) {
fetched = await fetchPromise;
}
processPackageData(fetched);
await processPackageData(fetched);
}
if (Module["calledRun"]) {
runWithFS(Module);
Expand Down
8 changes: 4 additions & 4 deletions test/codesize/test_codesize_file_preload.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"a.out.js": 22224,
"a.out.js.gz": 9207,
"a.out.js": 22126,
"a.out.js.gz": 9184,
"a.out.nodebug.wasm": 1666,
"a.out.nodebug.wasm.gz": 945,
"total": 23890,
"total_gz": 10152,
"total": 23792,
"total_gz": 10129,
"sent": [
"a (fd_write)"
],
Expand Down
27 changes: 8 additions & 19 deletions tools/file_packager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -679,25 +679,14 @@ def generate_preload_js(data_target, data_files, metadata):
try {
// canOwn this data in the filesystem, it is a slice into the heap that will never change
await Module['FS_preloadFile'](name, null, data, true, true, false, true);
Module['removeRunDependency'](`fp ${name}`);
} catch (e) {
err(`Preloading file ${name} failed`, e);
}\n'''
create_data = '''// canOwn this data in the filesystem, it is a slice into the heap that will never change
Module['FS_createDataFile'](name, null, data, true, true, true);
Module['removeRunDependency'](`fp ${name}`);'''
Module['FS_createDataFile'](name, null, data, true, true, true);'''

finish_handler = create_preloaded if options.use_preload_plugins else create_data

if not options.lz4:
# Data requests - for getting a block of data out of the big archive - have
# a similar API to XHRs
code += '''
for (var file of metadata['files']) {
var name = file['filename']
Module['addRunDependency'](`fp ${name}`);
}\n'''

catch_handler = ''
if options.export_es6:
catch_handler += '''
Expand Down Expand Up @@ -735,7 +724,7 @@ def generate_preload_js(data_target, data_files, metadata):
use_data = '''var compressedData = %s;
compressedData['data'] = byteArray;
assert(typeof Module['LZ4'] === 'object', 'LZ4 not present - was your app build with -sLZ4?');
Module['LZ4'].loadPackage({ 'metadata': metadata, 'compressedData': compressedData }, %s);
await Module['LZ4'].loadPackage({ 'metadata': metadata, 'compressedData': compressedData }, %s);
Module['removeRunDependency']('datafile_%s');''' % (meta, "true" if options.use_preload_plugins else "false", js_manipulation.escape_for_js_string(data_target))

if options.export_es6:
Expand Down Expand Up @@ -993,7 +982,7 @@ def generate_preload_js(data_target, data_files, metadata):
async function preloadFallback(error) {
console.error(error);
console.error('falling back to default preload behavior');
processPackageData(await fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE));
await processPackageData(await fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE));
}

try {
Expand All @@ -1002,14 +991,14 @@ def generate_preload_js(data_target, data_files, metadata):
var useCached = !!pkgMetadata;
Module['preloadResults'][PACKAGE_NAME] = {fromCache: useCached};
if (useCached) {
processPackageData(await fetchCachedPackage(db, PACKAGE_PATH + PACKAGE_NAME, pkgMetadata));
await processPackageData(await fetchCachedPackage(db, PACKAGE_PATH + PACKAGE_NAME, pkgMetadata));
} else {
var packageData = await fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE);
try {
processPackageData(await cacheRemotePackage(db, PACKAGE_PATH + PACKAGE_NAME, packageData, {uuid:PACKAGE_UUID}))
await processPackageData(await cacheRemotePackage(db, PACKAGE_PATH + PACKAGE_NAME, packageData, {uuid:PACKAGE_UUID}))
} catch (error) {
console.error(error);
processPackageData(packageData);
await processPackageData(packageData);
}
}
} catch(e) {
Expand Down Expand Up @@ -1037,7 +1026,7 @@ def generate_preload_js(data_target, data_files, metadata):
if (!fetched) {
fetched = await fetchPromise;
}
processPackageData(fetched);\n'''
await processPackageData(fetched);\n'''

ret += '''
async function runWithFS(Module) {\n'''
Expand Down Expand Up @@ -1077,7 +1066,7 @@ def generate_preload_js(data_target, data_files, metadata):
throw new Error(`${response.status}: ${response.url}`);
}
var json = await response.json();
await loadPackage(json);
loadPackage(json);
}

if (Module['calledRun']) {
Expand Down
Loading