diff --git a/Extensions/Spine/managers/pixi-spine-atlas-manager.ts b/Extensions/Spine/managers/pixi-spine-atlas-manager.ts index aa8038bf3477..849a91c42fe6 100644 --- a/Extensions/Spine/managers/pixi-spine-atlas-manager.ts +++ b/Extensions/Spine/managers/pixi-spine-atlas-manager.ts @@ -103,6 +103,11 @@ namespace gdjs { return imagesMap; }, {}); + // Note: "atlas" and "spine" resources are never put inside a resource + // pack at export (see `ResourcePackPlanner`), because `PIXI.Assets` + // decides which loader to use from the extension of the URL, and a + // `blob:` URL has none. Only the atlas pages, which are plain images + // given to the loader below as already-loaded textures, are packed. const url = this._resourceLoader.getFullUrl(resource.file); const alias = url; diff --git a/GDJS/GDJS/IDE/ExporterHelper.cpp b/GDJS/GDJS/IDE/ExporterHelper.cpp index bf918c3b38fc..eca9d38a2b21 100644 --- a/GDJS/GDJS/IDE/ExporterHelper.cpp +++ b/GDJS/GDJS/IDE/ExporterHelper.cpp @@ -1169,6 +1169,7 @@ void ExporterHelper::AddLibsInclude(bool pixiRenderers, InsertUnique(includesFiles, "inputmanager.js"); InsertUnique(includesFiles, "jsonmanager.js"); InsertUnique(includesFiles, "Model3DManager.js"); + InsertUnique(includesFiles, "ResourcePackManager.js"); InsertUnique(includesFiles, "ResourceLoader.js"); InsertUnique(includesFiles, "ResourceCache.js"); InsertUnique(includesFiles, "timemanager.js"); diff --git a/GDJS/Runtime/ResourceLoader.ts b/GDJS/Runtime/ResourceLoader.ts index 0bb1ee9e4046..30ea9547175d 100644 --- a/GDJS/Runtime/ResourceLoader.ts +++ b/GDJS/Runtime/ResourceLoader.ts @@ -148,6 +148,12 @@ namespace gdjs { private _spineManager: SpineManager | null = null; private _svgManager: InternalInGameEditorOnlySvgManager; + /** + * Gives access to the resources of a game exported with its resources + * packed into ".gdpak" archives. Does nothing otherwise. + */ + private _resourcePackManager = new gdjs.ResourcePackManager(); + private privateResourceManager = new PrivateResourceManager(this); private sceneResourceLoadingQueue = new ResourceLoadingQueue( 'scene', @@ -239,6 +245,12 @@ namespace gdjs { ): void { this._globalResources = globalResources; + // The exporter writes this at the end of `data.js` when it packed the + // game resources. It stays null for previews and for games exported + // without packing, in which case every resource is downloaded as its own + // file, as before. + this._resourcePackManager.setManifest(gdjs.resourcePacks); + // TODO We should probably instanciate new queues to avoid side effects from running tasks. this.sceneResourceLoadingQueue.clear(); for (const objectResourceLoadingQueue of this.objectResourceLoadingQueues.values()) { @@ -346,24 +358,52 @@ namespace gdjs { ...this._globalResources, ...firstSceneResourceNames, ]; - await ResourceLoader.processAndRetryIfNeededWithPromisePool( - resourceNames, - ResourceLoader.maxForegroundConcurrency, - ResourceLoader.maxAttempt, - async (resourceName) => { - const resource = - this.privateResourceManager._resources.get(resourceName); - if (!resource) { - logger.warn('Unable to find resource "' + resourceName + '".'); - return; - } - await this.privateResourceManager._loadResource(resource); - await this.privateResourceManager._processResource(resource); - loadedCount++; - onProgress(loadedCount, resourceNames.length); + + // No resource can be loaded while its pack is downloading, so without + // this the loading bar would stay at 0% for the whole download. Report + // the download itself as a fraction of a resource. + let lastReportedPackProgress = 0; + this._resourcePackManager.setOnProgressCallback( + (loadedBytes, totalBytes) => { + // A server compressing the pack on the fly announces the compressed + // size, while the bytes received are decompressed: don't go past 100%. + const packProgress = Math.min(1, loadedBytes / totalBytes); + if (Math.abs(packProgress - lastReportedPackProgress) < 0.01) return; + lastReportedPackProgress = packProgress; + onProgress(loadedCount + packProgress, resourceNames.length); } ); + try { + // Resources that are only reachable dynamically (a sound played by + // name from an expression) are in no loading task, so nothing else + // would ever download the pack holding them - and the engine asks for + // their URL synchronously, when it is too late to download anything. + const startupPacksPromise = + this._resourcePackManager.ensureStartupPacksLoaded(); + if (startupPacksPromise) await startupPacksPromise; + + await ResourceLoader.processAndRetryIfNeededWithPromisePool( + resourceNames, + ResourceLoader.maxForegroundConcurrency, + ResourceLoader.maxAttempt, + async (resourceName) => { + const resource = + this.privateResourceManager._resources.get(resourceName); + if (!resource) { + logger.warn('Unable to find resource "' + resourceName + '".'); + return; + } + await this.privateResourceManager._loadResource(resource); + await this.privateResourceManager._processResource(resource); + loadedCount++; + onProgress(loadedCount, resourceNames.length); + } + ); + } finally { + this._resourcePackManager.setOnProgressCallback(null); + } + this.sceneResourceLoadingQueue.setResourcesAs(firstSceneName, 'ready'); } @@ -526,11 +566,44 @@ namespace gdjs { this.getObjectResourceLoadingQueue(unloadedSceneName); objectResourceLoadingQueue.clear(); + this._unloadUnusedResourcePacks(); + debugLogger.log( `Unloading of resources for scene ${unloadedSceneName} finished.` ); } + /** + * Give back the memory used by the archives of the scenes that are not + * loaded anymore. Does nothing for a game exported without packed + * resources. + */ + private _unloadUnusedResourcePacks(): void { + const stillLoadedFiles = new Set(); + const addFilesOf = (resourceNames: Array) => { + for (const resourceName of resourceNames) { + const resource = + this.privateResourceManager._resources.get(resourceName); + if (resource) stillLoadedFiles.add(resource.file); + } + }; + + // Global resources are never unloaded. + addFilesOf(this._globalResources); + for (const loadingState of this.sceneResourceLoadingQueue.loadingStates.values()) { + if (loadingState.status === 'not-loaded') continue; + addFilesOf(loadingState.resourceNames); + } + for (const objectResourceLoadingQueue of this.objectResourceLoadingQueues.values()) { + for (const loadingState of objectResourceLoadingQueue.loadingStates.values()) { + if (loadingState.status === 'not-loaded') continue; + addFilesOf(loadingState.resourceNames); + } + } + + this._resourcePackManager.unloadPacksWithNoFileIn(stillLoadedFiles); + } + /** * Unload an object assets in background. */ @@ -573,6 +646,9 @@ namespace gdjs { for (const objectResourceLoadingQueue of this.objectResourceLoadingQueues.values()) { objectResourceLoadingQueue.clear(); } + // Keep the manifest: the packs are downloaded again when the resources + // are loaded back. + this._resourcePackManager.unloadAllPacks(); debugLogger.log(`Unloading of all resources finished.`); } @@ -610,6 +686,58 @@ namespace gdjs { return this.privateResourceManager._resources.get(resourceName) || null; } + /** + * Download the resource pack holding this resource, if the game was + * exported with packed resources and the pack is not downloaded yet. + * + * @returns null when there is nothing to wait for. Callers must check it + * rather than awaiting unconditionally, so that loading a resource keeps + * starting synchronously when there is no pack involved. + */ + ensurePackLoadedFor(resource: ResourceData): Promise | null { + if (!this._resourcePackManager.hasPacks()) return null; + + const loadingPromises: Array> = []; + const visitedResourceNames = new Set(); + const visit = (resource: ResourceData) => { + if (visitedResourceNames.has(resource.name)) return; + visitedResourceNames.add(resource.name); + + const loadingPromise = this._resourcePackManager.ensureLoadedFor( + resource.file + ); + if (loadingPromise) loadingPromises.push(loadingPromise); + + // Managers reach the resources embedded in another one synchronously + // (the Spine atlas manager asks the image manager for the page + // textures of an atlas), so their packs must be downloaded too. + for (const embeddedResourceName of this._runtimeGame.getEmbeddedResourcesNames( + resource.name + )) { + const embeddedResource = this.privateResourceManager._resources.get( + this._runtimeGame.resolveEmbeddedResource( + resource.name, + embeddedResourceName + ) + ); + if (embeddedResource) visit(embeddedResource); + } + }; + visit(resource); + + if (!loadingPromises.length) return null; + if (loadingPromises.length === 1) return loadingPromises[0]; + return Promise.all(loadingPromises).then(() => {}); + } + + /** + * @returns true when this file is stored inside a resource pack, and so is + * read from memory rather than downloaded on its own. + */ + isFileInResourcePack(file: string): boolean { + return this._resourcePackManager.isPacked(file); + } + // Helper methods used when resources are loaded from an URL. /** @@ -617,6 +745,23 @@ namespace gdjs { * the resource (this can be for example a token needed to access the resource). */ getFullUrl(url: string) { + // When the game was exported with packed resources, the file lives inside + // an archive that was already downloaded (`_loadResource` waits for it), + // and is read from a `blob:` URL instead of being fetched on its own. + const packedUrl = this._resourcePackManager.getObjectUrl(url); + if (packedUrl) return packedUrl; + + if (this._resourcePackManager.isPacked(url)) { + // The file is in a pack that is not downloaded yet. The URL returned + // below points to a file that the export does not contain, so loading + // it will fail: warn rather than let it look like a missing file. + logger.warn( + 'The resource file "' + + url + + '" was requested before its resource pack was downloaded.' + ); + } + if (this._runtimeGame.isInGameEdition()) { // Avoid adding cache burst to URLs which are assumed to be immutable files, // to avoid costly useless requests each time the game is hot-reloaded. @@ -977,6 +1122,14 @@ namespace gdjs { ); return; } + // Make sure the archive holding this file is downloaded before the + // manager asks for its URL. Concurrent calls share the same download. + // Nothing is awaited for a game exported without packed resources, so + // that the download of a resource still starts synchronously. + const packLoadingPromise = + this.resourceLoader.ensurePackLoadedFor(resource); + if (packLoadingPromise) await packLoadingPromise; + await resourceManager.loadResource(resource.name); } @@ -988,7 +1141,9 @@ namespace gdjs { ); if (resourceManager) { debugLogger.log( - `Unloading of resources of kind ${resourceData.kind} : ${resourceName}` + `Unloading of resources of kind ${ + resourceData.kind + } : ${resourceName}` ); resourceManager.unloadResource(resourceData); } @@ -1062,7 +1217,11 @@ namespace gdjs { debugLogger.log(`Loading all ${this.name} resources, in background.`); while (this.loadingTaskQueue.length > 0) { debugLogger.log( - `Still resources of ${this.loadingTaskQueue.length} ${this.name}(s) to load: ${this.loadingTaskQueue.map((task) => task.identifier).join(', ')}` + `Still resources of ${this.loadingTaskQueue.length} ${ + this.name + }(s) to load: ${this.loadingTaskQueue + .map((task) => task.identifier) + .join(', ')}` ); const task = this.loadingTaskQueue[this.loadingTaskQueue.length - 1]; if (task === undefined) { @@ -1071,7 +1230,9 @@ namespace gdjs { this.currentLoadingTaskIdentifier = task.identifier; if (!this.areAssetsLoaded(task.identifier)) { debugLogger.log( - `Loading (but not processing) resources for ${this.name} ${task.identifier}.` + `Loading (but not processing) resources for ${this.name} ${ + task.identifier + }.` ); const loadingState = this.loadingStates.get(task.identifier); if (loadingState) { @@ -1080,12 +1241,16 @@ namespace gdjs { ); } else { logger.warn( - `Can\'t load resource for unknown ${this.name}: "${task.identifier}".` + `Can\'t load resource for unknown ${this.name}: "${ + task.identifier + }".` ); return; } debugLogger.log( - `Done loading (but not processing) resources for ${this.name} ${task.identifier}.` + `Done loading (but not processing) resources for ${this.name} ${ + task.identifier + }.` ); // A task may have been moved last while awaiting resources to be @@ -1249,7 +1414,9 @@ namespace gdjs { } if (objectLoadingState.status !== 'not-loaded') { debugLogger.log( - `Resources for ${this.name} ${taskIdentifier} are already loading or loaded.` + `Resources for ${ + this.name + } ${taskIdentifier} are already loading or loaded.` ); return null; } @@ -1270,7 +1437,9 @@ namespace gdjs { } if (!unloadedTaskIdentifier) return; debugLogger.log( - `Unloading of resources for ${this.name} ${unloadedTaskIdentifier} was requested.` + `Unloading of resources for ${ + this.name + } ${unloadedTaskIdentifier} was requested.` ); const unloadedTaskState = this.loadingStates.get(unloadedTaskIdentifier); @@ -1290,7 +1459,9 @@ namespace gdjs { } debugLogger.log( - `Unloading of resources for ${this.name} ${unloadedTaskIdentifier} finished.` + `Unloading of resources for ${ + this.name + } ${unloadedTaskIdentifier} finished.` ); unloadedTaskState.status = 'not-loaded'; diff --git a/GDJS/Runtime/ResourcePackManager.ts b/GDJS/Runtime/ResourcePackManager.ts new file mode 100644 index 000000000000..753e83e4ab60 --- /dev/null +++ b/GDJS/Runtime/ResourcePackManager.ts @@ -0,0 +1,437 @@ +/* + * GDevelop JS Platform + * Copyright 2013-present Florian Rival (Florian.Rival@gmail.com). All rights reserved. + * This project is released under the MIT License. + */ +namespace gdjs { + const logger = new gdjs.Logger('ResourcePackManager'); + + const PACK_MAGIC = 'GDPK'; + const PACK_HEADER_SIZE = 12; + const SUPPORTED_PACK_VERSION = 1; + + /** + * An entry of the index stored at the beginning of a pack. + */ + type ResourcePackEntryData = { + path: string; + offset: integer; + size: integer; + type: string; + }; + + /** + * The list of packs of an exported game, and the pack each resource file + * lives in. + * + * This is written by the exporter at the end of `data.js`, and is left + * undefined when the game was exported without packing its resources (which + * is the case for previews and for in-game edition). + * @category Resources + */ + export type ResourcePacksManifest = { + version: integer; + /** The pack file names, relative to the game index.html. */ + packs: Array; + /** Resource file name -> index in `packs`. */ + files: Record; + /** + * Packs that must be downloaded before the first scene starts, even though + * no loading task refers to their files. + * + * A resource that is only reachable dynamically - a sound played by name + * from an expression, an animation picked by an expression - appears in no + * `usedResources` list, so nothing would ever trigger the download of its + * pack, and the engine asks for its URL synchronously when it is used. + * Those resources are gathered in a pack listed here. + */ + startupPacks?: Array; + }; + + /** + * Set by the exported `data.js` when the game resources were packed. + * @category Resources + */ + export let resourcePacks: ResourcePacksManifest | null = null; + + /** + * A single ".gdpak" archive, downloaded as one file and then sliced to give + * each resource its own `blob:` URL. + * + * See `newIDE/app/src/ExportAndShare/ResourcePacking/PackFormat.js` for the + * description of the format. + */ + class ResourcePack { + private readonly _url: string; + /** Resource file name -> a slice of the downloaded archive. */ + private _entries = new Map(); + /** Resource file name -> the object URL handed out for it. */ + private _objectUrls = new Map(); + + constructor(url: string) { + this._url = url; + } + + /** + * Download the archive, then index its content. + */ + load( + onProgress?: (loadedBytes: integer, totalBytes: integer) => void + ): Promise { + return this._download(onProgress).then((blob) => this._readIndex(blob)); + } + + /** + * `XMLHttpRequest` is used rather than `fetch`: it works on the `file:` + * URLs of Electron and Cordova games, and reports progress natively. + */ + private _download( + onProgress?: (loadedBytes: integer, totalBytes: integer) => void + ): Promise { + return new Promise((resolve, reject) => { + const request = new XMLHttpRequest(); + request.responseType = 'blob'; + request.onprogress = (event) => { + if (onProgress && event.lengthComputable && event.total > 0) { + onProgress(event.loaded, event.total); + } + }; + request.onload = () => { + const blob: Blob | null = request.response; + // A `file:` URL answers with a status of 0. + const isSuccess = + (request.status >= 200 && request.status < 300) || + (request.status === 0 && !!blob && blob.size > 0); + if (!isSuccess || !blob) { + reject( + new Error( + `Could not download the resource pack "${this._url}" (status ${request.status}).` + ) + ); + return; + } + resolve(blob); + }; + request.onerror = () => + reject( + new Error( + `Could not download the resource pack "${this._url}" (network error).` + ) + ); + request.onabort = () => + reject( + new Error( + `The download of the resource pack "${this._url}" was aborted.` + ) + ); + request.open('GET', this._url); + request.send(); + }); + } + + private async _readIndex(blob: Blob): Promise { + const headerBytes = await blob.slice(0, PACK_HEADER_SIZE).arrayBuffer(); + if (headerBytes.byteLength < PACK_HEADER_SIZE) { + throw new Error(`The resource pack "${this._url}" is truncated.`); + } + + const headerBytesArray = new Uint8Array(headerBytes); + const magic = String.fromCharCode( + headerBytesArray[0], + headerBytesArray[1], + headerBytesArray[2], + headerBytesArray[3] + ); + if (magic !== PACK_MAGIC) { + throw new Error( + `"${this._url}" is not a resource pack (unexpected magic "${magic}").` + ); + } + + const headerView = new DataView(headerBytes); + const version = headerView.getUint32(4, true); + if (version !== SUPPORTED_PACK_VERSION) { + throw new Error( + `The resource pack "${this._url}" uses the unsupported version ${version}.` + ); + } + + const indexByteLength = headerView.getUint32(8, true); + const indexJson = await blob + .slice(PACK_HEADER_SIZE, PACK_HEADER_SIZE + indexByteLength) + .text(); + const entries: Array = JSON.parse(indexJson).files; + + // Slicing a Blob does not copy anything: the browser owns the downloaded + // bytes (and may keep them out of memory), and each entry is only a view + // on them. + for (const entry of entries) { + this._entries.set( + entry.path, + blob.slice(entry.offset, entry.offset + entry.size, entry.type) + ); + } + } + + /** + * @returns a `blob:` URL for this file, or null if the pack does not + * contain it. The same URL is returned for subsequent calls. + */ + getObjectUrl(filePath: string): string | null { + const existingUrl = this._objectUrls.get(filePath); + if (existingUrl !== undefined) return existingUrl; + + const blob = this._entries.get(filePath); + if (!blob) return null; + + const objectUrl = URL.createObjectURL(blob); + this._objectUrls.set(filePath, objectUrl); + return objectUrl; + } + + getFilePaths(): Array { + return Array.from(this._entries.keys()); + } + + /** + * Release the archive and every URL handed out for it. + */ + dispose(): void { + for (const objectUrl of this._objectUrls.values()) { + URL.revokeObjectURL(objectUrl); + } + this._objectUrls.clear(); + this._entries.clear(); + } + } + + /** + * Gives access to the resources of a game whose export packed them into + * ".gdpak" archives, so that the exported game stays below the file count + * limits of hosting services (itch.io refuses archives with more than 1000 + * files). + * + * When a game was exported without packing, every method is a no-op and the + * engine downloads each resource file as usual. + * @category Resources + */ + export class ResourcePackManager { + private _manifest: ResourcePacksManifest | null = null; + private _packs: Array = []; + /** In-flight downloads, so that a pack is only ever downloaded once. */ + private _loadingPromises: Array | null> = []; + private _onProgress: + | ((loadedBytes: integer, totalBytes: integer) => void) + | null = null; + /** + * The download progress of the packs currently being downloaded, so that + * progress can be reported for all of them at once rather than having each + * pack fight over the loading bar. + */ + private _pendingDownloads = new Map< + integer, + { loadedBytes: integer; totalBytes: integer } + >(); + /** + * Incremented every time the packs are released, so that a download + * started before does not resurrect a pack that was disposed in between. + */ + private _generation: integer = 0; + + /** + * Read the manifest written by the exporter. Called by the resource loader + * when the game data is set, so that hot-reloading picks up changes too. + */ + setManifest(manifest: ResourcePacksManifest | null): void { + if (manifest && manifest.version !== SUPPORTED_PACK_VERSION) { + logger.error( + `Unsupported resource pack manifest version ${ + manifest.version + }, resources will be loaded as individual files.` + ); + manifest = null; + } + + this.dispose(); + this._manifest = manifest; + this._packs = manifest ? manifest.packs.map(() => null) : []; + this._loadingPromises = manifest ? manifest.packs.map(() => null) : []; + } + + /** + * Register a callback notified while a pack is being downloaded, so that + * the loading screen can show something is happening. + */ + setOnProgressCallback( + onProgress: ((loadedBytes: integer, totalBytes: integer) => void) | null + ): void { + this._onProgress = onProgress; + } + + /** + * @returns true when the game was exported with packed resources. + */ + hasPacks(): boolean { + return !!this._manifest; + } + + isPacked(filePath: string): boolean { + return !!this._manifest && this._manifest.files[filePath] !== undefined; + } + + /** + * Download the pack containing this file, if any and if not already done. + * + * @returns null when there is nothing to wait for: the game was exported + * without packing, the file was left as an individual file, or its pack is + * already downloaded. Callers must not await unconditionally, so that + * loading a resource keeps starting synchronously. + */ + ensureLoadedFor(filePath: string): Promise | null { + const manifest = this._manifest; + if (!manifest) return null; + + const packIndex = manifest.files[filePath]; + if (packIndex === undefined) return null; + + return this._ensurePackLoaded(packIndex); + } + + /** + * Download the packs holding the resources that no loading task refers to. + * To be awaited before the first scene is loaded. + * + * @returns null when there is nothing to wait for. + */ + ensureStartupPacksLoaded(): Promise | null { + const manifest = this._manifest; + if (!manifest || !manifest.startupPacks) return null; + + const loadingPromises: Array> = []; + for (const packIndex of manifest.startupPacks) { + const loadingPromise = this._ensurePackLoaded(packIndex); + if (loadingPromise) loadingPromises.push(loadingPromise); + } + if (!loadingPromises.length) return null; + + return Promise.all(loadingPromises).then(() => {}); + } + + private _ensurePackLoaded(packIndex: integer): Promise | null { + const manifest = this._manifest; + if (!manifest || !manifest.packs[packIndex]) return null; + + if (this._packs[packIndex]) return null; + + const existingPromise = this._loadingPromises[packIndex]; + if (existingPromise) return existingPromise; + + const generation = this._generation; + const pack = new ResourcePack(manifest.packs[packIndex]); + const loadingPromise = pack + .load((loadedBytes, totalBytes) => { + if (generation !== this._generation) return; + this._pendingDownloads.set(packIndex, { loadedBytes, totalBytes }); + this._reportProgress(); + }) + .then(() => { + if (generation !== this._generation) { + // The packs were released while this one was downloading. + pack.dispose(); + return; + } + this._packs[packIndex] = pack; + this._pendingDownloads.delete(packIndex); + }) + .catch((error) => { + if (generation !== this._generation) throw error; + // Forget the failed download, so that the retries done by the + // resource loader actually try again. + this._loadingPromises[packIndex] = null; + this._pendingDownloads.delete(packIndex); + throw error; + }); + + this._loadingPromises[packIndex] = loadingPromise; + return loadingPromise; + } + + private _reportProgress(): void { + const onProgress = this._onProgress; + if (!onProgress) return; + + let loadedBytes = 0; + let totalBytes = 0; + for (const pendingDownload of this._pendingDownloads.values()) { + loadedBytes += pendingDownload.loadedBytes; + totalBytes += pendingDownload.totalBytes; + } + if (totalBytes) onProgress(loadedBytes, totalBytes); + } + + /** + * @returns the `blob:` URL to read this file from its pack, or null if the + * file is not packed or its pack is not downloaded yet. + */ + getObjectUrl(filePath: string): string | null { + const manifest = this._manifest; + if (!manifest) return null; + + const packIndex = manifest.files[filePath]; + if (packIndex === undefined) return null; + + const pack = this._packs[packIndex]; + if (!pack) return null; + + return pack.getObjectUrl(filePath); + } + + /** + * Release every pack that holds none of the given files. + * + * Called when scenes are unloaded: as each scene has its own pack, the + * memory used by the archive of a scene that is not needed anymore can be + * given back. + */ + unloadPacksWithNoFileIn(stillLoadedFilePaths: Set): void { + for (let packIndex = 0; packIndex < this._packs.length; packIndex++) { + const pack = this._packs[packIndex]; + if (!pack) continue; + // A pack being downloaded must not be disposed: the promise waiting for + // it would then resolve on a pack that gives out nothing. + if (this._pendingDownloads.has(packIndex)) continue; + + const isStillNeeded = pack + .getFilePaths() + .some((filePath) => stillLoadedFilePaths.has(filePath)); + if (isStillNeeded) continue; + + pack.dispose(); + this._packs[packIndex] = null; + this._loadingPromises[packIndex] = null; + } + } + + /** + * Release every downloaded pack, but keep the manifest so that they are + * downloaded again when needed. Used when hot-reloading. + */ + unloadAllPacks(): void { + this._generation++; + for (let packIndex = 0; packIndex < this._packs.length; packIndex++) { + const pack = this._packs[packIndex]; + if (pack) pack.dispose(); + this._packs[packIndex] = null; + this._loadingPromises[packIndex] = null; + } + this._pendingDownloads.clear(); + } + + dispose(): void { + this.unloadAllPacks(); + this._manifest = null; + this._packs = []; + this._loadingPromises = []; + } + } +} diff --git a/GDJS/Runtime/fontfaceobserver-font-manager/fontfaceobserver-font-manager.ts b/GDJS/Runtime/fontfaceobserver-font-manager/fontfaceobserver-font-manager.ts index 0a55f1e1e806..0ba2d24aa4ec 100644 --- a/GDJS/Runtime/fontfaceobserver-font-manager/fontfaceobserver-font-manager.ts +++ b/GDJS/Runtime/fontfaceobserver-font-manager/fontfaceobserver-font-manager.ts @@ -98,12 +98,13 @@ namespace gdjs { */ private _loadFont(fontFamily: string, src: string): Promise { const descriptors = {}; - const srcWithUrl = 'url(' + encodeURI(src) + ')'; + const fullUrl = this._resourceLoader.getFullUrl(src); + const srcWithUrl = 'url(' + encodeURI(fullUrl) + ')'; // @ts-ignore if (typeof FontFace !== 'undefined') { // Load the given font using CSS Font Loading API. - return fetch(this._resourceLoader.getFullUrl(src), { + return fetch(fullUrl, { credentials: this._resourceLoader.checkIfCredentialsRequired(src) ? // Any resource stored on the GDevelop Cloud buckets needs the "credentials" of the user, // i.e: its gdevelop.io cookie, to be passed. diff --git a/GDJS/Runtime/howler-sound-manager/howler-sound-manager.ts b/GDJS/Runtime/howler-sound-manager/howler-sound-manager.ts index 9f6f8ba2b33d..5da2fb9ad414 100644 --- a/GDJS/Runtime/howler-sound-manager/howler-sound-manager.ts +++ b/GDJS/Runtime/howler-sound-manager/howler-sound-manager.ts @@ -17,6 +17,49 @@ namespace gdjs { logger.error('Error while loading an audio file: ' + error), }; + /** + * The file extensions Howler knows how to check support for. + * See https://github.com/goldfire/howler.js#format-array- + */ + const supportedAudioFormats = [ + 'mp3', + 'mpeg', + 'opus', + 'ogg', + 'oga', + 'wav', + 'aac', + 'caf', + 'm4a', + 'm4b', + 'mp4', + 'weba', + 'webm', + 'dolby', + 'flac', + ]; + + /** + * Howler guesses the codec of a sound from the extension of its URL. This + * does not work when the game resources were packed at export: the sound is + * then read from a `blob:` URL, which has no extension. Tell Howler the + * format explicitly, using the name the resource file had. + */ + const getAudioFormats = (file: string): Array | undefined => { + const lastDotIndex = file.lastIndexOf('.'); + if (lastDotIndex === -1) return undefined; + + const extension = file + .slice(lastDotIndex + 1) + .toLowerCase() + // A resource file can keep a search parameter when it comes from a URL. + .replace(/[?#].*$/, ''); + + return supportedAudioFormats.indexOf(extension) === -1 + ? undefined + : [extension]; + }; + /** * Ensure the volume is between 0 and 1. */ @@ -656,6 +699,7 @@ namespace gdjs { container[file] = new Howl( Object.assign({}, HowlParameters, { src: this._getSoundUrlsFromResource(resource), + format: getAudioFormats(resource.file), onload: resolve, onloaderror: (soundId: number, error?: string) => reject(error), html5: isMusic, @@ -751,6 +795,7 @@ namespace gdjs { Object.assign( { src: this._getSoundUrlsFromResource(resource), + format: getAudioFormats(resource.file), html5: isMusic, xhr: { withCredentials: @@ -790,6 +835,7 @@ namespace gdjs { Object.assign( { src: this._getSoundUrlsFromResource(resource), + format: getAudioFormats(resource.file), html5: isMusic, xhr: { withCredentials: @@ -1104,13 +1150,16 @@ namespace gdjs { throw error; } } else if ( - resource.preloadInCache || - // Force downloading of sounds. - // TODO Decide if sounds should be allowed to be downloaded after the scene starts. - // - they should be requested automatically at the end of the scene loading - // - they will be downloaded while the scene is playing - // - other scenes will be pre-loaded only when all the sounds for the current scene are in cache - !resource.preloadAsMusic + // A file read from a resource pack is already in memory: requesting it + // to put it in the browser cache would only copy it for nothing. + !this._resourceLoader.isFileInResourcePack(resource.file) && + (resource.preloadInCache || + // Force downloading of sounds. + // TODO Decide if sounds should be allowed to be downloaded after the scene starts. + // - they should be requested automatically at the end of the scene loading + // - they will be downloaded while the scene is playing + // - other scenes will be pre-loaded only when all the sounds for the current scene are in cache + !resource.preloadAsMusic) ) { // preloading as sound already does a XHR request, hence "else if" try { @@ -1124,7 +1173,9 @@ namespace gdjs { resolve(undefined); } else { reject( - `HTTP error while preloading audio file in cache. Status is ${sound.status}.` + `HTTP error while preloading audio file in cache. Status is ${ + sound.status + }.` ); } }); diff --git a/GDJS/Runtime/pixi-renderers/pixi-image-manager.ts b/GDJS/Runtime/pixi-renderers/pixi-image-manager.ts index 7e28fb2379e8..f491fb9c59fb 100644 --- a/GDJS/Runtime/pixi-renderers/pixi-image-manager.ts +++ b/GDJS/Runtime/pixi-renderers/pixi-image-manager.ts @@ -426,18 +426,21 @@ namespace gdjs { // to continue, otherwise if we try to play the video too soon (at the beginning of scene for instance), // it will fail. await new Promise((resolve, reject) => { - const texture = PIXI.Texture.from(resourceUrl, { - resourceOptions: { - crossorigin: this._resourceLoader.checkIfCredentialsRequired( - resource.file - ) - ? 'use-credentials' - : 'anonymous', - autoPlay: false, - }, - }).on('error', (error) => { - reject(error); + // The resource is explicitly built as a video one: `PIXI.Texture.from` + // picks the kind of resource from the file extension of the URL, + // and there is none when the game resources were packed at export + // (the video is then read from a `blob:` URL). + const videoResource = new PIXI.VideoResource(resourceUrl, { + crossorigin: this._resourceLoader.checkIfCredentialsRequired( + resource.file + ) + ? 'use-credentials' + : 'anonymous', + autoPlay: false, }); + const texture = new PIXI.Texture( + new PIXI.BaseTexture(videoResource) + ); const baseTexture = texture.baseTexture; baseTexture diff --git a/GDJS/tests/karma.conf.js b/GDJS/tests/karma.conf.js index f1b0da5796e3..4224d980cb36 100644 --- a/GDJS/tests/karma.conf.js +++ b/GDJS/tests/karma.conf.js @@ -69,6 +69,7 @@ module.exports = function (config) { './newIDE/app/resources/GDJS/Runtime/fontfaceobserver-font-manager/fontfaceobserver-font-manager.js', './newIDE/app/resources/GDJS/Runtime/Model3DManager.js', './newIDE/app/resources/GDJS/Runtime/jsonmanager.js', + './newIDE/app/resources/GDJS/Runtime/ResourcePackManager.js', './newIDE/app/resources/GDJS/Runtime/ResourceLoader.js', './newIDE/app/resources/GDJS/Runtime/ResourceCache.js', './newIDE/app/resources/GDJS/Runtime/timemanager.js', diff --git a/GDJS/tests/tests/ResourcePackManager.js b/GDJS/tests/tests/ResourcePackManager.js new file mode 100644 index 000000000000..c3b4c6dcd165 --- /dev/null +++ b/GDJS/tests/tests/ResourcePackManager.js @@ -0,0 +1,362 @@ +// @ts-check + +/** + * Tests for gdjs.ResourcePackManager, and its integration in gdjs.ResourceLoader. + * + * The packs read here are built exactly like the exporter builds them, see + * `newIDE/app/src/ExportAndShare/ResourcePacking/PackFormat.js`. + */ +describe('gdjs.ResourcePackManager', () => { + const PACK_HEADER_SIZE = 12; + const PACK_ALIGNMENT = 16; + + const alignUp = value => { + const remainder = value % PACK_ALIGNMENT; + return remainder === 0 ? value : value + (PACK_ALIGNMENT - remainder); + }; + + /** + * @param {string} file + * @returns {ResourceData} + */ + const makeResourceData = file => ({ + kind: 'fake-resource-kind-for-testing-only', + name: file, + metadata: '', + file, + userAdded: true, + }); + + /** + * A scene with no resource of its own. + * @param {string} name + * @returns {LayoutData} + */ + const makeEmptySceneData = name => ({ + r: 0, + v: 0, + b: 0, + mangledName: name, + name, + objects: [], + objectsGroups: [], + layers: [], + instances: [], + behaviorsSharedData: [], + stopSoundsOnStartup: false, + title: '', + variables: [], + usedResources: [], + uiSettings: { + grid: false, + gridType: 'rectangular', + gridWidth: 10, + gridHeight: 10, + gridDepth: 10, + gridOffsetX: 0, + gridOffsetY: 0, + gridOffsetZ: 0, + gridColor: 0, + gridAlpha: 1, + snap: false, + }, + }); + + /** + * Build a ".gdpak" archive and return an URL to download it from. + * @param {Array<{path: string, content: string, type: string}>} files + * @returns {string} + */ + const createPackUrl = files => { + const encoder = new TextEncoder(); + const contents = files.map(file => encoder.encode(file.content)); + + // The offsets depend on the length of the index, which depends on the + // offsets: grow the index until it settles. + let indexByteLength = 0; + let indexJson = ''; + const entries = files.map((file, index) => ({ + path: file.path, + offset: 0, + size: contents[index].length, + type: file.type, + })); + for (let attempt = 0; attempt < 8; attempt++) { + let offset = alignUp(PACK_HEADER_SIZE + indexByteLength); + for (const entry of entries) { + entry.offset = offset; + offset = alignUp(offset + entry.size); + } + indexJson = JSON.stringify({ files: entries }); + const newIndexByteLength = encoder.encode(indexJson).length; + if (newIndexByteLength <= indexByteLength) break; + indexByteLength = newIndexByteLength; + } + + const contentStart = alignUp(PACK_HEADER_SIZE + indexByteLength); + const packBytes = new Uint8Array( + entries.length + ? alignUp( + entries[entries.length - 1].offset + + entries[entries.length - 1].size + ) + : contentStart + ); + packBytes.set(encoder.encode('GDPK'), 0); + const view = new DataView(packBytes.buffer); + view.setUint32(4, 1, true); + view.setUint32(8, indexByteLength, true); + packBytes.set( + encoder.encode( + indexJson + + ' '.repeat(indexByteLength - encoder.encode(indexJson).length) + ), + PACK_HEADER_SIZE + ); + entries.forEach((entry, index) => { + packBytes.set(contents[index], entry.offset); + }); + + return URL.createObjectURL( + new Blob([packBytes], { type: 'application/octet-stream' }) + ); + }; + + /** @type {Array} */ + let createdUrls = []; + + const createPackedGame = (files, extraResourceFiles = []) => { + const packUrl = createPackUrl(files); + createdUrls.push(packUrl); + + gdjs.resourcePacks = { + version: 1, + packs: [packUrl], + files: files.reduce((filesMap, file) => { + filesMap[file.path] = 0; + return filesMap; + }, {}), + }; + + const allFiles = [...files.map(({ path }) => path), ...extraResourceFiles]; + return gdjs.getPixiRuntimeGame({ + resources: { + resources: allFiles.map(filePath => ({ + kind: 'fake-resource-kind-for-testing-only', + name: filePath, + metadata: '', + file: filePath, + userAdded: true, + })), + }, + }); + }; + + afterEach(() => { + gdjs.resourcePacks = null; + createdUrls.forEach(url => URL.revokeObjectURL(url)); + createdUrls = []; + }); + + it('reads a file back from a pack, keeping its content and its MIME type', async () => { + const runtimeGame = createPackedGame([ + { path: 'a.png', content: 'content of a', type: 'image/png' }, + { path: 'b.mp3', content: 'the content of b', type: 'audio/mpeg' }, + ]); + const resourceLoader = runtimeGame.getResourceLoader(); + + await resourceLoader.ensurePackLoadedFor(makeResourceData('a.png')); + + const aUrl = resourceLoader.getFullUrl('a.png'); + expect(aUrl.startsWith('blob:')).to.be(true); + const aResponse = await fetch(aUrl); + expect(await aResponse.text()).to.be('content of a'); + expect(aResponse.headers.get('Content-Type')).to.be('image/png'); + + // Every file of the pack is available once it is downloaded. + const bResponse = await fetch(resourceLoader.getFullUrl('b.mp3')); + expect(await bResponse.text()).to.be('the content of b'); + expect(bResponse.headers.get('Content-Type')).to.be('audio/mpeg'); + }); + + it('hands out the same URL for a file, so that caches stay valid', async () => { + const runtimeGame = createPackedGame([ + { path: 'a.png', content: 'content of a', type: 'image/png' }, + ]); + const resourceLoader = runtimeGame.getResourceLoader(); + await resourceLoader.ensurePackLoadedFor(makeResourceData('a.png')); + + expect(resourceLoader.getFullUrl('a.png')).to.be( + resourceLoader.getFullUrl('a.png') + ); + }); + + it('downloads a pack only once, even for concurrent requests', async () => { + const runtimeGame = createPackedGame([ + { path: 'a.png', content: 'content of a', type: 'image/png' }, + { path: 'b.png', content: 'content of b', type: 'image/png' }, + ]); + const resourceLoader = runtimeGame.getResourceLoader(); + + await Promise.all([ + resourceLoader.ensurePackLoadedFor(makeResourceData('a.png')), + resourceLoader.ensurePackLoadedFor(makeResourceData('b.png')), + resourceLoader.ensurePackLoadedFor(makeResourceData('a.png')), + ]); + + // Both files come from the same downloaded archive. + expect( + await (await fetch(resourceLoader.getFullUrl('a.png'))).text() + ).to.be('content of a'); + expect( + await (await fetch(resourceLoader.getFullUrl('b.png'))).text() + ).to.be('content of b'); + }); + + it('leaves the files that were not packed alone', async () => { + const runtimeGame = createPackedGame( + [{ path: 'a.png', content: 'content of a', type: 'image/png' }], + ['loading-screen.png'] + ); + const resourceLoader = runtimeGame.getResourceLoader(); + + expect(resourceLoader.isFileInResourcePack('a.png')).to.be(true); + expect(resourceLoader.isFileInResourcePack('loading-screen.png')).to.be( + false + ); + + // A file left out of the packs keeps being downloaded on its own. + await resourceLoader.ensurePackLoadedFor( + makeResourceData('loading-screen.png') + ); + expect(resourceLoader.getFullUrl('loading-screen.png')).to.be( + 'loading-screen.png' + ); + }); + + it('does nothing for a game exported without packed resources', async () => { + gdjs.resourcePacks = null; + const runtimeGame = gdjs.getPixiRuntimeGame({ + resources: { + resources: [ + { + kind: 'fake-resource-kind-for-testing-only', + name: 'a.png', + metadata: '', + file: 'a.png', + userAdded: true, + }, + ], + }, + }); + const resourceLoader = runtimeGame.getResourceLoader(); + + await resourceLoader.ensurePackLoadedFor(makeResourceData('a.png')); + expect(resourceLoader.isFileInResourcePack('a.png')).to.be(false); + expect(resourceLoader.getFullUrl('a.png')).to.be('a.png'); + }); + + it('releases the packs when all the resources are unloaded', async () => { + const runtimeGame = createPackedGame([ + { path: 'a.png', content: 'content of a', type: 'image/png' }, + ]); + const resourceLoader = runtimeGame.getResourceLoader(); + await resourceLoader.ensurePackLoadedFor(makeResourceData('a.png')); + + const urlBeforeUnload = resourceLoader.getFullUrl('a.png'); + expect(urlBeforeUnload.startsWith('blob:')).to.be(true); + + resourceLoader.unloadAllResources(); + + // The archive is not held in memory anymore... + expect(resourceLoader.getFullUrl('a.png')).to.be('a.png'); + // ...but the game knows it can download it again. + expect(resourceLoader.isFileInResourcePack('a.png')).to.be(true); + + await resourceLoader.ensurePackLoadedFor(makeResourceData('a.png')); + const response = await fetch(resourceLoader.getFullUrl('a.png')); + expect(await response.text()).to.be('content of a'); + }); + + it('downloads the startup packs before the first scene, for resources no scene refers to', async () => { + // A sound played by name from an expression is in no `usedResources` list, + // so nothing triggers the download of its pack - and the sound manager asks + // for its URL synchronously when it is played. Without the startup packs, + // the game would ask the server for a file that is not there anymore. + const packUrl = createPackUrl([ + { path: 'dynamic.wav', content: 'the sound', type: 'audio/wav' }, + ]); + createdUrls.push(packUrl); + gdjs.resourcePacks = { + version: 1, + packs: [packUrl], + files: { 'dynamic.wav': 0 }, + startupPacks: [0], + }; + + const runtimeGame = gdjs.getPixiRuntimeGame({ + layouts: [makeEmptySceneData('Scene1')], + resources: { + resources: [ + { + kind: 'audio', + name: 'dynamicSound', + metadata: '', + file: 'dynamic.wav', + userAdded: true, + }, + ], + }, + }); + const resourceLoader = runtimeGame.getResourceLoader(); + + // Before the game starts, the pack is not downloaded yet. + expect(resourceLoader.getFullUrl('dynamic.wav')).to.be('dynamic.wav'); + + await runtimeGame.loadFirstAssetsAndStartBackgroundLoading('Scene1'); + + const url = resourceLoader.getFullUrl('dynamic.wav'); + expect(url.startsWith('blob:')).to.be(true); + expect(await (await fetch(url)).text()).to.be('the sound'); + }); + + it('fails clearly when a pack cannot be downloaded, and allows retrying', async () => { + gdjs.resourcePacks = { + version: 1, + packs: ['this-pack-does-not-exist.gdpak'], + files: { 'a.png': 0 }, + }; + const runtimeGame = gdjs.getPixiRuntimeGame({ + resources: { + resources: [ + { + kind: 'fake-resource-kind-for-testing-only', + name: 'a.png', + metadata: '', + file: 'a.png', + userAdded: true, + }, + ], + }, + }); + const resourceLoader = runtimeGame.getResourceLoader(); + + let firstError = null; + try { + await resourceLoader.ensurePackLoadedFor(makeResourceData('a.png')); + } catch (error) { + firstError = error; + } + expect(firstError).not.to.be(null); + + // The failed download must not be remembered, otherwise the retries done by + // the resource loader would all resolve to the same failure. + let secondError = null; + try { + await resourceLoader.ensurePackLoadedFor(makeResourceData('a.png')); + } catch (error) { + secondError = error; + } + expect(secondError).not.to.be(null); + }); +}); diff --git a/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserCordovaExport.js b/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserCordovaExport.js index be8bc7c868c8..e76b74c3e21a 100644 --- a/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserCordovaExport.js +++ b/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserCordovaExport.js @@ -11,6 +11,7 @@ import { downloadUrlFilesToBlobFiles, archiveFiles, } from '../../Utils/BrowserArchiver'; +import { packResourcesInBlobFiles } from '../ResourcePacking/BrowserResourcePacker'; import { type ExportFlowProps, type ExportPipeline, @@ -135,14 +136,27 @@ export const browserCordovaExportPipeline: ExportPipeline< })); }, - launchCompression: ( + launchCompression: async ( context: ExportPipelineContext, { textFiles, blobFiles }: ResourcesDownloadOutput ): Promise => { + const basePath = '/export/'; + + // Gather the resources into a few ".gdpak" archives before zipping, so + // that the archive holds a few files rather than one per resource. + const filesToArchive = context.packResources + ? await packResourcesInBlobFiles({ + textFiles, + blobFiles, + basePath: basePath + 'www/', + onProgress: context.updateStepProgress, + }) + : { textFiles, blobFiles }; + return archiveFiles({ - blobFiles, - textFiles, - basePath: '/export/', + blobFiles: filesToArchive.blobFiles, + textFiles: filesToArchive.textFiles, + basePath, onProgress: context.updateStepProgress, }); }, diff --git a/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserElectronExport.js b/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserElectronExport.js index a830f414961e..26d38f5ca3be 100644 --- a/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserElectronExport.js +++ b/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserElectronExport.js @@ -11,6 +11,7 @@ import { downloadUrlFilesToBlobFiles, archiveFiles, } from '../../Utils/BrowserArchiver'; +import { packResourcesInBlobFiles } from '../ResourcePacking/BrowserResourcePacker'; import { type ExportFlowProps, type ExportPipeline, @@ -135,14 +136,27 @@ export const browserElectronExportPipeline: ExportPipeline< })); }, - launchCompression: ( + launchCompression: async ( context: ExportPipelineContext, { textFiles, blobFiles }: ResourcesDownloadOutput ): Promise => { + const basePath = '/export/'; + + // Gather the resources into a few ".gdpak" archives before zipping, so + // that the archive holds a few files rather than one per resource. + const filesToArchive = context.packResources + ? await packResourcesInBlobFiles({ + textFiles, + blobFiles, + basePath: basePath + 'app/', + onProgress: context.updateStepProgress, + }) + : { textFiles, blobFiles }; + return archiveFiles({ - blobFiles, - textFiles, - basePath: '/export/', + blobFiles: filesToArchive.blobFiles, + textFiles: filesToArchive.textFiles, + basePath, onProgress: context.updateStepProgress, }); }, diff --git a/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserFacebookInstantGamesExport.js b/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserFacebookInstantGamesExport.js index 7a8c65a2694e..e6d8ced5dd04 100644 --- a/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserFacebookInstantGamesExport.js +++ b/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserFacebookInstantGamesExport.js @@ -11,6 +11,7 @@ import { downloadUrlFilesToBlobFiles, archiveFiles, } from '../../Utils/BrowserArchiver'; +import { packResourcesInBlobFiles } from '../ResourcePacking/BrowserResourcePacker'; import { type ExportFlowProps, type ExportPipeline, @@ -136,14 +137,27 @@ export const browserFacebookInstantGamesExportPipeline: ExportPipeline< })); }, - launchCompression: ( + launchCompression: async ( context: ExportPipelineContext, { textFiles, blobFiles }: ResourcesDownloadOutput ): Promise => { + const basePath = '/export/'; + + // Gather the resources into a few ".gdpak" archives before zipping, so + // that the archive holds a few files rather than one per resource. + const filesToArchive = context.packResources + ? await packResourcesInBlobFiles({ + textFiles, + blobFiles, + basePath: basePath + '', + onProgress: context.updateStepProgress, + }) + : { textFiles, blobFiles }; + return archiveFiles({ - blobFiles, - textFiles, - basePath: '/export/', + blobFiles: filesToArchive.blobFiles, + textFiles: filesToArchive.textFiles, + basePath, onProgress: context.updateStepProgress, }); }, diff --git a/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserHTML5Export.js b/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserHTML5Export.js index f6c1f7aa9c65..cdf2172bcc23 100644 --- a/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserHTML5Export.js +++ b/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserHTML5Export.js @@ -26,6 +26,7 @@ import { DoneFooter, ExportFlow, } from '../GenericExporters/HTML5Export'; +import { packResourcesInBlobFiles } from '../ResourcePacking/BrowserResourcePacker'; const gd: libGDevelop = global.gd; @@ -133,14 +134,27 @@ export const browserHTML5ExportPipeline: ExportPipeline< })); }, - launchCompression: ( + launchCompression: async ( context: ExportPipelineContext, { textFiles, blobFiles }: ResourcesDownloadOutput ): Promise => { + const basePath = '/export/'; + + // Gather the resources into a few ".gdpak" archives before zipping, so + // that the zip stays below the file count limit of hosting services. + const filesToArchive = context.packResources + ? await packResourcesInBlobFiles({ + textFiles, + blobFiles, + basePath, + onProgress: context.updateStepProgress, + }) + : { textFiles, blobFiles }; + return archiveFiles({ - blobFiles, - textFiles, - basePath: '/export/', + blobFiles: filesToArchive.blobFiles, + textFiles: filesToArchive.textFiles, + basePath, onProgress: context.updateStepProgress, }); }, diff --git a/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserOnlineWebExport.js b/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserOnlineWebExport.js index 0f4007c19287..6a8f0f21b6a1 100644 --- a/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserOnlineWebExport.js +++ b/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserOnlineWebExport.js @@ -17,6 +17,7 @@ import { downloadUrlFilesToBlobFiles, archiveFiles, } from '../../Utils/BrowserArchiver'; +import { packResourcesInBlobFiles } from '../ResourcePacking/BrowserResourcePacker'; import { type ExportPipeline, type ExportPipelineContext, @@ -135,14 +136,27 @@ export const browserOnlineWebExportPipeline: ExportPipeline< })); }, - launchCompression: ( + launchCompression: async ( context: ExportPipelineContext, { textFiles, blobFiles }: ResourcesDownloadOutput ): Promise => { + const basePath = '/export/'; + + // Gather the resources into a few ".gdpak" archives before zipping, so + // that the archive holds a few files rather than one per resource. + const filesToArchive = context.packResources + ? await packResourcesInBlobFiles({ + textFiles, + blobFiles, + basePath: basePath + '', + onProgress: context.updateStepProgress, + }) + : { textFiles, blobFiles }; + return archiveFiles({ - blobFiles, - textFiles, - basePath: '/export/', + blobFiles: filesToArchive.blobFiles, + textFiles: filesToArchive.textFiles, + basePath, onProgress: context.updateStepProgress, sizeLimit: 250 * 1000 * 1000, }); diff --git a/newIDE/app/src/ExportAndShare/ExportPipeline.flow.js b/newIDE/app/src/ExportAndShare/ExportPipeline.flow.js index 59a305e1dcef..243a4900bea9 100644 --- a/newIDE/app/src/ExportAndShare/ExportPipeline.flow.js +++ b/newIDE/app/src/ExportAndShare/ExportPipeline.flow.js @@ -13,6 +13,11 @@ export type ExportPipelineContext = {| exportState: ExportState, updateStepProgress: (count: number, total: number) => void, i18n: I18nType, + /** + * Gather the game resources into a few ".gdpak" archives instead of leaving + * one file per resource (see `ResourcePacking`). Comes from the preferences. + */ + packResources: boolean, |}; export type HeaderProps = {| diff --git a/newIDE/app/src/ExportAndShare/Headless/ExportLocalHtml5Headless.js b/newIDE/app/src/ExportAndShare/Headless/ExportLocalHtml5Headless.js index e279e012b60e..72a0602bacbf 100644 --- a/newIDE/app/src/ExportAndShare/Headless/ExportLocalHtml5Headless.js +++ b/newIDE/app/src/ExportAndShare/Headless/ExportLocalHtml5Headless.js @@ -29,6 +29,12 @@ type Options = {| project: gdProject, i18n: I18nType, outputDir?: string, + /** + * Gather the game resources into a few ".gdpak" archives, so that the export + * stays below the file count limit of hosting services. On by default, as + * the preference is in the editor. + */ + packResources?: boolean, |}; type Result = {| outputDir: string |}; @@ -38,6 +44,7 @@ export const exportLocalHtml5Headless = async ({ project, i18n, outputDir, + packResources = true, }: Options): Promise => { const resolvedOutputDir = outputDir || resolveHtml5OutputDir(project); project.setLastCompilationDirectory(resolvedOutputDir); @@ -47,6 +54,7 @@ export const exportLocalHtml5Headless = async ({ exportState: { outputDir: resolvedOutputDir }, updateStepProgress: (count: number, total: number) => {}, i18n, + packResources, }; const preparedExporter = await localHTML5ExportPipeline.prepareExporter( diff --git a/newIDE/app/src/ExportAndShare/LocalExporters/LocalCordovaExport.js b/newIDE/app/src/ExportAndShare/LocalExporters/LocalCordovaExport.js index 489c121a72f7..f6f31bf39f05 100644 --- a/newIDE/app/src/ExportAndShare/LocalExporters/LocalCordovaExport.js +++ b/newIDE/app/src/ExportAndShare/LocalExporters/LocalCordovaExport.js @@ -20,12 +20,14 @@ import { ExportFlow, } from '../GenericExporters/CordovaExport'; import { downloadUrlsToLocalFiles } from '../../Utils/LocalFileDownloader'; +import { packResourcesInFolder } from '../ResourcePacking/LocalResourcePacker'; // It's important to use remote and not electron for folder actions, // otherwise they will be opened in the background. // See https://github.com/electron/electron/issues/4349#issuecomment-777475765 const remote = optionalRequire('@electron/remote'); const shell = remote ? remote.shell : null; +const path = optionalRequire('path'); const gd: libGDevelop = global.gd; type ExportState = { @@ -161,11 +163,21 @@ export const localCordovaExportPipeline: ExportPipeline< return null; }, - launchCompression: ( + launchCompression: async ( context: ExportPipelineContext, exportOutput: ResourcesDownloadOutput ): Promise => { - return Promise.resolve(null); + // The export is a folder, so there is nothing to compress. This is where + // the resources are gathered into a few ".gdpak" archives instead, now + // that the ones stored as URLs have been downloaded. + if (context.packResources) { + await packResourcesInFolder({ + exportDir: path.join(context.exportState.outputDir, 'www'), + onProgress: context.updateStepProgress, + }); + } + + return null; }, renderDoneFooter: ({ exportState }) => { diff --git a/newIDE/app/src/ExportAndShare/LocalExporters/LocalElectronExport.js b/newIDE/app/src/ExportAndShare/LocalExporters/LocalElectronExport.js index 051664af8749..952f6474a952 100644 --- a/newIDE/app/src/ExportAndShare/LocalExporters/LocalElectronExport.js +++ b/newIDE/app/src/ExportAndShare/LocalExporters/LocalElectronExport.js @@ -20,12 +20,14 @@ import { ExportFlow, } from '../GenericExporters/ElectronExport'; import { downloadUrlsToLocalFiles } from '../../Utils/LocalFileDownloader'; +import { packResourcesInFolder } from '../ResourcePacking/LocalResourcePacker'; // It's important to use remote and not electron for folder actions, // otherwise they will be opened in the background. // See https://github.com/electron/electron/issues/4349#issuecomment-777475765 const remote = optionalRequire('@electron/remote'); const shell = remote ? remote.shell : null; +const path = optionalRequire('path'); const gd: libGDevelop = global.gd; type ExportState = { @@ -161,11 +163,21 @@ export const localElectronExportPipeline: ExportPipeline< return null; }, - launchCompression: ( + launchCompression: async ( context: ExportPipelineContext, exportOutput: ResourcesDownloadOutput ): Promise => { - return Promise.resolve(null); + // The export is a folder, so there is nothing to compress. This is where + // the resources are gathered into a few ".gdpak" archives instead, now + // that the ones stored as URLs have been downloaded. + if (context.packResources) { + await packResourcesInFolder({ + exportDir: path.join(context.exportState.outputDir, 'app'), + onProgress: context.updateStepProgress, + }); + } + + return null; }, renderDoneFooter: ({ exportState }) => { diff --git a/newIDE/app/src/ExportAndShare/LocalExporters/LocalFacebookInstantGamesExport.js b/newIDE/app/src/ExportAndShare/LocalExporters/LocalFacebookInstantGamesExport.js index 9df9bc6dac28..9c537e290dcf 100644 --- a/newIDE/app/src/ExportAndShare/LocalExporters/LocalFacebookInstantGamesExport.js +++ b/newIDE/app/src/ExportAndShare/LocalExporters/LocalFacebookInstantGamesExport.js @@ -21,6 +21,7 @@ import { ExportFlow, } from '../GenericExporters/FacebookInstantGamesExport'; import { downloadUrlsToLocalFiles } from '../../Utils/LocalFileDownloader'; +import { packResourcesInFolder } from '../ResourcePacking/LocalResourcePacker'; const path = optionalRequire('path'); // It's important to use remote and not electron for folder actions, @@ -184,10 +185,19 @@ export const localFacebookInstantGamesExportPipeline: ExportPipeline< return { temporaryOutputDir }; }, - launchCompression: ( + launchCompression: async ( context: ExportPipelineContext, { temporaryOutputDir }: ResourcesDownloadOutput ): Promise => { + // Gather the resources into a few ".gdpak" archives before zipping, so + // that the archive holds a few files rather than one per resource. + if (context.packResources) { + await packResourcesInFolder({ + exportDir: temporaryOutputDir, + onProgress: context.updateStepProgress, + }); + } + return archiveLocalFolder({ path: temporaryOutputDir, outputFilename: context.exportState.archiveOutputFilename, diff --git a/newIDE/app/src/ExportAndShare/LocalExporters/LocalHTML5Export.js b/newIDE/app/src/ExportAndShare/LocalExporters/LocalHTML5Export.js index f1730ca25355..15b8c41a79fa 100644 --- a/newIDE/app/src/ExportAndShare/LocalExporters/LocalHTML5Export.js +++ b/newIDE/app/src/ExportAndShare/LocalExporters/LocalHTML5Export.js @@ -20,6 +20,7 @@ import { ExportFlow, } from '../GenericExporters/HTML5Export'; import { downloadUrlsToLocalFiles } from '../../Utils/LocalFileDownloader'; +import { packResourcesInFolder } from '../ResourcePacking/LocalResourcePacker'; import DismissableTutorialMessage from '../../Hints/DismissableTutorialMessage'; // It's important to use remote and not electron for folder actions, @@ -163,11 +164,21 @@ export const localHTML5ExportPipeline: ExportPipeline< return null; }, - launchCompression: ( + launchCompression: async ( context: ExportPipelineContext, exportOutput: ResourcesDownloadOutput ): Promise => { - return Promise.resolve(null); + // The export is a folder, so there is nothing to compress. This is where + // the resources are gathered into a few ".gdpak" archives instead, now + // that the ones stored as URLs have been downloaded. + if (context.packResources) { + await packResourcesInFolder({ + exportDir: context.exportState.outputDir, + onProgress: context.updateStepProgress, + }); + } + + return null; }, renderDoneFooter: ({ exportState }) => { diff --git a/newIDE/app/src/ExportAndShare/LocalExporters/LocalOnlineCordovaExport.js b/newIDE/app/src/ExportAndShare/LocalExporters/LocalOnlineCordovaExport.js index e785f7f05c5c..b864f24d7c32 100644 --- a/newIDE/app/src/ExportAndShare/LocalExporters/LocalOnlineCordovaExport.js +++ b/newIDE/app/src/ExportAndShare/LocalExporters/LocalOnlineCordovaExport.js @@ -24,6 +24,7 @@ import { ExportFlow, } from '../GenericExporters/OnlineCordovaExport'; import { downloadUrlsToLocalFiles } from '../../Utils/LocalFileDownloader'; +import { packResourcesInFolder } from '../ResourcePacking/LocalResourcePacker'; const path = optionalRequire('path'); const os = optionalRequire('os'); @@ -161,10 +162,19 @@ export const localOnlineCordovaExportPipeline: ExportPipeline< return { temporaryOutputDir }; }, - launchCompression: ( + launchCompression: async ( context: ExportPipelineContext, { temporaryOutputDir }: ResourcesDownloadOutput ): Promise => { + // Gather the resources into a few ".gdpak" archives before zipping, so + // that the archive holds a few files rather than one per resource. + if (context.packResources) { + await packResourcesInFolder({ + exportDir: path.join(temporaryOutputDir, 'www'), + onProgress: context.updateStepProgress, + }); + } + const archiveOutputDir = os.tmpdir(); return archiveLocalFolder({ path: temporaryOutputDir, diff --git a/newIDE/app/src/ExportAndShare/LocalExporters/LocalOnlineElectronExport.js b/newIDE/app/src/ExportAndShare/LocalExporters/LocalOnlineElectronExport.js index 1bb682e5984f..eeb0ec31640d 100644 --- a/newIDE/app/src/ExportAndShare/LocalExporters/LocalOnlineElectronExport.js +++ b/newIDE/app/src/ExportAndShare/LocalExporters/LocalOnlineElectronExport.js @@ -24,6 +24,7 @@ import { ExportFlow, } from '../GenericExporters/OnlineElectronExport'; import { downloadUrlsToLocalFiles } from '../../Utils/LocalFileDownloader'; +import { packResourcesInFolder } from '../ResourcePacking/LocalResourcePacker'; const path = optionalRequire('path'); const os = optionalRequire('os'); @@ -157,10 +158,19 @@ export const localOnlineElectronExportPipeline: ExportPipeline< return { temporaryOutputDir }; }, - launchCompression: ( + launchCompression: async ( context: ExportPipelineContext, { temporaryOutputDir }: ResourcesDownloadOutput ): Promise => { + // Gather the resources into a few ".gdpak" archives before zipping, so + // that the archive holds a few files rather than one per resource. + if (context.packResources) { + await packResourcesInFolder({ + exportDir: path.join(temporaryOutputDir, 'app'), + onProgress: context.updateStepProgress, + }); + } + const archiveOutputDir = os.tmpdir(); return archiveLocalFolder({ path: temporaryOutputDir, diff --git a/newIDE/app/src/ExportAndShare/LocalExporters/LocalOnlineWebExport.js b/newIDE/app/src/ExportAndShare/LocalExporters/LocalOnlineWebExport.js index 237f5bb0be4f..3e527394e5a3 100644 --- a/newIDE/app/src/ExportAndShare/LocalExporters/LocalOnlineWebExport.js +++ b/newIDE/app/src/ExportAndShare/LocalExporters/LocalOnlineWebExport.js @@ -20,6 +20,7 @@ import { } from '../ExportPipeline.flow'; import { ExplanationHeader } from '../GenericExporters/OnlineWebExport'; import { downloadUrlsToLocalFiles } from '../../Utils/LocalFileDownloader'; +import { packResourcesInFolder } from '../ResourcePacking/LocalResourcePacker'; import OnlineWebExportFlow from '../GenericExporters/OnlineWebExport/OnlineWebExportFlow'; const path = optionalRequire('path'); @@ -148,10 +149,19 @@ export const localOnlineWebExportPipeline: ExportPipeline< return { temporaryOutputDir }; }, - launchCompression: ( + launchCompression: async ( context: ExportPipelineContext, { temporaryOutputDir }: ResourcesDownloadOutput ): Promise => { + // Gather the resources into a few ".gdpak" archives before zipping, so + // that the archive holds a few files rather than one per resource. + if (context.packResources) { + await packResourcesInFolder({ + exportDir: temporaryOutputDir, + onProgress: context.updateStepProgress, + }); + } + const archiveOutputDir = os.tmpdir(); return archiveLocalFolder({ path: temporaryOutputDir, diff --git a/newIDE/app/src/ExportAndShare/ResourcePacking/BrowserResourcePacker.js b/newIDE/app/src/ExportAndShare/ResourcePacking/BrowserResourcePacker.js new file mode 100644 index 000000000000..6e3380030226 --- /dev/null +++ b/newIDE/app/src/ExportAndShare/ResourcePacking/BrowserResourcePacker.js @@ -0,0 +1,124 @@ +// @flow +import path from 'path-browserify'; +import { buildPackLayout } from './PackFormat'; +import { + RESOURCE_KINDS_NEVER_PACKED, + appendResourcePacksManifestToDataJs, + buildResourcePacksManifest, + planResourcePacks, + readProjectDataFromDataJs, +} from './index'; +import { + type BlobFileDescriptor, + type TextFileDescriptor, +} from '../../Utils/BrowserArchiver'; + +// See BrowserFileSystem for why `path.posix` is not used directly. +const pathPosix = path.posix || path; + +type Args = {| + textFiles: Array, + blobFiles: Array, + basePath: string, + onProgress: (count: number, total: number) => void, +|}; + +type Output = {| + textFiles: Array, + blobFiles: Array, +|}; + +/** + * Replace the individual resource files of an exported game by a handful of + * ".gdpak" archives, so that the game can be uploaded to services limiting the + * number of files in an archive (itch.io allows 1000). + * + * Nothing is read back into memory: a pack is a `Blob` built from the blobs of + * the files it contains, which the browser keeps where they already are. + */ +export const packResourcesInBlobFiles = async ({ + textFiles, + blobFiles, + basePath, + onProgress, +}: Args): Promise => { + const dataJsFilePath = pathPosix.join(basePath, 'data.js'); + const dataJsFile = textFiles.find( + ({ filePath }) => filePath === dataJsFilePath + ); + if (!dataJsFile) { + throw new Error( + `Could not find "${dataJsFilePath}" in the exported game, so its resources can't be packed.` + ); + } + + const plan = planResourcePacks(readProjectDataFromDataJs(dataJsFile.text), { + excludedResourceKinds: RESOURCE_KINDS_NEVER_PACKED, + }); + if (!plan.packs.length) return { textFiles, blobFiles }; + + const blobByRelativePath: Map = new Map(); + blobFiles.forEach(({ filePath, blob }) => { + blobByRelativePath.set(pathPosix.relative(basePath, filePath), blob); + }); + + const packedFilePaths: Set = new Set(); + const packBlobFiles: Array = []; + let packedCount = 0; + + for (const pack of plan.packs) { + const contents: Array<{| filePath: string, blob: Blob |}> = []; + pack.filePaths.forEach(filePath => { + const blob = blobByRelativePath.get(filePath); + // A resource can be missing when the project references a file that was + // not exported. The engine already copes with a missing resource, so skip + // it rather than failing the whole export. + if (blob) contents.push({ filePath, blob }); + }); + + packedCount++; + onProgress(packedCount, plan.packs.length); + // Writing an empty archive would only waste a file. Nothing refers to it, + // as the manifest is built from the files that were really packed. + if (!contents.length) continue; + + const layout = buildPackLayout( + contents.map(({ filePath, blob }) => ({ filePath, size: blob.size })) + ); + + const parts: Array = [layout.headerBytes]; + layout.entries.forEach((entry, index) => { + parts.push(contents[index].blob); + if (layout.paddings[index] > 0) { + parts.push(new Uint8Array(layout.paddings[index])); + } + }); + + packBlobFiles.push({ + filePath: pathPosix.join(basePath, pack.name), + blob: new Blob(parts, { type: 'application/octet-stream' }), + }); + contents.forEach(({ filePath }) => packedFilePaths.add(filePath)); + } + + // The files that made it into a pack must not be exported on their own + // anymore - that is the whole point. + const remainingBlobFiles = blobFiles.filter( + ({ filePath }) => + !packedFilePaths.has(pathPosix.relative(basePath, filePath)) + ); + + const manifest = buildResourcePacksManifest(plan, packedFilePaths); + + return { + textFiles: textFiles.map(textFile => + textFile.filePath === dataJsFilePath + ? { + filePath: textFile.filePath, + text: appendResourcePacksManifestToDataJs(textFile.text, manifest), + } + : textFile + ), + blobFiles: [...remainingBlobFiles, ...packBlobFiles], + }; +}; diff --git a/newIDE/app/src/ExportAndShare/ResourcePacking/BrowserResourcePacker.spec.js b/newIDE/app/src/ExportAndShare/ResourcePacking/BrowserResourcePacker.spec.js new file mode 100644 index 000000000000..c1496b82b97c --- /dev/null +++ b/newIDE/app/src/ExportAndShare/ResourcePacking/BrowserResourcePacker.spec.js @@ -0,0 +1,186 @@ +// @flow +import { packResourcesInBlobFiles } from './BrowserResourcePacker'; +import { parsePackIndex } from './PackFormat'; +import { readProjectDataFromDataJs } from './index'; + +const BASE_PATH = '/export/'; + +// The test environment has no `Blob`, while the browser this code runs in +// always has one. Node's implementation supports everything used here +// (`size`, `slice`, `text`, `arrayBuffer`, and Blobs as constructor parts). +beforeAll(() => { + if (typeof global.Blob === 'undefined') { + global.Blob = require('buffer').Blob; + } +}); + +const makeDataJs = (projectData: Object) => + 'gdjs.projectData = ' + + JSON.stringify(projectData) + + ';\ngdjs.runtimeGameOptions = {};\n'; + +const makeResource = (name: string, file: string, kind: string = 'image') => ({ + name, + file, + kind, + metadata: '', + userAdded: true, +}); + +const readFromPack = async (packBlob: Blob, filePath: string) => { + const packBytes = new Uint8Array(await packBlob.arrayBuffer()); + const entry = parsePackIndex(packBytes).entries.find( + entry => entry.path === filePath + ); + if (!entry) throw new Error(`"${filePath}" is not in this pack.`); + + return { + text: await packBlob.slice(entry.offset, entry.offset + entry.size).text(), + type: entry.type, + }; +}; + +describe('packResourcesInBlobFiles', () => { + const projectData = { + properties: { loadingScreen: { backgroundImageResourceName: 'splash' } }, + resources: { + resources: [ + makeResource('global', 'global.png'), + makeResource('music', 'music.mp3', 'audio'), + makeResource('menu', 'menu.png'), + makeResource('splash', 'splash.png'), + ], + }, + usedResources: [{ name: 'global' }, { name: 'music' }], + objects: [], + layouts: [{ name: 'Menu', usedResources: [{ name: 'menu' }], objects: [] }], + }; + + const makeInput = () => ({ + textFiles: [ + { filePath: '/export/data.js', text: makeDataJs(projectData) }, + { filePath: '/export/runtimegame.js', text: 'gdjs.RuntimeGame = ...' }, + ], + blobFiles: [ + { filePath: '/export/global.png', blob: new Blob(['the global image']) }, + { filePath: '/export/music.mp3', blob: new Blob(['the music']) }, + { filePath: '/export/menu.png', blob: new Blob(['the menu image']) }, + { filePath: '/export/splash.png', blob: new Blob(['the splash image']) }, + // A binary engine file, which must be left alone. + { + filePath: '/export/pixi-renderers/draco/gltf/draco_decoder.wasm', + blob: new Blob(['not a resource']), + }, + ], + basePath: BASE_PATH, + onProgress: (count: number, total: number) => {}, + }); + + it('replaces the resource blobs by packs, leaving the engine files alone', async () => { + const { textFiles, blobFiles } = await packResourcesInBlobFiles( + makeInput() + ); + + expect(blobFiles.map(({ filePath }) => filePath).sort()).toEqual([ + // The loading screen background is needed before the loading screen can + // be shown, so it stays an individual file. + '/export/pixi-renderers/draco/gltf/draco_decoder.wasm', + '/export/resources.gdpak', + '/export/scene-0.gdpak', + '/export/splash.png', + ]); + // Text files are untouched, apart from data.js. + expect( + textFiles.find(({ filePath }) => filePath === '/export/runtimegame.js') + ?.text + ).toBe('gdjs.RuntimeGame = ...'); + }); + + it('writes contents that can be read back, with their MIME type', async () => { + const { blobFiles } = await packResourcesInBlobFiles(makeInput()); + + const globalPack = blobFiles.find( + ({ filePath }) => filePath === '/export/resources.gdpak' + ); + if (!globalPack) throw new Error('The global pack was not written.'); + + expect(await readFromPack(globalPack.blob, 'global.png')).toEqual({ + text: 'the global image', + type: 'image/png', + }); + expect(await readFromPack(globalPack.blob, 'music.mp3')).toEqual({ + text: 'the music', + type: 'audio/mpeg', + }); + + const scenePack = blobFiles.find( + ({ filePath }) => filePath === '/export/scene-0.gdpak' + ); + if (!scenePack) throw new Error('The scene pack was not written.'); + expect(await readFromPack(scenePack.blob, 'menu.png')).toEqual({ + text: 'the menu image', + type: 'image/png', + }); + }); + + it('declares the packs in data.js without touching the project data', async () => { + const { textFiles } = await packResourcesInBlobFiles(makeInput()); + + const dataJs = textFiles.find( + ({ filePath }) => filePath === '/export/data.js' + ); + if (!dataJs) throw new Error('data.js is missing.'); + + expect(readProjectDataFromDataJs(dataJs.text)).toEqual(projectData); + + const manifest = JSON.parse( + dataJs.text + .slice( + dataJs.text.indexOf('gdjs.resourcePacks = ') + + 'gdjs.resourcePacks = '.length + ) + .trim() + .replace(/;$/, '') + ); + expect(manifest).toEqual({ + version: 1, + packs: ['resources.gdpak', 'scene-0.gdpak'], + files: { 'global.png': 0, 'music.mp3': 0, 'menu.png': 1 }, + // The global pack must be downloaded up front, as it holds the resources + // that no loading task refers to. + startupPacks: [0], + }); + }); + + it('leaves the export untouched when there is nothing to pack', async () => { + const emptyProjectData = { + properties: { loadingScreen: { backgroundImageResourceName: '' } }, + resources: { resources: [] }, + usedResources: [], + objects: [], + layouts: [], + }; + const { textFiles, blobFiles } = await packResourcesInBlobFiles({ + textFiles: [ + { filePath: '/export/data.js', text: makeDataJs(emptyProjectData) }, + ], + blobFiles: [], + basePath: BASE_PATH, + onProgress: (count: number, total: number) => {}, + }); + + expect(blobFiles).toEqual([]); + expect(textFiles[0].text).not.toContain('gdjs.resourcePacks'); + }); + + it('fails clearly when data.js is not in the export', async () => { + await expect( + packResourcesInBlobFiles({ + textFiles: [], + blobFiles: [], + basePath: BASE_PATH, + onProgress: (count: number, total: number) => {}, + }) + ).rejects.toThrow(/Could not find "\/export\/data.js"/); + }); +}); diff --git a/newIDE/app/src/ExportAndShare/ResourcePacking/FullExport.spec.js b/newIDE/app/src/ExportAndShare/ResourcePacking/FullExport.spec.js new file mode 100644 index 000000000000..720e13300fc7 --- /dev/null +++ b/newIDE/app/src/ExportAndShare/ResourcePacking/FullExport.spec.js @@ -0,0 +1,217 @@ +// @flow +/** + * Runs a real HTML5 export (through libGD.js and the actual exporter) and packs + * its resources, so that the whole chain is checked: the exporter writes + * `data.js` and `index.html`, the packer reads them back, and the game engine + * would find `gdjs.ResourcePackManager` in the script list. + */ +import assignIn from 'lodash/assignIn'; +import { packResourcesInFolder } from './LocalResourcePacker'; +import { parsePackIndex } from './PackFormat'; +import optionalRequire from '../../Utils/OptionalRequire'; + +const fs = optionalRequire('fs-extra'); +const path = optionalRequire('path'); +const os = optionalRequire('os'); +const process = optionalRequire('process'); + +const gd: libGDevelop = global.gd; + +// The tests are run from `newIDE/app`, where the built game engine lives. +const GDJS_ROOT = path.resolve(process.cwd(), 'resources/GDJS'); + +const addImageResource = ( + project: gdProject, + name: string, + absoluteFilePath: string +) => { + const resource = new gd.ImageResource(); + resource.setName(name); + resource.setFile(absoluteFilePath); + project.getResourcesManager().addResource(resource); + resource.delete(); +}; + +/** + * Add a sprite object using the given image, so that the resource is really + * "used" by the scene and ends up in its `usedResources`. + */ +const addSpriteObject = ( + container: gdObjectsContainer, + objectName: string, + imageResourceName: string +) => { + const object = container.insertNewObject( + // $FlowFixMe[prop-missing] - the project is the platform holder here. + global.testProject, + 'Sprite', + objectName, + container.getObjectsCount() + ); + const configuration = gd.asSpriteConfiguration(object.getConfiguration()); + const animation = new gd.Animation(); + animation.setDirectionsCount(1); + const direction = animation.getDirection(0); + const sprite = new gd.Sprite(); + sprite.setImageName(imageResourceName); + direction.addSprite(sprite); + animation.setDirection(direction, 0); + configuration.getAnimations().addAnimation(animation); + animation.delete(); + sprite.delete(); +}; + +describe('Full HTML5 export with packed resources', () => { + let workingDir = ''; + let exportDir = ''; + let project: any = null; + + beforeAll(async () => { + workingDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gdevelop-export-')); + exportDir = path.join(workingDir, 'export'); + await fs.ensureDir(exportDir); + + // Real files on disk, so that the exporter really copies them. + const assetsDir = path.join(workingDir, 'assets'); + await fs.ensureDir(assetsDir); + const imagePaths: { [string]: string } = {}; + for (const name of ['global', 'menu', 'level']) { + const filePath = path.join(assetsDir, `${name}.png`); + await fs.writeFile(filePath, `the ${name} image`, 'utf8'); + imagePaths[name] = filePath; + } + + project = gd.ProjectHelper.createNewGDJSProject(); + global.testProject = project; + project.setName('Packing test'); + + addImageResource(project, 'globalImage', imagePaths.global); + addImageResource(project, 'menuImage', imagePaths.menu); + addImageResource(project, 'levelImage', imagePaths.level); + + // A global object, so that its image lands in the project-wide resources. + addSpriteObject(project.getObjects(), 'GlobalSprite', 'globalImage'); + + const menuScene = project.insertNewLayout('Menu', 0); + addSpriteObject(menuScene.getObjects(), 'MenuSprite', 'menuImage'); + const levelScene = project.insertNewLayout('Level', 1); + addSpriteObject(levelScene.getObjects(), 'LevelSprite', 'levelImage'); + + // `LocalFileSystem` transitively imports a web worker module that expects + // `self` to exist, so it is required here rather than imported at the top. + if (typeof global.self === 'undefined') global.self = global; + const LocalFileSystem = require('../LocalExporters/LocalFileSystem') + .default; + + // Run the actual exporter, as the export pipeline does. + const localFileSystem = new LocalFileSystem({ + downloadUrlsToLocalFiles: true, + }); + const fileSystem = assignIn(new gd.AbstractFileSystemJS(), localFileSystem); + const exporter = new gd.Exporter(fileSystem, GDJS_ROOT); + const exportOptions = new gd.ExportOptions(project, exportDir); + const exportSucceeded = exporter.exportWholePixiProject(exportOptions); + exportOptions.delete(); + exporter.delete(); + + if (!exportSucceeded) throw new Error('The export failed.'); + }, 60000); + + afterAll(async () => { + if (project) project.delete(); + global.testProject = null; + if (workingDir) await fs.remove(workingDir); + }); + + it('exports a game whose index.html loads the resource pack manager', async () => { + const indexHtml = await fs.readFile( + path.join(exportDir, 'index.html'), + 'utf8' + ); + + // Without this script, `gdjs.ResourcePackManager` would be undefined and + // the game would not start. + expect(indexHtml).toContain('