diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..575d3093 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "eslint.enable": false, + "[javascript]": { + "editor.formatOnSave": false + } +} \ No newline at end of file diff --git a/README.md b/README.md index 8ffaf77c..596b3742 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ user.storage.on('read', function(filename, callback) { }); ``` -In this manner, you can save data to a database, a cloud service, or anything else you choose. +In this manner, you can save data to a database, a cloud service, or anything else you choose. If both [`savePicsCache`](#savePicsCache) and [`picsCacheAll`](#picsCacheAll) are enabled, it is possible to maintain your own Steam database this way. ### autoRelogin @@ -175,6 +175,16 @@ Added in 3.3.0. Defaults to `false`. +### savePicsCache + +If `enablePicsCache` is enabled, saves all product info from [the PICS cache](#picscache) to disk (in [`dataDirectory`](#dataDirectory)) or to your [Custom Storage Engine](#custom-storage-engine). This will significantly speed up the [`appOwnershipCached`](#appOwnershipCache) event from firing and reduce the amount product info requests to Steam. It will only save product info if it's not missing its access token. + +Added in {{TODO}} + +Defaults to `false`. + +**Warning:** Mind that this will significantly increase the storage usage and space! + ### picsCacheAll If `picsCacheAll` is enabled, `enablePicsCache` is enabled, and `changelistUpdateInterval` is nonzero, then apps and @@ -879,6 +889,7 @@ Requests a list of game servers from the master server. - `callback` - Called when requested data is available - `err` - An `Error` object on failure, or `null` on success - `apps` - An object whose keys are AppIDs and whose values are objects + - `sha` - The SHA hash of the app info in hex format (useful to compare to old data) - `changenumber` - The changenumber of the latest changelist in which this app changed - `missingToken` - `true` if you need to provide an access token to get more details about this app - `appinfo` - An object whose structure is identical to the output of `app_info_print` in the [Steam console](steam://nav/console) diff --git a/components/05-filestorage.js b/components/05-filestorage.js index 8f271ccf..490989d4 100644 --- a/components/05-filestorage.js +++ b/components/05-filestorage.js @@ -17,6 +17,19 @@ class SteamUserFileStorage extends SteamUserUtility { } } + /** + * @param {Object} files - Keys are filenames, values are Buffer objects containing the file contents + * @return {Promise} + * @protected + */ + async _saveFiles(files) { + if (!this.storage) { + return Promise.reject(new Error('Storage system disabled')); + } + + return await this.storage.saveFiles(files); + } + /** * @param {string} filename * @returns {Promise} @@ -43,7 +56,7 @@ class SteamUserFileStorage extends SteamUserUtility { */ async _readFiles(filenames) { if (!this.storage) { - return filenames.map(filename => ({filename, error: new Error('Storage system disabled')})); + return filenames.map(filename => ({ filename, error: new Error('Storage system disabled') })); } // No need for a try/catch because readFiles can't reject diff --git a/components/08-sentry.js b/components/08-sentry.js new file mode 100644 index 00000000..37e62df6 --- /dev/null +++ b/components/08-sentry.js @@ -0,0 +1,51 @@ +const StdLib = require('@doctormckay/stdlib'); + +const EMsg = require('../enums/EMsg.js'); + +const SteamUserBase = require('./00-base.js'); +const SteamUserWeb = require('./07-web.js'); + +class SteamUserSentry extends SteamUserWeb { + /** + * @param {Buffer} sentry + */ + setSentry(sentry) { + this._sentry = sentry; + } + + async _getSentryFileContent() { + let files = await this._readFiles(this._getSentryFilenames()); + let firstFile = files.find(f => f.contents); + return firstFile ? firstFile.contents : null; + } + + _getSentryFilenames() { + if (this.options.singleSentryfile) { + return ['sentry.bin']; + } else { + return [ + // prefer the steamid-named sentry file if we have it + `sentry.${this._logOnDetails._steamid}.bin`, + `sentry.${this._logOnDetails.account_name || this._logOnDetails._newAuthAccountName}.bin` + ]; + } + } +} + +// Handlers + +SteamUserBase.prototype._handlerManager.add(EMsg.ClientUpdateMachineAuth, async function(body, hdr, callback) { + // TODO: Handle partial updates + + this.emit('debug', 'Got new sentry file'); + await Promise.all(this._getSentryFilenames().map(filename => this._saveFile(filename, body.bytes))); + this.emit('sentry', body.bytes); + + // Accept the sentry + + callback(EMsg.ClientUpdateMachineAuthResponse, { + sha_file: StdLib.Hashing.sha1(body.bytes, 'buffer') + }); +}); + +module.exports = SteamUserSentry; diff --git a/components/apps.js b/components/apps.js index 206b50eb..b25f0266 100644 --- a/components/apps.js +++ b/components/apps.js @@ -60,19 +60,19 @@ class SteamUserApps extends SteamUserAppAuth { let processedApps = apps.map((app) => { if (typeof app == 'string') { - app = {game_id: '15190414816125648896', game_extra_info: app}; + app = { game_id: '15190414816125648896', game_extra_info: app }; } else if (typeof app != 'object') { - app = {game_id: app}; + app = { game_id: app }; } if (typeof app.game_ip_address == 'number') { - app.game_ip_address = {v4: app.game_ip_address}; + app.game_ip_address = { v4: app.game_ip_address }; } return app; }); - this._send(EMsg.ClientGamesPlayedWithDataBlob, {games_played: processedApps}); + this._send(EMsg.ClientGamesPlayedWithDataBlob, { games_played: processedApps }); processedApps.forEach((app) => { if (app.game_id > Math.pow(2, 32)) { @@ -113,7 +113,7 @@ class SteamUserApps extends SteamUserAppAuth { } else if (blocked) { return reject(new Error('Cannot kick other session')); } else { - return resolve({playingApp}); + return resolve({ playingApp }); } }); }); @@ -127,12 +127,12 @@ class SteamUserApps extends SteamUserAppAuth { */ getPlayerCount(appid, callback) { return StdLib.Promises.timeoutCallbackPromise(10000, ['playerCount'], callback, (resolve, reject) => { - this._send(EMsg.ClientGetNumberOfCurrentPlayersDP, {appid}, (body) => { + this._send(EMsg.ClientGetNumberOfCurrentPlayersDP, { appid }, (body) => { let err = Helpers.eresultError(body.eresult); if (err) { reject(err); } else { - resolve({playerCount: body.player_count}); + resolve({ playerCount: body.player_count }); } }); }); @@ -142,7 +142,7 @@ class SteamUserApps extends SteamUserAppAuth { * Get a list of apps or packages which have changed since a particular changenumber. * @param {int} sinceChangenumber - Changenumber to get changes since. Use 0 to get the latest changenumber, but nothing else * @param {function} [callback] - * @returns {Promise<{currentChangeNumber: number, appChanges: number[], packageChanges: number[]}>} + * @returns {Promise<{currentChangeNumber: number, appChanges: object[], packageChanges: object[]}>} */ getProductChanges(sinceChangenumber, callback) { let args = ['currentChangeNumber', 'appChanges', 'packageChanges']; @@ -161,28 +161,164 @@ class SteamUserApps extends SteamUserAppAuth { }); } + /** + * @param {apps: Object, packages: Object} - Response from getProductInfo + * @returns {Promise} + * @protected + */ + async _saveProductInfo({ apps, packages }) { + if (!this.options.savePicsCache) { + return []; + } + + let toSave = {}; + + for (let appid in apps) { + // We want to avoid saving apps that are missing a token + // Public only apps are weird... + if (apps[appid].missingToken && !(apps[appid].appinfo && apps[appid].appinfo.public_only)) { + continue; + } + let filename = `app_info_${appid}.json`; + let contents = JSON.stringify(apps[appid]); + toSave[filename] = contents; + } + + for (let packageid in packages) { + if (packages[packageid].missingToken) { + continue; + } + let filename = `package_info_${packageid}.json`; + let contents = JSON.stringify(packages[packageid]); + toSave[filename] = contents; + } + + try { + return await this._saveFiles(toSave); + } catch (ex) { + this.emit('debug', `Error saving PICS cache to disk: ${ex.message}`); + return []; + } + } + + /** + * Get cached product info. + * @param {object[]} apps - Array of apps {appid, access_token} + * @param {object[]} packages - Array of packages {packageid, access_token} + * @returns {Promise<{apps: Object, packages: Object, notCachedApps: number[], notCachedPackages: number[]}>} + * @protected + */ + async _getCachedProductInfo(apps, packages) { + let response = { + apps: {}, + packages: {}, + notCachedApps: [], + notCachedPackages: [] + }; + + // With pics cache disabled, we cannot assure pics cache is up to date. + if (!this.options.enablePicsCache) { + return response; + } + + // From this point, we can assume pics cache is up to date (via changelist updates). + for (let { appid } of apps) { + if (this.picsCache.apps[appid]) { + response.apps[appid] = this.picsCache.apps[appid]; + } else { + response.notCachedApps.push(appid); + } + } + for (let { packageid } of packages) { + if (this.picsCache.packages[packageid]) { + response.packages[packageid] = this.picsCache.packages[packageid]; + } else { + response.notCachedPackages.push(packageid); + } + } + + // If everything was already in memory cache, we're done. + if (response.notCachedApps.length === 0 && response.notCachedPackages.length === 0) { + return response; + } + + // If pics cache is not being saved to disk, we're done. + if (!this.options.savePicsCache) { + return response; + } + + // Otherwise, we try loading the missing apps & packages from disk. + let appids = response.notCachedApps; + let packageids = response.notCachedPackages; + response.notCachedApps = []; + response.notCachedPackages = []; + + let appFiles = {}; + let packageFiles = {}; + for (let appid of appids) { + let filename = `app_info_${appid}.json`; + appFiles[filename] = appid; + } + for (let packageid of packageids) { + let filename = `package_info_${packageid}.json`; + packageFiles[filename] = packageid; + } + let files = await this._readFiles(Object.keys(appFiles).concat(Object.keys(packageFiles))); + + for (let { filename, error, contents } of files) { + if (Buffer.isBuffer(contents)) { + contents = contents.toString('utf8'); + } + let appid = appFiles[filename]; + let packageid = packageFiles[filename]; + + if (appid !== undefined) { + if (error || !contents) { + response.notCachedApps.push(appid); + } else { + try { + // Save to memory cache + this.picsCache.apps[appid] = JSON.parse(contents); + response.apps[appid] = this.picsCache.apps[appid]; + } catch (ex) { + // A corrupted cache file shouldn't take down the whole request + this.emit('debug', `Error parsing cached app info for ${appid}: ${ex.message}`); + response.notCachedApps.push(appid); + } + } + } else if (packageid !== undefined) { // Remember, package ID can be 0 + if (error || !contents) { + response.notCachedPackages.push(packageid); + } else { + try { + this.picsCache.packages[packageid] = JSON.parse(contents); + response.packages[packageid] = this.picsCache.packages[packageid]; + } catch (ex) { + this.emit('debug', `Error parsing cached package info for ${packageid}: ${ex.message}`); + response.notCachedPackages.push(packageid); + } + } + } else { + this.emit('debug', `Error retrieving origins of file ${filename}`); + } + } + + return response; + } + /** * Get info about some apps and/or packages from Steam. * @param {int[]|object[]} apps - Array of AppIDs. May be empty. May also contain objects with keys {appid, access_token} * @param {int[]|object[]} packages - Array of package IDs. May be empty. May also contain objects with keys {packageid, access_token} - * @param {boolean} [inclTokens=false] - If true, automatically retrieve access tokens if needed - * @param {function} [callback] + * @param {boolean} inclTokens - If true, automatically retrieve access tokens if needed + * @param {function} callback * @param {int} [requestType] - Don't touch * @returns {Promise<{apps: Object, packages: Object, unknownApps: number[], unknownPackages: number[]}>} + * @protected */ - getProductInfo(apps, packages, inclTokens, callback, requestType) { - // Adds support for the previous syntax - if (typeof inclTokens !== 'boolean' && typeof inclTokens === 'function') { - requestType = callback; - callback = inclTokens; - inclTokens = false; - } - - // This one actually can take a while, so allow it to go as long as 60 minutes - return StdLib.Promises.timeoutCallbackPromise(3600000, ['apps', 'packages', 'unknownApps', 'unknownPackages'], callback, (resolve, reject) => { - requestType = requestType || PICSRequestType.User; - - // Steam can send us the full response in multiple responses, so we need to buffer them into one callback + _getProductInfo(apps, packages, inclTokens, callback, requestType = PICSRequestType.User) { + // Steam can send us the full response in multiple responses, so we need to buffer them into one callback + return StdLib.Promises.timeoutCallbackPromise(7200000, ['apps', 'packages', 'unknownApps', 'unknownPackages'], callback, async (resolve, reject) => { let appids = []; let packageids = []; let response = { @@ -191,55 +327,128 @@ class SteamUserApps extends SteamUserAppAuth { unknownApps: [], unknownPackages: [] }; + let cached = { + apps: {}, + packages: {}, + notCachedApps: [], + notCachedPackages: [] + }; + let shaList = { + apps: {}, + packages: {} + }; - apps = apps.map((app) => { + // Preprocess input: apps and packages + let _apps = []; + for (let app of apps) { + let appid = parseInt(typeof app === 'object' ? app.appid : app, 10); + // Ensure uniqueness to prevent nasty bugs + if (appids.includes(appid)) { + continue; + } if (typeof app === 'object') { + app.appid = appid; appids.push(app.appid); - return app; } else { - appids.push(app); - return {appid: app}; + appids.push(appid); + app = { appid }; + } + _apps.push(app); + } + apps = _apps; + + let _packages = []; + for (let pkg of packages) { + let packageid = parseInt(typeof pkg === 'object' ? pkg.packageid : pkg, 10); + // Ensure uniqueness to prevent nasty bugs + if (packageids.includes(packageid)) { + continue; } - }); - - packages = packages.map((pkg) => { if (typeof pkg === 'object') { + pkg.packageid = packageid; packageids.push(pkg.packageid); - return pkg; } else { - packageids.push(pkg); - return {packageid: pkg}; + packageids.push(packageid); + pkg = { packageid }; } - }); - - if (inclTokens) { - packages.filter(pkg => !pkg.access_token).forEach((pkg) => { + if (inclTokens && !pkg.access_token) { // Check if we have a license for this package which includes a token let license = (this.licenses || []).find(lic => lic.package_id == pkg.packageid && lic.access_token != 0); if (license) { this.emit('debug', `Using token "${license.access_token}" from license for package ${pkg.packageid}`); pkg.access_token = license.access_token; } - }); + } + _packages.push(pkg); } + packages = _packages; + + // If we have no apps or packages, we're done + if (appids.length === 0 && packageids.length === 0) { + resolve(response); + return; + } + + // Function to handle response of ClientPICSProductInfoRequest (may be called multiple times) + let onResponse = async (body) => { + // Some requested apps/packages might not exist, or we might not have access to them. Steam + // reports these separately instead of including them in body.apps/body.packages, so we need to + // remove them from our "still waiting on" lists here too. Otherwise we'd wait forever (until the + // request times out) for data that Steam is never going to send us. + (body.unknown_appids || []).forEach((appid) => { + if (!response.unknownApps.includes(appid)) { + response.unknownApps.push(appid); + } + let index = appids.indexOf(appid); + if (index != -1) { + appids.splice(index, 1); + } + }); + + (body.unknown_packageids || []).forEach((packageid) => { + if (!response.unknownPackages.includes(packageid)) { + response.unknownPackages.push(packageid); + } + let index = packageids.indexOf(packageid); + if (index != -1) { + packageids.splice(index, 1); + } + }); - this._send(EMsg.ClientPICSProductInfoRequest, {apps, packages}, async (body) => { // If we're using the PICS cache, then add the items in this response to it if (this.options.enablePicsCache) { let cache = this.picsCache; cache.apps = cache.apps || {}; cache.packages = cache.packages || {}; + // Remember (in the internal PICS cache only - see the equivalent comment further below for why + // we don't also add these to the response we hand back to the caller) that Steam doesn't have + // info for these, so that we don't treat them as "not yet fetched" forever. Without this, + // every future ownership check would keep re-warning about them (they'd never show up as + // cached), and we'd keep re-requesting them from Steam too. + (body.unknown_appids || []).forEach((appid) => { + cache.apps[appid] = cache.apps[appid] || { sha: null, changenumber: 0, missingToken: false, appinfo: null }; + }); + + (body.unknown_packageids || []).forEach((packageid) => { + cache.packages[packageid] = cache.packages[packageid] || { sha: null, changenumber: 0, missingToken: false, packageinfo: null }; + }); + (body.apps || []).forEach((app) => { - let appInfoVdf = app.buffer.toString('utf8'); - // It seems that Steam appends a NUL byte. Unsure if this is universal or not, but to make sure - // that things work regardless of whether there's a NUL byte at the end, just remove it if it's there. - appInfoVdf = appInfoVdf.replace(/\0$/, ''); + let appinfo = null; + if (app.buffer) { + let appInfoVdf = app.buffer.toString('utf8'); + // It seems that Steam appends a NUL byte. Unsure if this is universal or not, but to make sure + // that things work regardless of whether there's a NUL byte at the end, just remove it if it's there. + appInfoVdf = appInfoVdf.replace(/\0$/, ''); + appinfo = VDF.parse(appInfoVdf).appinfo; + } let data = { + sha: app.sha ? app.sha.toString('hex') : null, changenumber: app.change_number, missingToken: !!app.missing_token, - appinfo: VDF.parse(appInfoVdf).appinfo + appinfo }; if ((!cache.apps[app.appid] && requestType == PICSRequestType.Changelist) || (cache.apps[app.appid] && cache.apps[app.appid].changenumber != data.changenumber)) { @@ -248,12 +457,12 @@ class SteamUserApps extends SteamUserAppAuth { } cache.apps[app.appid] = data; - app._parsedData = data; }); (body.packages || []).forEach((pkg) => { let data = { + sha: pkg.sha ? pkg.sha.toString('hex') : null, changenumber: pkg.change_number, missingToken: !!pkg.missing_token, packageinfo: pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null @@ -264,33 +473,16 @@ class SteamUserApps extends SteamUserAppAuth { } cache.packages[pkg.packageid] = data; - pkg._parsedData = data; - // Request info for all the apps in this package, if this request didn't originate from the license list + // Request info for all the apps in this package, if this request didn't originate from the license list, because then we'll first process all packages before requesting all package contents if (requestType != PICSRequestType.Licenses) { let appids = (pkg.packageinfo || {}).appids || []; - this.getProductInfo(appids, [], false, null, PICSRequestType.PackageContents).catch(() => {}); + this.getProductInfo(appids, [], false, null, PICSRequestType.PackageContents).catch(() => { }); } }); } - (body.unknown_appids || []).forEach((appid) => { - response.unknownApps.push(appid); - let index = appids.indexOf(appid); - if (index != -1) { - appids.splice(index, 1); - } - }); - - (body.unknown_packageids || []).forEach((packageid) => { - response.unknownPackages.push(packageid); - let index = packageids.indexOf(packageid); - if (index != -1) { - packageids.splice(index, 1); - } - }); - (body.apps || []).forEach((app) => { // _parsedData will be populated if we have the PICS cache enabled. // If we don't, we need to parse the data here. @@ -301,6 +493,7 @@ class SteamUserApps extends SteamUserAppAuth { appInfoVdf = appInfoVdf.replace(/\0$/, ''); response.apps[app.appid] = app._parsedData || { + sha: app.sha ? app.sha.toString('hex') : null, changenumber: app.change_number, missingToken: !!app.missing_token, appinfo: VDF.parse(appInfoVdf).appinfo @@ -314,6 +507,7 @@ class SteamUserApps extends SteamUserAppAuth { (body.packages || []).forEach((pkg) => { response.packages[pkg.packageid] = pkg._parsedData || { + sha: pkg.sha ? pkg.sha.toString('hex') : null, changenumber: pkg.change_number, missingToken: !!pkg.missing_token, packageinfo: pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null @@ -327,77 +521,208 @@ class SteamUserApps extends SteamUserAppAuth { // appids and packageids contain the list of IDs that we're still waiting on data for if (appids.length === 0 && packageids.length === 0) { - if (!inclTokens) { - return resolve(response); - } + this._saveProductInfo(response); + let combined = { + apps: Object.assign(cached.apps, response.apps), + packages: Object.assign(cached.packages, response.packages), + unknownApps: response.unknownApps, + unknownPackages: response.unknownPackages + }; + return resolve(combined); + } + }; - // We want tokens - let tokenlessAppids = []; - let tokenlessPackages = []; + // If we don't use PICS cache or require fresh product info for changelist request, then just perform a normal request + if (!this.options.enablePicsCache || requestType == PICSRequestType.Changelist) { + return this._send(EMsg.ClientPICSProductInfoRequest, { apps, packages }, onResponse); + } - for (let appid in response.apps) { - if (response.apps[appid].missingToken) { - tokenlessAppids.push(parseInt(appid, 10)); - } - } + cached = await this._getCachedProductInfo(apps, packages); + // Note: This callback can be called multiple times + this._send(EMsg.ClientPICSProductInfoRequest, { apps, packages, meta_data_only: true }, async (body) => { + (body.apps || []).forEach((app) => { + shaList.apps[app.appid] = app.sha ? app.sha.toString('hex') : null; + }); - for (let packageid in response.packages) { - if (response.packages[packageid].missingToken) { - tokenlessPackages.push(parseInt(packageid, 10)); - } + (body.packages || []).forEach((pkg) => { + shaList.packages[pkg.packageid] = pkg.sha ? pkg.sha.toString('hex') : null; + }); + + (body.unknown_appids || []).forEach((appid) => { + if (!response.unknownApps.includes(appid)) { + response.unknownApps.push(appid); } + }); - if (tokenlessAppids.length == 0 && tokenlessPackages.length == 0) { - // No tokens needed - return resolve(response); + (body.unknown_packageids || []).forEach((packageid) => { + if (!response.unknownPackages.includes(packageid)) { + response.unknownPackages.push(packageid); } + }); - try { - let { - appTokens, - packageTokens - } = await this.getProductAccessToken(tokenlessAppids, tokenlessPackages); - let tokenApps = []; - let tokenPackages = []; - - for (let appid in appTokens) { - tokenApps.push({appid: parseInt(appid, 10), access_token: appTokens[appid]}); - } + // Remember (in the internal PICS cache only - not in the response we hand back to the caller, + // since the public contract is that unknown ids show up in unknownApps/unknownPackages, not in + // apps/packages) that Steam doesn't have info for these, instead of leaving them permanently + // absent from the cache. See the equivalent comment in the non-cached onResponse function above + // for why that matters. + if (this.options.enablePicsCache) { + let cache = this.picsCache; + cache.apps = cache.apps || {}; + cache.packages = cache.packages || {}; - for (let packageid in packageTokens) { - tokenPackages.push({ - packageid: parseInt(packageid, 10), - access_token: packageTokens[packageid] - }); - } + (body.unknown_appids || []).forEach((appid) => { + cache.apps[appid] = cache.apps[appid] || { sha: null, changenumber: 0, missingToken: false, appinfo: null }; + }); - // Now we have the tokens. Request the data. - let {apps, packages} = await this.getProductInfo(tokenApps, tokenPackages, false); - for (let appid in apps) { - response.apps[appid] = apps[appid]; - let index = response.unknownApps.indexOf(parseInt(appid, 10)); - if (index != -1) { - response.unknownApps.splice(index, 1); - } - } + (body.unknown_packageids || []).forEach((packageid) => { + cache.packages[packageid] = cache.packages[packageid] || { sha: null, changenumber: 0, missingToken: false, packageinfo: null }; + }); + } - for (let packageid in packages) { - response.packages[packageid] = packages[packageid]; - let index = response.unknownPackages.indexOf(parseInt(packageid, 10)); - if (index != -1) { - response.unknownPackages.splice(index, 1); + let appTotal = Object.keys(shaList.apps).length + response.unknownApps.length; + let packageTotal = Object.keys(shaList.packages).length + response.unknownPackages.length; + + // If our collected totals match the (unique) total we requested + if (appTotal === appids.length && packageTotal === packageids.length) { + // Filter out any apps & packages we already have cached and do not need to be refreshed + apps = apps.filter((app) => !response.unknownApps.includes(app.appid) && (cached.apps[app.appid] || {}).sha !== shaList.apps[app.appid]); + packages = packages.filter((pkg) => !response.unknownPackages.includes(pkg.packageid) && (cached.packages[pkg.packageid] || {}).sha !== shaList.packages[pkg.packageid]); + // console.log( + // "requestType:", requestType, + // "app request:", appids.length, + // "pkg request:", packageids.length, + // "app unknown:", response.unknownApps.length, + // "pkg unknown:", response.unknownPackages.length, + // "app cache:", Object.keys(cached.apps).length, + // "pkg cache:", Object.keys(cached.packages).length, + // "app to refresh:", apps.length, + // "pkg to refresh:", packages.length, + // ); + + // If we have nothing to refresh / no stale data (e.g. all of them were unknown) + if (apps.length === 0 && packages.length === 0) { + this._saveProductInfo(response); + let combined = { + apps: Object.assign(cached.apps, response.apps), + packages: Object.assign(cached.packages, response.packages), + unknownApps: response.unknownApps, + unknownPackages: response.unknownPackages + }; + return resolve(combined); + } + + // We want tokens + if (inclTokens) { + let _appids = apps.map(app => app.appid); + let _packageids = packages.map(pkg => pkg.packageid); + let tokenlessAppids = body.apps.filter(app => _appids.includes(app.appid) && !!app.missing_token).map(app => app.appid); + let tokenlessPackages = body.packages.filter(pkg => _packageids.includes(pkg.packageid) && !!pkg.missing_token).map(pkg => pkg.packageid); + if (tokenlessAppids.length > 0 || tokenlessPackages.length > 0) { + try { + let { + appTokens, + packageTokens + } = await this.getProductAccessToken(tokenlessAppids, tokenlessPackages); + let tokenApps = {}; + let tokenPackages = {}; + + for (let appid in appTokens) { + tokenApps[appid] = { + appid: parseInt(appid, 10), + access_token: appTokens[appid] + }; + } + + for (let packageid in packageTokens) { + tokenPackages[packageid] = { + packageid: parseInt(packageid, 10), + access_token: packageTokens[packageid] + }; + } + + // Replace products to request with included tokens + apps = apps.filter(app => !appTokens[app.appid]).concat(Object.values(tokenApps)); + packages = packages.filter(pkg => !tokenPackages[pkg.packageid]).concat(Object.values(tokenPackages)); + } catch (ex) { + return reject(ex); } } - - resolve(response); - } catch (ex) { - return reject(ex); } + + appids = apps.map(app => app.appid); + packageids = packages.map(pkg => pkg.packageid); + + // Request the apps & packages we need to refresh + this._send(EMsg.ClientPICSProductInfoRequest, { apps, packages }, onResponse); } }); }); } + /** + * Get info about some apps and/or packages from Steam, but first split it into chunks + * @param {int[]|object[]} apps - Array of AppIDs. May be empty. May also contain objects with keys {appid, access_token} + * @param {int[]|object[]} packages - Array of package IDs. May be empty. May also contain objects with keys {packageid, access_token} + * @param {boolean} [inclTokens=false] - If true, automatically retrieve access tokens if needed + * @param {function} [callback] + * @param {int} [requestType] - Don't touch + * @returns {Promise<{apps: Object, packages: Object, unknownApps: number[], unknownPackages: number[]}>} + */ + getProductInfo(apps, packages, inclTokens, callback, requestType) { + // Adds support for the previous syntax + if (typeof inclTokens !== 'boolean' && typeof inclTokens === 'function') { + requestType = callback; + callback = inclTokens; + inclTokens = false; + } + + // Add support for optional callback + if (!requestType && typeof callback !== 'function') { + requestType = callback; + callback = undefined; + } + + requestType = requestType || PICSRequestType.User; + + // This one actually can take a while, so allow it to go as long as 120 minutes + return StdLib.Promises.timeoutCallbackPromise(7200000, ['apps', 'packages', 'unknownApps', 'unknownPackages'], callback, async (resolve, reject) => { + try { + let response = { + apps: {}, + packages: {}, + unknownApps: [], + unknownPackages: [] + }; + // Split apps + packages into chunks of 2000 + let chunkSize = 2000; + for (let i = 0; i < packages.length; i += chunkSize) { + let packagesChunk = packages.slice(i, i + chunkSize); + // Do not include callback in the request, it will be called multiple times + let result = await this._getProductInfo([], packagesChunk, inclTokens, null, requestType); + response = { + apps: Object.assign(response.apps, result.apps), + packages: Object.assign(response.packages, result.packages), + unknownApps: response.unknownApps.concat(result.unknownApps), + unknownPackages: response.unknownPackages.concat(result.unknownPackages) + }; + } + for (let i = 0; i < apps.length; i += chunkSize) { + let appsChunk = apps.slice(i, i + chunkSize); + let result = await this._getProductInfo(appsChunk, [], inclTokens, null, requestType); + response = { + apps: Object.assign(response.apps, result.apps), + packages: Object.assign(response.packages, result.packages), + unknownApps: response.unknownApps.concat(result.unknownApps), + unknownPackages: response.unknownPackages.concat(result.unknownPackages) + }; + } + resolve(response); + } catch (ex) { + return reject(ex); + } + }); + } + /** * Get access tokens for some apps and/or packages * @param {int[]} apps - Array of appids @@ -469,6 +794,12 @@ class SteamUserApps extends SteamUserAppAuth { return; } + // First wait for ownership cache to be loaded, so we can calculate ourApps & ourPackages correctly. + if (!this.picsCache.ownershipModified) { + this._resetChangelistUpdateTimer(); + return; + } + let result = null; try { result = await this.getProductChanges(this.picsCache.changenumber); @@ -479,7 +810,7 @@ class SteamUserApps extends SteamUserAppAuth { } let cache = this.picsCache; - let {appChanges, packageChanges, currentChangeNumber} = result; + let { appChanges, packageChanges, currentChangeNumber } = result; cache.apps = cache.apps || {}; cache.packages = cache.packages || {}; @@ -498,6 +829,9 @@ class SteamUserApps extends SteamUserAppAuth { } cache.changenumber = currentChangeNumber; + if (this.options.savePicsCache) { + this._saveFile('changenumber.txt', currentChangeNumber); + } this._resetChangelistUpdateTimer(); return; } @@ -513,26 +847,46 @@ class SteamUserApps extends SteamUserAppAuth { this.emit('changelist', currentChangeNumber, appChanges, packageChanges); - let {appTokens, packageTokens} = result; + let { appTokens, packageTokens } = result; cache.changenumber = currentChangeNumber; + if (this.options.savePicsCache) { + this._saveFile('changenumber.txt', currentChangeNumber); + } this._resetChangelistUpdateTimer(); let index = -1; for (let appid in appTokens) { if (Object.hasOwnProperty.call(appTokens, appid) && (index = ourApps.indexOf(parseInt(appid, 10))) != -1) { - ourApps[index] = {appid: parseInt(appid, 10), access_token: appTokens[appid]}; + ourApps[index] = { appid: parseInt(appid, 10), access_token: appTokens[appid] }; } } for (let packageid in packageTokens) { if (Object.hasOwnProperty.call(packageTokens, packageid) && (index = ourPackages.indexOf(parseInt(packageid, 10))) != -1) { - ourPackages[index] = {packageid: parseInt(packageid, 10), access_token: packageTokens[packageid]}; + ourPackages[index] = { packageid: parseInt(packageid, 10), access_token: packageTokens[packageid] }; } } - // Add a no-op catch in case there's some kind of error - this.getProductInfo(ourApps, ourPackages, false, null, PICSRequestType.Changelist).catch(() => { + // Fall back to an empty package list in case there's some kind of error. Note: the fallback must be an + // object with a "packages" key, or the destructure below throws (since it can't destructure undefined). + let { packages } = await this.getProductInfo(ourApps, ourPackages, false, null, PICSRequestType.Changelist).catch((ex) => { + this.emit('debug', `Error retrieving product info for changelist packages: ${ex.message}`); + return { packages: {} }; }); + + // Request info for all the apps in these packages + let appids = []; + + for (let pkgid in packages) { + ((packages[pkgid].packageinfo || {}).appids || []).filter(appid => !appids.includes(appid)).forEach(appid => appids.push(appid)); + } + + try { + // Request type is changelist, because we need to refresh their pics cache + await this.getProductInfo(appids, [], true, undefined, PICSRequestType.Changelist); + } catch (ex) { + this.emit('debug', `Error retrieving product info of changed apps: ${ex.message}`); + } } /** @@ -549,7 +903,7 @@ class SteamUserApps extends SteamUserAppAuth { return; } - this.getProductInfo([appid], [], false, null, PICSRequestType.AddToCache).catch(() => { + this.getProductInfo([appid], [], true, null, PICSRequestType.AddToCache).catch(() => { }); } @@ -567,6 +921,48 @@ class SteamUserApps extends SteamUserAppAuth { } } + /** + * Same as _warn(), but only emits a given message once for the lifetime of this client. Ownership checks + * (ownsApp, ownsDepot, getOwnedApps, getOwnedDepots, and filtered getOwnedPackages calls) recompute their + * result from scratch on every call and aren't cached, so a single package that's missing PICS info would + * otherwise produce the same warning over and over if the caller checks ownership of many things in a loop. + * @param {string} message + * @protected + */ + _warnOnce(message) { + this._warnedMessages = this._warnedMessages || new Set(); + if (this._warnedMessages.has(message)) { + return; + } + + this._warnedMessages.add(message); + this._warn(message); + } + + /** + * Look up a package's packageinfo in the PICS cache, warning (at most once per distinct message) if it isn't + * available. Shared by getOwnedApps, getOwnedDepots, and the ownership filter function, which all previously + * had their own copy of this same not-in-cache/no-package-info check. + * @param {int} packageid + * @param {string} context - Used in the warning message, e.g. 'get owned apps for', 'filter' + * @returns {object|null} - packageinfo, or null if it isn't available + * @protected + */ + _getCachedPackageInfo(packageid, context) { + let cached = this.picsCache.packages[packageid]; + if (!cached) { + this._warnOnce(`Failed to ${context} package ${packageid}: not in cache`); + return null; + } + + if (!cached.packageinfo) { + this._warnOnce(`Failed to ${context} package ${packageid}: no package info`); + return null; + } + + return cached.packageinfo; + } + /** * @protected */ @@ -588,8 +984,8 @@ class SteamUserApps extends SteamUserAppAuth { return; } - let {packages} = result; - // Request info for all the apps in these packages + let { packages } = result; + // Request info for all the apps in these packages, and only after all packages have been processed let appids = []; for (let pkgid in packages) { @@ -629,20 +1025,13 @@ class SteamUserApps extends SteamUserAppAuth { let ownedPackages = this.getOwnedPackages(filter); let appids = {}; - ownedPackages.forEach((pkg) => { - if (!this.picsCache.packages[pkg]) { - this._warn(`Failed to get owned apps for package ${pkg}`); - return; - } - - pkg = this.picsCache.packages[pkg]; - if (!pkg.packageinfo) { - this._warn(`Failed to get owned apps for package ${pkg}`); + ownedPackages.forEach((pkgid) => { + let packageinfo = this._getCachedPackageInfo(pkgid, 'get owned apps for'); + if (!packageinfo) { return; } - pkg = pkg.packageinfo; - (pkg.appids || []).forEach((appid) => { + (packageinfo.appids || []).forEach((appid) => { if (!appids[appid]) { appids[appid] = true; } @@ -677,20 +1066,13 @@ class SteamUserApps extends SteamUserAppAuth { let ownedPackages = this.getOwnedPackages(filter); let depotids = {}; - ownedPackages.forEach((pkg) => { - if (!this.picsCache.packages[pkg]) { - this._warn(`Failed to get owned depots for package ${pkg}`); + ownedPackages.forEach((pkgid) => { + let packageinfo = this._getCachedPackageInfo(pkgid, 'get owned depots for'); + if (!packageinfo) { return; } - pkg = this.picsCache.packages[pkg]; - if (!pkg.packageinfo) { - this._warn(`Failed to get owned depots for package ${pkg}`); - return; - } - - pkg = pkg.packageinfo; - (pkg.depotids || []).forEach(function(depotid) { + (packageinfo.depotids || []).forEach((depotid) => { if (!depotids[depotid]) { depotids[depotid] = true; } @@ -757,7 +1139,7 @@ class SteamUserApps extends SteamUserAppAuth { } // If filter options is a boolean, we asssume it's excludeSharedLicenses if (typeof filter === 'boolean') { - filter = {excludeShared: filter}; + filter = { excludeShared: filter }; } // New behavior from this point on @@ -814,14 +1196,11 @@ class SteamUserApps extends SteamUserAppAuth { return false; } - let id = license.package_id; - if (!this.picsCache.packages[id] || !this.picsCache.packages[id].packageinfo) { - this._warn(`Failed to filter package ${id} (no PICS cache info available)`); + let pkg = this._getCachedPackageInfo(license.package_id, 'filter'); + if (!pkg) { return false; } - let pkg = this.picsCache.packages[id].packageinfo; - // If exclude all free (sub 0 is covered by NoCost) if (filter.excludeFree && freeLicenseBillingTypes.includes(pkg.billingtype)) { return false; @@ -886,7 +1265,7 @@ class SteamUserApps extends SteamUserAppAuth { */ redeemKey(key, callback) { return StdLib.Promises.timeoutCallbackPromise(90000, ['purchaseResultDetails', 'packageList'], callback, (resolve, reject) => { - this._send(EMsg.ClientRegisterKey, {key: key}, (body) => { + this._send(EMsg.ClientRegisterKey, { key: key }, (body) => { let packageList = {}; let receiptDetails = BinaryKVParser.parse(body.purchase_receipt_info).MessageObject; @@ -924,7 +1303,7 @@ class SteamUserApps extends SteamUserAppAuth { } return StdLib.Promises.timeoutCallbackPromise(10000, ['grantedPackageIds', 'grantedAppIds'], callback, (resolve, reject) => { - this._send(EMsg.ClientRequestFreeLicense, {appids: appIDs}, (body) => { + this._send(EMsg.ClientRequestFreeLicense, { appids: appIDs }, (body) => { if (body.eresult != EResult.OK) { reject(Helpers.eresultError(body.eresult)); } else { @@ -972,7 +1351,7 @@ class SteamUserApps extends SteamUserAppAuth { return reject(new Error(`Incorrect key length: expected ${keyLength - 1} but got ${key.length}`)); } - return resolve({key}); + return resolve({ key }); }); }); } @@ -983,15 +1362,13 @@ SteamUserBase.prototype._handlerManager.add(EMsg.ClientLicenseList, function(bod this.licenses = body.licenses; // Request info for our licenses - if (this.options.enablePicsCache) { - this._getLicenseInfo(); - } + this._getLicenseInfo(); }); SteamUserBase.prototype._handlerManager.add(EMsg.ClientPlayingSessionState, function(body) { this._playingBlocked = body.playing_blocked; this.emit('playingState', body.playing_blocked, body.playing_app); - this.playingState = {blocked: body.playing_blocked, appid: body.playing_app}; + this.playingState = { blocked: body.playing_blocked, appid: body.playing_app }; }); function sortNumeric(a, b) { diff --git a/index.js b/index.js index 68e9d045..4c0bc4ac 100644 --- a/index.js +++ b/index.js @@ -86,6 +86,18 @@ class SteamUser extends SteamUserTwoFactor { this.storage = new FileManager(this.options.dataDirectory); } + if (this.options.savePicsCache) { + this._readFile('changenumber.txt').then((changenumber) => { + if (changenumber && isFinite(changenumber)) { + this.picsCache.changenumber = parseInt(changenumber, 10); + } + }); + } + + if (this.options.picsCacheAll && this.options.savePicsCache) { + this._warn('Both picsCacheAll and savePicsCache are enabled. Unless a custom storage engine is used, beware that this will cause a lot of disk IO and space usage!'); + } + this._initialized = true; } diff --git a/package.json b/package.json index 34a1650e..f5f3732e 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "adm-zip": "^0.5.10", "binarykvparser": "^2.2.0", "bytebuffer": "^5.0.0", - "file-manager": "^2.0.0", + "file-manager": "^2.0.1", "kvparser": "^1.0.1", "lzma": "^2.3.2", "protobufjs": "^7.2.4", @@ -60,4 +60,4 @@ "engines": { "node": ">=14.0.0" } -} +} \ No newline at end of file diff --git a/resources/default_options.js b/resources/default_options.js index bdd51483..979e14ba 100644 --- a/resources/default_options.js +++ b/resources/default_options.js @@ -8,6 +8,7 @@ const EMachineIDType = require('./EMachineIDType.js'); * @property {EMachineIDType} [machineIdType] * @property {string[]} [machineIdFormat] * @property {boolean} [enablePicsCache=false] + * @property {boolean} [savePicsCache=false] * @property {boolean} [picsCacheAll=false] * @property {number} [changelistUpdateInterval=60000] * @property {PackageFilter|PackageFilterFunction|null} [ownershipFilter=null] @@ -28,6 +29,7 @@ module.exports = { machineIdType: EMachineIDType.AccountNameGenerated, machineIdFormat: ['SteamUser Hash BB3 {account_name}', 'SteamUser Hash FF2 {account_name}', 'SteamUser Hash 3B3 {account_name}'], enablePicsCache: false, + savePicsCache: false, picsCacheAll: false, changelistUpdateInterval: 60000, additionalHeaders: {}, diff --git a/scripts/prepublish.js b/scripts/prepublish.js new file mode 100644 index 00000000..87bbe936 --- /dev/null +++ b/scripts/prepublish.js @@ -0,0 +1,40 @@ +// This file is run by npm prior to the package being published to the registry + +// Update the CM list +download('https://api.steampowered.com/ISteamDirectory/GetCMList/v1/?format=json&cellid=0', function(data) { + let json = JSON.parse(data); + + if (!json.response || json.response.result != 1) { + throw new Error('Cannot get CM list'); + } + + let servers = { + tcp_servers: json.response.serverlist, + websocket_servers: json.response.serverlist_websockets, + time: Date.now() + }; + + console.log('Got list of ' + servers.tcp_servers.length + ' TCP CMs and ' + servers.websocket_servers.length + ' WebSocket CMs from WebAPI'); + require('fs').writeFileSync(__dirname + '/../resources/servers.json', JSON.stringify(servers, null, '\t')); +}); + + +// Helper functions +function download(url, callback) { + let reqData = require('url').parse(url); + reqData.servername = reqData.hostname; + reqData.headers = { 'User-Agent': 'node-steam-user data parser' }; + reqData.method = 'GET'; + + // This will crash if there's an error. But that's fine. + require('https').request(reqData, function(res) { + let data = ''; + res.on('data', function(chunk) { + data += chunk; + }); + + res.on('end', function() { + callback(data); + }); + }).end(); +}