diff --git a/backend/apps/cloud/src/analytics/analytics.controller.ts b/backend/apps/cloud/src/analytics/analytics.controller.ts index c43ee7eb0..dd95b567e 100644 --- a/backend/apps/cloud/src/analytics/analytics.controller.ts +++ b/backend/apps/cloud/src/analytics/analytics.controller.ts @@ -111,6 +111,7 @@ import { GetKeywordsDto } from './dto/get-keywords.dto' import { GetBotStatsDto } from './dto/get-bot-stats.dto' import { GSC_ALL_TIME_DAYS, GSCService } from '../project/gsc.service' import { GetProfileIdDto, GetSessionIdDto } from './dto/get-id.dto' +import { IdentifyDto } from './dto/identify.dto' import { ExperimentService } from '../experiment/experiment.service' import { ExperimentStatus, @@ -3297,6 +3298,104 @@ export class AnalyticsController { return { ...result, appliedFilters, timeBucket: timeBucketForAllTime } } + /** + * Links the visitor's current anonymous profile to an identified profile + * derived from the user ID supplied by the site (e.g. after log in). Events + * previously recorded for the anonymous profile get attributed to the + * identified profile at query time; the tracker stamps all subsequent + * events with the supplied profileId directly. + * + * Optional traits (email, plan, ...) are stored against the identified + * profile and shown on its dashboard page. + */ + @Post('identify') + @Public() + async identify( + @Body() dto: IdentifyDto, + @Headers() headers, + @Ip() reqIP, + ): Promise<{ profileId: string } | typeof BOT_RESPONSE> { + const { 'user-agent': userAgent, origin } = headers + const ip = getIPFromHeaders(headers) || reqIP || '' + + await checkRateLimit(ip, 'identify', 120, 60) + await checkRateLimit(dto.pid, 'identify', 2000, 60) + + const botResult = await this.analyticsService.checkBot( + dto.pid, + userAgent, + headers, + ip, + headers.referer || headers.referrer, + null, + 'identify', + ) + + if (botResult.isBot) { + return BOT_RESPONSE + } + + const profileId = this.analyticsService.validateUserSuppliedProfileId( + dto.profileId, + ) + + await this.analyticsService.validate(dto, origin, ip) + + const userProfileId = await this.analyticsService.generateProfileId( + dto.pid, + userAgent, + ip, + profileId, + ) + + if (!_isEmpty(dto.traits)) { + await this.analyticsService.saveProfileTraits( + dto.pid, + userProfileId, + dto.traits, + ) + } + + const anonProfileId = await this.analyticsService.generateProfileId( + dto.pid, + userAgent, + ip, + ) + + const linked = await this.analyticsService.linkProfiles( + dto.pid, + anonProfileId, + userProfileId, + ) + + // Flip the visitor's current session (if any) to the identified profile. + // Skipped when the anonymous profile is already linked to a different + // identified profile (e.g. a second account on a shared device) - the + // session stays with the current identity until new events re-stamp it. + if (linked) { + const { exists, psid } = await this.analyticsService.getSessionId( + dto.pid, + userAgent, + ip, + ) + + if (exists) { + await this.analyticsService.recordSessionActivity( + psid, + dto.pid, + userProfileId, + ) + } + } + + this.logger.log( + `pid: ${dto.pid}, profileId: ${userProfileId}`, + 'POST /analytics/identify', + ) + + return { profileId: userProfileId } + } + // Revenue attribution endpoints @Post('profile-id') diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index 08cde15b7..afcd32fe9 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -116,6 +116,7 @@ import { import { ErrorDto } from './dto/error.dto' import { GetPagePropertyMetaDto } from './dto/get-page-property-meta.dto' import { DATA_DELETION_EVENT_TYPES } from './dto/data-deletion.dto' +import { MAX_USER_PROFILE_ID_LENGTH } from './dto/identify.dto' import { ProjectViewCustomEventMetaValueType } from '../project/entity/project-view-custom-event.entity' import { ProjectViewCustomEventDto } from '../project/dto/create-project-view.dto' import { UAParser } from '@ua-parser-js/pro-business' @@ -2399,11 +2400,14 @@ export class AnalyticsService { userSupplied?: string, ): Promise { if (userSupplied) { - const cleanId = userSupplied - .replace(AnalyticsService.PROFILE_PREFIX_ANON, '') - .replace(AnalyticsService.PROFILE_PREFIX_USER, '') - const hash = this.hashToNumericString(`${cleanId}${pid}`) - return `${AnalyticsService.PROFILE_PREFIX_USER}${hash}` + const normalised = this.normaliseUserSuppliedProfileId(userSupplied) + + if (normalised) { + return `${AnalyticsService.PROFILE_PREFIX_USER}${normalised}` + } + + // Unusable identifier - fall back to the anonymous fingerprint rather + // than fusing every visitor sending it into one profile. } const salt = await this.saltService.getSaltForProfile() @@ -2417,6 +2421,358 @@ export class AnalyticsService { return profileId.startsWith(AnalyticsService.PROFILE_PREFIX_USER) } + // Values that are almost certainly instrumentation bugs rather than real user + // IDs (e.g. stringified nulls). Identifying with them would fuse unrelated + // visitors into a single profile. + private static readonly ILLEGAL_USER_PROFILE_IDS = new Set([ + '', + 'null', + 'undefined', + 'nan', + 'none', + 'nil', + 'true', + 'false', + '0', + 'anonymous', + 'anon', + 'guest', + 'user', + 'id', + 'distinct_id', + 'distinctid', + 'profileid', + 'profile_id', + 'not_authenticated', + 'email', + '[object object]', + ]) + + // Control (\p{Cc}) and format (\p{Cf}) characters never appear in a genuine + // identifier, but would corrupt log lines and let a profile masquerade as + // another one in the dashboard (e.g. via a right-to-left override). + private static readonly UNPRINTABLE_REGEX = /[\p{Cc}\p{Cf}]/u + + // Traits accumulate across identify() calls, so the number of distinct keys + // a profile ends up with is bounded on read rather than on write. + static readonly MAX_PROFILE_TRAITS = 100 + + /** + * Normalises a profile ID supplied by the site (via identify() or the + * `profileId` field of an event). The value is stored as-is behind the + * `usr_` prefix - it belongs to the site, and hashing it would only make + * the dashboard unreadable. + * + * @returns the identifier to store, or null when it can't be used as one. + */ + normaliseUserSuppliedProfileId(userSupplied: string): string | null { + if (typeof userSupplied !== 'string') { + return null + } + + const trimmed = userSupplied.trim() + + if ( + !trimmed || + trimmed.length > MAX_USER_PROFILE_ID_LENGTH || + AnalyticsService.UNPRINTABLE_REGEX.test(trimmed) || + AnalyticsService.ILLEGAL_USER_PROFILE_IDS.has(trimmed.toLowerCase()) + ) { + return null + } + + return trimmed + } + + validateUserSuppliedProfileId(userSupplied: string): string { + const normalised = this.normaliseUserSuppliedProfileId(userSupplied) + + if (!normalised) { + const echoed = String(userSupplied ?? '').slice(0, 64) + + throw new BadRequestException( + `"${echoed}" is not a valid profileId. Pass a unique, stable identifier of the user (e.g. an internal user ID) of up to ${MAX_USER_PROFILE_ID_LENGTH} characters, see https://docs.swetrix.com/visitor-identification`, + ) + } + + return normalised + } + + private getProfileAliasKey(pid: string, anonProfileId: string): string { + return `pfa:${pid}:${anonProfileId}` + } + + // Sentinel cached for anonymous profiles with no alias, so unlinked (the + // overwhelmingly common case) lookups don't hit ClickHouse every time. + // Cannot collide with a real value - those always start with usr_. + private static readonly PROFILE_ALIAS_NONE = '-' + + /** + * Looks up the identified (usr_) profile an anonymous (anon_) profile has + * been linked to. First identification wins, so the mapping is resolved + * with argMin(created). + */ + async getUserProfileForAnon( + pid: string, + anonProfileId: string, + ): Promise { + const cacheKey = this.getProfileAliasKey(pid, anonProfileId) + + const cached = await redis.get(cacheKey) + + if (cached) { + return cached === AnalyticsService.PROFILE_ALIAS_NONE ? null : cached + } + + const query = ` + SELECT argMin(userProfileId, created) AS userProfileId + FROM profile_aliases + WHERE pid = {pid:FixedString(12)} + AND anonProfileId = {anonProfileId:String} + ` + + const { data } = await clickhouse + .query({ + query, + query_params: { pid, anonProfileId }, + }) + .then((resultSet) => resultSet.json<{ userProfileId: string | null }>()) + + const userProfileId = data[0]?.userProfileId || null + + if (userProfileId) { + await redis.set(cacheKey, userProfileId, 'EX', 3600) + } else { + // Short TTL: a link created concurrently on another node (linkProfiles + // uses async_insert, so the row may not be SELECTable yet) must become + // visible shortly after this sentinel expires. + await redis.set(cacheKey, AnalyticsService.PROFILE_ALIAS_NONE, 'EX', 60) + } + + return userProfileId + } + + /** + * Links an anonymous (anon_) profile to an identified (usr_) profile. + * The link is created only once per anonymous profile - if it is already + * linked to a different user, the existing link is kept (first + * identification wins; e.g. a second person logging in on a shared device + * must not steal the anonymous history of the first). + * + * @returns whether the anonymous profile is linked to the supplied user + * profile after the call + */ + async linkProfiles( + pid: string, + anonProfileId: string, + userProfileId: string, + ): Promise { + const existing = await this.getUserProfileForAnon(pid, anonProfileId) + + if (existing) { + return existing === userProfileId + } + + try { + await clickhouse.insert({ + table: 'profile_aliases', + format: 'JSONEachRow', + values: [ + { + pid, + anonProfileId, + userProfileId, + created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'), + }, + ], + clickhouse_settings: { async_insert: 1 }, + }) + + await redis.set( + this.getProfileAliasKey(pid, anonProfileId), + userProfileId, + 'EX', + 3600, + ) + } catch (error) { + this.logger.error(`[linkProfiles] Failed to link profiles: ${error}`) + return false + } + + return true + } + + private getProfileTraitsKey(pid: string, profileId: string): string { + return `pft:${pid}:${profileId}` + } + + /** + * Stores the traits (arbitrary key/value metadata a site attaches to a user - + * email, plan, signup date, ...) of an identified profile. + * + * Traits are merged per key: a later call only overwrites the keys it + * carries, and an empty value removes a trait. Storing one row per key lets + * ClickHouse handle that without a read-modify-write cycle. + */ + async saveProfileTraits( + pid: string, + profileId: string, + traits: Record, + ): Promise { + const keys = _sortBy(_keys(traits)) + + if (_isEmpty(keys)) { + return + } + + // identify() is called on every page load, so the same traits arrive over + // and over again - skip the insert when nothing changed since the last one. + const cacheKey = this.getProfileTraitsKey(pid, profileId) + const fingerprint = this.hashToNumericString( + JSON.stringify(_map(keys, (key) => [key, traits[key]])), + ) + + try { + if ((await redis.get(cacheKey)) === fingerprint) { + return + } + + // Millisecond precision: `created` is the ReplacingMergeTree version, so + // two updates of the same trait within a second must still be ordered. + const created = dayjs.utc().format('YYYY-MM-DD HH:mm:ss.SSS') + + await clickhouse.insert({ + table: 'profile_traits', + format: 'JSONEachRow', + values: _map(keys, (key) => ({ + pid, + profileId, + key, + value: traits[key], + created, + })), + clickhouse_settings: { async_insert: 1 }, + }) + + await redis.set(cacheKey, fingerprint, 'EX', 3600) + } catch (error) { + this.logger.error(`[saveProfileTraits] Failed to save traits: ${error}`) + } + } + + /** + * Latest value of every trait set for the given profiles. Traits whose value + * was cleared are dropped. + */ + async getProfileTraits( + pid: string, + profileIds: string[], + ): Promise> { + if (_isEmpty(profileIds)) { + return {} + } + + const query = ` + SELECT key, argMax(value, created) AS value + FROM profile_traits + WHERE pid = {pid:FixedString(12)} + AND profileId IN {profileIds:Array(String)} + GROUP BY key + HAVING value != '' + ORDER BY key ASC + LIMIT {limit:UInt32} + ` + + try { + const { data } = await clickhouse + .query({ + query, + query_params: { + pid, + profileIds, + limit: AnalyticsService.MAX_PROFILE_TRAITS, + }, + }) + .then((resultSet) => resultSet.json<{ key: string; value: string }>()) + + return _reduce( + data, + (acc, { key, value }) => ({ ...acc, [key]: value }), + {} as Record, + ) + } catch (error) { + this.logger.error(`[getProfileTraits] Failed to load traits: ${error}`) + return {} + } + } + + /** + * Resolves the full identity of a profile: the canonical profile ID to + * display and every profile ID whose events belong to it. + * + * For an identified (usr_) profile this is the profile itself plus all + * anonymous profiles linked to it. For a linked anonymous profile the + * identity is canonicalised to the identified profile. For an unlinked + * anonymous profile it is just the profile itself. + */ + async resolveProfileIdentity( + pid: string, + profileId: string, + ): Promise<{ canonicalId: string; profileIds: string[] }> { + if (!this.isUserSuppliedProfile(profileId)) { + const userProfileId = await this.getUserProfileForAnon(pid, profileId) + + if (!userProfileId) { + return { canonicalId: profileId, profileIds: [profileId] } + } + + profileId = userProfileId + } + + const query = ` + SELECT anonProfileId + FROM ( + SELECT + anonProfileId, + argMin(userProfileId, created) AS userProfileId + FROM profile_aliases + WHERE pid = {pid:FixedString(12)} + GROUP BY anonProfileId + ) + WHERE userProfileId = {userProfileId:String} + ` + + const { data } = await clickhouse + .query({ + query, + query_params: { pid, userProfileId: profileId }, + }) + .then((resultSet) => resultSet.json<{ anonProfileId: string }>()) + + return { + canonicalId: profileId, + profileIds: [profileId, ..._map(data, 'anonProfileId')], + } + } + + /** + * CTE mapping anonymous profile IDs to the identified profiles they are + * linked to. LEFT JOIN it on profileId and coalesce(nullIf(pam.userProfileId, ''), + * profileId) to attribute pre-identification events to the identified + * profile. Requires a {pid:FixedString(12)} query param. + */ + private buildProfileAliasMapCTE(): string { + return ` + profile_alias_map AS ( + SELECT + anonProfileId, + argMin(userProfileId, created) AS userProfileId + FROM profile_aliases + WHERE pid = {pid:FixedString(12)} + GROUP BY anonProfileId + )` + } + private getSessionFirstSeenKey(pid: string, psid: string): string { return `ses:fs:${pid}:${psid}` } @@ -3651,7 +4007,8 @@ export class AnalyticsService { ) const query = ` - WITH funnel_qualified AS ( + WITH ${this.buildProfileAliasMapCTE()}, + funnel_qualified AS ( SELECT psid FROM ( SELECT @@ -3728,20 +4085,22 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration, - argMax(profileId, lastSeen) as profileId - FROM sessions + argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as profileId + FROM sessions AS s + LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId WHERE pid = {pid:FixedString(12)} AND psid IN (SELECT psid FROM funnel_qualified) GROUP BY psidCasted, pid ), first_session_per_profile AS ( SELECT - profileId, + coalesce(nullIf(pam.userProfileId, ''), s.profileId) AS profileId, argMin(CAST(psid, 'String'), firstSeen) AS firstPsid - FROM sessions FINAL + FROM sessions AS s FINAL + LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId WHERE pid = {pid:FixedString(12)} - AND profileId IS NOT NULL - AND profileId != '' + AND s.profileId IS NOT NULL + AND s.profileId != '' GROUP BY profileId ) SELECT @@ -3810,7 +4169,8 @@ export class AnalyticsService { : '' const query = ` - WITH journey_qualified AS ( + WITH ${this.buildProfileAliasMapCTE()}, + journey_qualified AS ( SELECT psid FROM ( SELECT @@ -3883,20 +4243,22 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration, - argMax(profileId, lastSeen) as profileId - FROM sessions + argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as profileId + FROM sessions AS s + LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId WHERE pid = {pid:FixedString(12)} AND psid IN (SELECT psid FROM journey_qualified) GROUP BY psidCasted, pid ), first_session_per_profile AS ( SELECT - profileId, + coalesce(nullIf(pam.userProfileId, ''), s.profileId) AS profileId, argMin(CAST(psid, 'String'), firstSeen) AS firstPsid - FROM sessions FINAL + FROM sessions AS s FINAL + LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId WHERE pid = {pid:FixedString(12)} - AND profileId IS NOT NULL - AND profileId != '' + AND s.profileId IS NOT NULL + AND s.profileId != '' GROUP BY profileId ) SELECT @@ -7650,6 +8012,19 @@ export class AnalyticsService { const replay = await this.getSessionReplaySummary(pid, psid) + // If the session's profile was linked to an identified profile + // (via the identify API), display the identified profile + if (details?.profileId && !this.isUserSuppliedProfile(details.profileId)) { + const userProfileId = await this.getUserProfileForAnon( + pid, + details.profileId, + ) + + if (userProfileId) { + details.profileId = userProfileId + } + } + return { pages: this.processPageflow(pages), details: { @@ -7684,7 +8059,8 @@ export class AnalyticsService { ) const query = ` - WITH distinct_sessions_filtered AS ( + WITH ${this.buildProfileAliasMapCTE()}, + distinct_sessions_filtered AS ( SELECT psidCasted, pid, @@ -7757,8 +8133,9 @@ export class AnalyticsService { toString(psid) AS psidCasted, pid, dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration, - argMax(profileId, lastSeen) as profileId - FROM sessions + argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as profileId + FROM sessions AS s + LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId WHERE pid = {pid:FixedString(12)} GROUP BY psidCasted, pid ), @@ -7778,12 +8155,13 @@ export class AnalyticsService { ), first_session_per_profile AS ( SELECT - profileId, + coalesce(nullIf(pam.userProfileId, ''), s.profileId) AS profileId, argMin(toString(psid), firstSeen) AS firstPsid - FROM sessions FINAL + FROM sessions AS s FINAL + LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId WHERE pid = {pid:FixedString(12)} - AND profileId IS NOT NULL - AND profileId != '' + AND s.profileId IS NOT NULL + AND s.profileId != '' GROUP BY profileId ), sessions_enriched AS ( @@ -7930,7 +8308,8 @@ export class AnalyticsService { )` const query = ` - WITH ${filteredSessionsCTE}, + WITH ${this.buildProfileAliasMapCTE()}, + ${filteredSessionsCTE}, replay_summary AS ( SELECT psidCasted, @@ -8025,8 +8404,9 @@ export class AnalyticsService { toString(psid) AS psidCasted, pid, dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration, - argMax(profileId, lastSeen) as profileId - FROM sessions + argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as profileId + FROM sessions AS s + LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId WHERE pid = {pid:FixedString(12)} GROUP BY psidCasted, pid ), @@ -8065,12 +8445,13 @@ export class AnalyticsService { ), first_session_per_profile AS ( SELECT - profileId, + coalesce(nullIf(pam.userProfileId, ''), s.profileId) AS profileId, argMin(toString(psid), firstSeen) AS firstPsid - FROM sessions FINAL + FROM sessions AS s FINAL + LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId WHERE pid = {pid:FixedString(12)} - AND profileId IS NOT NULL - AND profileId != '' + AND s.profileId IS NOT NULL + AND s.profileId != '' GROUP BY profileId ) SELECT @@ -8308,12 +8689,11 @@ export class AnalyticsService { private buildProfilesListDataCTE( filtersQuery: string, - profileTypeFilter: string, customEVFilterApplied: boolean, ): string { const scopedProfileFilter = customEVFilterApplied ? ` - AND profileId IN ( + AND events.profileId IN ( SELECT DISTINCT profileId FROM events WHERE pid = {pid:FixedString(12)} @@ -8321,16 +8701,17 @@ export class AnalyticsService { AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL AND profileId != '' - ${profileTypeFilter} ${filtersQuery} ) ` : filtersQuery + // Events recorded for an anonymous profile before the visitor was + // identified get attributed to the identified profile via profile_alias_map return ` all_profile_data AS ( SELECT - profileId, + coalesce(nullIf(pam.userProfileId, ''), events.profileId) AS profileId, psid, cc, os, @@ -8341,12 +8722,12 @@ export class AnalyticsService { if(type = 'custom_event', 1, 0) AS isEvent, if(type = 'error', 1, 0) AS isError FROM events + LEFT JOIN profile_alias_map pam ON events.profileId = pam.anonProfileId WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'error') AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND profileId IS NOT NULL - AND profileId != '' - ${profileTypeFilter} + AND events.profileId IS NOT NULL + AND events.profileId != '' ${scopedProfileFilter} )` } @@ -8361,21 +8742,23 @@ export class AnalyticsService { profileType: 'all' | 'anonymous' | 'identified' = 'all', customEVFilterApplied = false, ): Promise { + // The profile type filter is applied to the alias-resolved profileId, so + // anonymous history linked to an identified profile counts as identified let profileTypeFilter = '' if (profileType === 'anonymous') { - profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'` + profileTypeFilter = `WHERE profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'` } else if (profileType === 'identified') { - profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'` + profileTypeFilter = `WHERE profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'` } const allProfileDataCTE = this.buildProfilesListDataCTE( filtersQuery, - profileTypeFilter, customEVFilterApplied, ) const query = ` - WITH ${allProfileDataCTE}, + WITH ${this.buildProfileAliasMapCTE()}, + ${allProfileDataCTE}, profile_aggregated AS ( SELECT profileId, @@ -8390,6 +8773,7 @@ export class AnalyticsService { any(br) AS br_agg, any(dv) AS dv_agg FROM all_profile_data + ${profileTypeFilter} GROUP BY profileId ) SELECT @@ -8430,6 +8814,11 @@ export class AnalyticsService { profileId: string, _safeTimezone: string, ): Promise { + const { canonicalId, profileIds } = await this.resolveProfileIdentity( + pid, + profileId, + ) + // Query session count from sessions table const querySessionCount = ` SELECT @@ -8438,7 +8827,7 @@ export class AnalyticsService { max(sessions.lastSeen) AS lastSeen FROM sessions FINAL WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} ` // Prefer pageview timings, but fall back to sessions for event/error-only profiles. @@ -8453,7 +8842,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} AND psid IS NOT NULL GROUP BY psid HAVING session_duration > 0 @@ -8468,7 +8857,7 @@ export class AnalyticsService { dateDiff('second', min(firstSeen), max(lastSeen)) AS session_duration FROM sessions FINAL WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} GROUP BY psid HAVING session_duration > 0 ) @@ -8483,7 +8872,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} ` const queryEvents = ` @@ -8491,7 +8880,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} ` const queryErrors = ` @@ -8499,7 +8888,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type = 'error' - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} ` const queryDetails = ` @@ -8515,7 +8904,7 @@ export class AnalyticsService { any(lc) AS lc FROM events WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} AND type IN ('pageview', 'custom_event', 'error') ` @@ -8529,14 +8918,14 @@ export class AnalyticsService { argMax(currency, synced_at) AS currency FROM revenue WHERE pid = {pid:FixedString(12)} - AND profile_id = {profileId:String} + AND profile_id IN {profileIds:Array(String)} AND type IN ('sale', 'subscription') AND status = 'completed' GROUP BY transaction_id ) ` - const params = { pid, profileId } + const params = { pid, profileIds } const [ sessionCountResult, @@ -8546,6 +8935,7 @@ export class AnalyticsService { errorsResult, detailsResult, revenueResult, + traits, ] = await Promise.all([ clickhouse .query({ query: querySessionCount, query_params: params }) @@ -8568,6 +8958,7 @@ export class AnalyticsService { clickhouse .query({ query: queryRevenue, query_params: params }) .then((resultSet) => resultSet.json()), + this.getProfileTraits(pid, profileIds), ]) const sessionCount = (sessionCountResult.data[0] || {}) as Record< @@ -8582,8 +8973,9 @@ export class AnalyticsService { const revenue = (revenueResult.data[0] || {}) as Record return { - profileId, - isIdentified: this.isUserSuppliedProfile(profileId), + profileId: canonicalId, + isIdentified: this.isUserSuppliedProfile(canonicalId), + traits, sessionsCount: sessionCount.sessionsCount || 0, pageviewsCount: pageviews.pageviewsCount || 0, eventsCount: events.eventsCount || 0, @@ -8602,6 +8994,8 @@ export class AnalyticsService { profileId: string, limit = 10, ): Promise<{ page: string; count: number }[]> { + const { profileIds } = await this.resolveProfileIdentity(pid, profileId) + const query = ` SELECT pg AS page, @@ -8609,7 +9003,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} GROUP BY pg ORDER BY count DESC LIMIT {limit:UInt32} @@ -8618,7 +9012,7 @@ export class AnalyticsService { const { data } = await clickhouse .query({ query, - query_params: { pid, profileId, limit: Number(limit) }, + query_params: { pid, profileIds, limit: Number(limit) }, }) .then((resultSet) => resultSet.json()) @@ -8630,6 +9024,8 @@ export class AnalyticsService { profileId: string, months = 4, ): Promise<{ date: string; pageviews: number; events: number }[]> { + const { profileIds } = await this.resolveProfileIdentity(pid, profileId) + const startDate = dayjs .utc() .subtract(months, 'month') @@ -8644,7 +9040,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event') - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} AND created >= {startDate:Date} GROUP BY date ORDER BY date ASC @@ -8653,7 +9049,7 @@ export class AnalyticsService { const { data } = await clickhouse .query({ query, - query_params: { pid, profileId, startDate }, + query_params: { pid, profileIds, startDate }, }) .then((resultSet) => resultSet.json()) @@ -8671,7 +9067,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} @@ -8684,7 +9080,6 @@ export class AnalyticsService { SELECT CAST(psid, 'String') AS psidCasted, pid, - profileId, cc, os, br, @@ -8692,7 +9087,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'error') - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${scopedSessionFilter} @@ -8703,7 +9098,7 @@ export class AnalyticsService { pid: string, psids: string[], safeTimezone: string, - profileId: string, + profileIds: string[], groupFrom: string, groupTo: string, ): Promise> { @@ -8750,7 +9145,7 @@ export class AnalyticsService { WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'error') - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} AND psid IS NOT NULL AND toString(psid) IN {psids:Array(String)} AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -8795,7 +9190,7 @@ export class AnalyticsService { FROM revenue WHERE pid = {pid:FixedString(12)} - AND profile_id = {profileId:String} + AND profile_id IN {profileIds:Array(String)} AND session_id IS NOT NULL AND toString(session_id) IN {psids:Array(String)} AND revenue.created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -8821,7 +9216,7 @@ export class AnalyticsService { pid, psids, timezone: safeTimezone, - profileId, + profileIds, groupFrom, groupTo, }, @@ -8856,6 +9251,8 @@ export class AnalyticsService { skip = 0, customEVFilterApplied = false, ): Promise { + const { profileIds } = await this.resolveProfileIdentity(pid, profileId) + const allProfileEventsCTE = this.buildProfileSessionsEventsCTE( filtersQuery, customEVFilterApplied, @@ -8867,14 +9264,13 @@ export class AnalyticsService { SELECT psidCasted, pid, - profileId, any(cc) AS cc_agg, any(os) AS os_agg, any(br) AS br_agg, min(created_tz) AS sessionStart, max(created_tz) AS lastActivity FROM all_profile_events - GROUP BY psidCasted, pid, profileId + GROUP BY psidCasted, pid ), pageview_counts AS ( SELECT @@ -8884,7 +9280,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid @@ -8897,7 +9293,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid @@ -8910,7 +9306,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type = 'error' - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid @@ -8922,7 +9318,7 @@ export class AnalyticsService { dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration FROM sessions WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} GROUP BY psidCasted, pid ) SELECT @@ -8953,7 +9349,7 @@ export class AnalyticsService { query, query_params: { pid, - profileId, + profileIds, ...paramsData.params, timezone: safeTimezone, take, @@ -8967,7 +9363,7 @@ export class AnalyticsService { pid, sessions.map((session) => String(session.psid)).filter(Boolean), safeTimezone, - profileId, + profileIds, paramsData.params.groupFrom, paramsData.params.groupTo, ) diff --git a/backend/apps/cloud/src/analytics/bot-detection.service.ts b/backend/apps/cloud/src/analytics/bot-detection.service.ts index c05da8e4f..c6f85889a 100644 --- a/backend/apps/cloud/src/analytics/bot-detection.service.ts +++ b/backend/apps/cloud/src/analytics/bot-detection.service.ts @@ -23,6 +23,7 @@ export type BotEndpoint = | 'error' | 'feature_flag' | 'heartbeat' + | 'identify' | 'noscript' | 'session_replay' diff --git a/backend/apps/cloud/src/analytics/dto/get-profile.dto.ts b/backend/apps/cloud/src/analytics/dto/get-profile.dto.ts index 6962d2bac..52d1d6d88 100644 --- a/backend/apps/cloud/src/analytics/dto/get-profile.dto.ts +++ b/backend/apps/cloud/src/analytics/dto/get-profile.dto.ts @@ -13,6 +13,7 @@ import { import { DEFAULT_TIMEZONE } from '../../user/entities/user.entity' import { PID_REGEX } from '../../common/constants' import { GetDataDto } from './getData.dto' +import { MAX_STORED_PROFILE_ID_LENGTH } from './identify.dto' export class GetProfileDto { @ApiProperty({ @@ -27,11 +28,11 @@ export class GetProfileDto { @ApiProperty({ required: true, description: 'The profile ID', - maxLength: 256, + maxLength: MAX_STORED_PROFILE_ID_LENGTH, }) @IsNotEmpty() @IsString() - @MaxLength(256) + @MaxLength(MAX_STORED_PROFILE_ID_LENGTH) profileId: string @ApiProperty({ @@ -53,11 +54,11 @@ export class GetProfileSessionsDto extends PickType(GetDataDto, [ @ApiProperty({ required: true, description: 'The profile ID', - maxLength: 256, + maxLength: MAX_STORED_PROFILE_ID_LENGTH, }) @IsNotEmpty() @IsString() - @MaxLength(256) + @MaxLength(MAX_STORED_PROFILE_ID_LENGTH) profileId: string @ApiProperty({ required: false, default: 30 }) diff --git a/backend/apps/cloud/src/analytics/dto/identify.dto.ts b/backend/apps/cloud/src/analytics/dto/identify.dto.ts new file mode 100644 index 000000000..9d90d1744 --- /dev/null +++ b/backend/apps/cloud/src/analytics/dto/identify.dto.ts @@ -0,0 +1,164 @@ +import _keys from 'lodash/keys' +import _some from 'lodash/some' +import _values from 'lodash/values' +import { ApiProperty } from '@nestjs/swagger' +import { Transform, Type } from 'class-transformer' +import { + IsNotEmpty, + IsObject, + IsOptional, + IsString, + Matches, + MaxLength, + Validate, + ValidatorConstraint, + ValidatorConstraintInterface, +} from 'class-validator' +import { PID_REGEX } from '../../common/constants' + +// The raw identifier a site passes to identify() (or as `profileId` on an +// event). It is stored as provided, behind the `usr_` prefix. +export const MAX_USER_PROFILE_ID_LENGTH = 256 +export const MAX_STORED_PROFILE_ID_LENGTH = + MAX_USER_PROFILE_ID_LENGTH + 'usr_'.length + +const MAX_TRAITS_KEYS = 50 +const MAX_TRAIT_KEY_LENGTH = 128 +const MAX_TRAITS_TOTAL_LENGTH = 2000 + +@ValidatorConstraint() +class TraitsKeysQuantity implements ValidatorConstraintInterface { + validate(traits: Record) { + return _keys(traits).length <= MAX_TRAITS_KEYS + } +} + +// Control / format characters never appear in a genuine trait name or value, +// but would corrupt the dashboard rendering them (e.g. via a right-to-left +// override). +const UNPRINTABLE_REGEX = /[\p{Cc}\p{Cf}]/u + +@ValidatorConstraint() +class TraitsKeyFormat implements ValidatorConstraintInterface { + validate(traits: Record) { + return !_some( + _keys(traits), + (key) => + !key.trim() || + key.length > MAX_TRAIT_KEY_LENGTH || + UNPRINTABLE_REGEX.test(key), + ) + } +} + +@ValidatorConstraint() +class TraitsValueType implements ValidatorConstraintInterface { + validate(traits: Record) { + return !_some( + _values(traits), + (value) => typeof value !== 'string' || UNPRINTABLE_REGEX.test(value), + ) + } +} + +@ValidatorConstraint() +class TraitsSizeLimit implements ValidatorConstraintInterface { + validate(traits: Record) { + let totalSize = 0 + + for (const key of _keys(traits)) { + const value = traits[key] + totalSize += key.length + (typeof value === 'string' ? value.length : 0) + + if (totalSize > MAX_TRAITS_TOTAL_LENGTH) { + return false + } + } + + return true + } +} + +/** + * Traits are free-form, so accept the primitives a JSON payload naturally + * carries and store them as strings. `null` / `undefined` mean "remove this + * trait" and are normalised to an empty value. + */ +function transformTraits(value: any): any { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return value + } + + const transformed: Record = {} + + for (const key of _keys(value)) { + const trait = value[key] + + if (trait === null || trait === undefined) { + transformed[key.trim()] = '' + } else if (typeof trait === 'number' || typeof trait === 'boolean') { + transformed[key.trim()] = String(trait) + } else if (typeof trait === 'string') { + transformed[key.trim()] = trait.trim() + } else { + // Objects and arrays are left as-is for the validation below to reject + transformed[key.trim()] = trait + } + } + + return transformed +} + +export class IdentifyDto { + @ApiProperty({ + example: 'aUn1quEid-3', + required: true, + description: 'The project ID', + }) + @IsNotEmpty() + @Matches(PID_REGEX, { message: 'The provided Project ID (pid) is incorrect' }) + pid: string + + @ApiProperty({ + example: 'user_12345', + required: true, + description: + 'A unique, stable identifier of the user (e.g. an internal user ID), stored as provided. The current anonymous profile of the visitor gets linked to the resulting identified profile.', + }) + @IsNotEmpty() + @IsString() + @MaxLength(MAX_USER_PROFILE_ID_LENGTH) + profileId: string + + @ApiProperty({ + example: { + email: 'john@example.com', + plan: 'premium', + }, + required: false, + description: + 'Traits of the identified user, displayed on their profile. Values must be primitive JSON types and are stored as strings; a null value removes the trait.', + }) + @IsOptional() + @IsObject() + // Pins the target type of the nested object. Without it class-transformer + // guesses one from `value.constructor`, which a `{"constructor": "..."}` + // trait turns into a string and blows up with a TypeError (a 500 on this + // public endpoint) before any validation below runs. + @Type(() => Object) + @Transform(({ value }) => transformTraits(value)) + @Validate(TraitsKeysQuantity, { + message: `Traits object can't have more than ${MAX_TRAITS_KEYS} keys`, + }) + @Validate(TraitsKeyFormat, { + message: `Traits keys must be non-empty and no longer than ${MAX_TRAIT_KEY_LENGTH} characters`, + }) + @Validate(TraitsValueType, { + message: + 'All of traits object values must be primitive JSON values without control characters', + }) + @Validate(TraitsSizeLimit, { + message: `Traits object can't have keys and values with total length more than ${MAX_TRAITS_TOTAL_LENGTH} characters`, + }) + traits?: Record +} diff --git a/backend/apps/cloud/src/analytics/v2/dto/entities.dto.ts b/backend/apps/cloud/src/analytics/v2/dto/entities.dto.ts index b8579e767..1f1534853 100644 --- a/backend/apps/cloud/src/analytics/v2/dto/entities.dto.ts +++ b/backend/apps/cloud/src/analytics/v2/dto/entities.dto.ts @@ -9,10 +9,12 @@ import { IsOptional, IsString, Max, + MaxLength, Min, } from 'class-validator' import { TimeBucketType } from '../../dto/getData.dto' +import { MAX_STORED_PROFILE_ID_LENGTH } from '../../dto/identify.dto' import { V2BaseQueryDto, V2ProjectParamsDto } from './v2-base.dto' export const V2_MAX_ENTITY_LIMIT = 150 @@ -138,6 +140,7 @@ export class V2ProfileParamsDto extends V2ProjectParamsDto { @ApiProperty({ description: 'The profile identifier' }) @IsNotEmpty() @IsString() + @MaxLength(MAX_STORED_PROFILE_ID_LENGTH) profileId: string } diff --git a/backend/apps/cloud/src/project/project.controller.ts b/backend/apps/cloud/src/project/project.controller.ts index 2c18e3130..b2273605b 100644 --- a/backend/apps/cloud/src/project/project.controller.ts +++ b/backend/apps/cloud/src/project/project.controller.ts @@ -753,6 +753,8 @@ export class ProjectController { const queries = [ 'ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)}', 'ALTER TABLE error_statuses DELETE WHERE pid={pid:FixedString(12)}', + 'ALTER TABLE profile_aliases DELETE WHERE pid={pid:FixedString(12)}', + 'ALTER TABLE profile_traits DELETE WHERE pid={pid:FixedString(12)}', ] try { @@ -1641,6 +1643,8 @@ export class ProjectController { const queries = [ 'ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)}', 'ALTER TABLE error_statuses DELETE WHERE pid={pid:FixedString(12)}', + 'ALTER TABLE profile_aliases DELETE WHERE pid={pid:FixedString(12)}', + 'ALTER TABLE profile_traits DELETE WHERE pid={pid:FixedString(12)}', ] try { diff --git a/backend/apps/cloud/src/project/project.service.ts b/backend/apps/cloud/src/project/project.service.ts index 071ff3a8c..dac927f35 100644 --- a/backend/apps/cloud/src/project/project.service.ts +++ b/backend/apps/cloud/src/project/project.service.ts @@ -514,6 +514,8 @@ export class ProjectService { const queries = [ 'ALTER TABLE events DELETE WHERE pid IN ({pids:Array(FixedString(12))})', 'ALTER TABLE error_statuses DELETE WHERE pid IN ({pids:Array(FixedString(12))})', + 'ALTER TABLE profile_aliases DELETE WHERE pid IN ({pids:Array(FixedString(12))})', + 'ALTER TABLE profile_traits DELETE WHERE pid IN ({pids:Array(FixedString(12))})', ] await Promise.all( diff --git a/backend/apps/community/src/analytics/analytics.controller.ts b/backend/apps/community/src/analytics/analytics.controller.ts index 62ef26360..60cb697bd 100644 --- a/backend/apps/community/src/analytics/analytics.controller.ts +++ b/backend/apps/community/src/analytics/analytics.controller.ts @@ -63,6 +63,7 @@ import { GetSessionDto } from './dto/get-session.dto' import { GetProfilesDto } from './dto/get-profiles.dto' import { GetProfileDto, GetProfileSessionsDto } from './dto/get-profile.dto' import { GetProfileIdDto, GetSessionIdDto } from './dto/get-id.dto' +import { IdentifyDto } from './dto/identify.dto' import { ErrorDto } from './dto/error.dto' import { GetErrorsDto } from './dto/get-errors.dto' import { GetErrorDto } from './dto/get-error.dto' @@ -2788,6 +2789,104 @@ export class AnalyticsController { return { sessions, appliedFilters, take, skip } } + /** + * Links the visitor's current anonymous profile to an identified profile + * derived from the user ID supplied by the site (e.g. after log in). Events + * previously recorded for the anonymous profile get attributed to the + * identified profile at query time; the tracker stamps all subsequent + * events with the supplied profileId directly. + * + * Optional traits (email, plan, ...) are stored against the identified + * profile and shown on its dashboard page. + */ + @Post('identify') + @Public() + async identify( + @Body() dto: IdentifyDto, + @Headers() headers, + @Ip() reqIP, + ): Promise<{ profileId: string } | typeof BOT_RESPONSE> { + const { 'user-agent': userAgent, origin } = headers + const ip = getIPFromHeaders(headers) || reqIP || '' + + await checkRateLimit(ip, 'identify', 120, 60) + await checkRateLimit(dto.pid, 'identify', 2000, 60) + + const botResult = await this.analyticsService.checkBot( + dto.pid, + userAgent, + headers, + ip, + headers.referer || headers.referrer, + null, + 'identify', + ) + + if (botResult.isBot) { + return BOT_RESPONSE + } + + const profileId = this.analyticsService.validateUserSuppliedProfileId( + dto.profileId, + ) + + await this.analyticsService.validate(dto, origin, ip) + + const userProfileId = await this.analyticsService.generateProfileId( + dto.pid, + userAgent, + ip, + profileId, + ) + + if (!_isEmpty(dto.traits)) { + await this.analyticsService.saveProfileTraits( + dto.pid, + userProfileId, + dto.traits, + ) + } + + const anonProfileId = await this.analyticsService.generateProfileId( + dto.pid, + userAgent, + ip, + ) + + const linked = await this.analyticsService.linkProfiles( + dto.pid, + anonProfileId, + userProfileId, + ) + + // Flip the visitor's current session (if any) to the identified profile. + // Skipped when the anonymous profile is already linked to a different + // identified profile (e.g. a second account on a shared device) - the + // session stays with the current identity until new events re-stamp it. + if (linked) { + const { exists, psid } = await this.analyticsService.getSessionId( + dto.pid, + userAgent, + ip, + ) + + if (exists) { + await this.analyticsService.recordSessionActivity( + psid, + dto.pid, + userProfileId, + ) + } + } + + this.logger.log( + `pid: ${dto.pid}, profileId: ${userProfileId}`, + 'POST /analytics/identify', + ) + + return { profileId: userProfileId } + } + @Post('profile-id') @Public() async getOrCreateProfileId( diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts index c00dd5dfc..40fb133d1 100644 --- a/backend/apps/community/src/analytics/analytics.service.ts +++ b/backend/apps/community/src/analytics/analytics.service.ts @@ -102,6 +102,7 @@ import { import { ErrorDto } from './dto/error.dto' import { GetPagePropertyMetaDto } from './dto/get-page-property-meta.dto' import { DATA_DELETION_EVENT_TYPES } from './dto/data-deletion.dto' +import { MAX_USER_PROFILE_ID_LENGTH } from './dto/identify.dto' import { ProjectViewCustomEventMetaValueType } from '../project/entity/project-view-custom-event.entity' import { ProjectViewCustomEventDto } from '../project/dto/create-project-view.dto' import { UAParser } from '@ua-parser-js/pro-business' @@ -690,7 +691,11 @@ export class AnalyticsService { } async validate( - logDTO: PageviewsDto | EventsDto | ErrorDto, + logDTO: + | PageviewsDto + | EventsDto + | ErrorDto + | { pid: string; lc?: string | null }, origin: string, ip?: string, ): Promise { @@ -2036,11 +2041,14 @@ export class AnalyticsService { userSupplied?: string, ): Promise { if (userSupplied) { - const cleanId = userSupplied - .replace(AnalyticsService.PROFILE_PREFIX_ANON, '') - .replace(AnalyticsService.PROFILE_PREFIX_USER, '') - const hash = this.hashToNumericString(`${cleanId}${pid}`) - return `${AnalyticsService.PROFILE_PREFIX_USER}${hash}` + const normalised = this.normaliseUserSuppliedProfileId(userSupplied) + + if (normalised) { + return `${AnalyticsService.PROFILE_PREFIX_USER}${normalised}` + } + + // Unusable identifier - fall back to the anonymous fingerprint rather + // than fusing every visitor sending it into one profile. } const salt = await this.saltService.getSaltForProfile() @@ -2054,6 +2062,358 @@ export class AnalyticsService { return profileId.startsWith(AnalyticsService.PROFILE_PREFIX_USER) } + // Values that are almost certainly instrumentation bugs rather than real user + // IDs (e.g. stringified nulls). Identifying with them would fuse unrelated + // visitors into a single profile. + private static readonly ILLEGAL_USER_PROFILE_IDS = new Set([ + '', + 'null', + 'undefined', + 'nan', + 'none', + 'nil', + 'true', + 'false', + '0', + 'anonymous', + 'anon', + 'guest', + 'user', + 'id', + 'distinct_id', + 'distinctid', + 'profileid', + 'profile_id', + 'not_authenticated', + 'email', + '[object object]', + ]) + + // Control (\p{Cc}) and format (\p{Cf}) characters never appear in a genuine + // identifier, but would corrupt log lines and let a profile masquerade as + // another one in the dashboard (e.g. via a right-to-left override). + private static readonly UNPRINTABLE_REGEX = /[\p{Cc}\p{Cf}]/u + + // Traits accumulate across identify() calls, so the number of distinct keys + // a profile ends up with is bounded on read rather than on write. + static readonly MAX_PROFILE_TRAITS = 100 + + /** + * Normalises a profile ID supplied by the site (via identify() or the + * `profileId` field of an event). The value is stored as-is behind the + * `usr_` prefix - it belongs to the site, and hashing it would only make + * the dashboard unreadable. + * + * @returns the identifier to store, or null when it can't be used as one. + */ + normaliseUserSuppliedProfileId(userSupplied: string): string | null { + if (typeof userSupplied !== 'string') { + return null + } + + const trimmed = userSupplied.trim() + + if ( + !trimmed || + trimmed.length > MAX_USER_PROFILE_ID_LENGTH || + AnalyticsService.UNPRINTABLE_REGEX.test(trimmed) || + AnalyticsService.ILLEGAL_USER_PROFILE_IDS.has(trimmed.toLowerCase()) + ) { + return null + } + + return trimmed + } + + validateUserSuppliedProfileId(userSupplied: string): string { + const normalised = this.normaliseUserSuppliedProfileId(userSupplied) + + if (!normalised) { + const echoed = String(userSupplied ?? '').slice(0, 64) + + throw new BadRequestException( + `"${echoed}" is not a valid profileId. Pass a unique, stable identifier of the user (e.g. an internal user ID) of up to ${MAX_USER_PROFILE_ID_LENGTH} characters, see https://docs.swetrix.com/visitor-identification`, + ) + } + + return normalised + } + + private getProfileAliasKey(pid: string, anonProfileId: string): string { + return `pfa:${pid}:${anonProfileId}` + } + + // Sentinel cached for anonymous profiles with no alias, so unlinked (the + // overwhelmingly common case) lookups don't hit ClickHouse every time. + // Cannot collide with a real value - those always start with usr_. + private static readonly PROFILE_ALIAS_NONE = '-' + + /** + * Looks up the identified (usr_) profile an anonymous (anon_) profile has + * been linked to. First identification wins, so the mapping is resolved + * with argMin(created). + */ + async getUserProfileForAnon( + pid: string, + anonProfileId: string, + ): Promise { + const cacheKey = this.getProfileAliasKey(pid, anonProfileId) + + const cached = await redis.get(cacheKey) + + if (cached) { + return cached === AnalyticsService.PROFILE_ALIAS_NONE ? null : cached + } + + const query = ` + SELECT argMin(userProfileId, created) AS userProfileId + FROM profile_aliases + WHERE pid = {pid:FixedString(12)} + AND anonProfileId = {anonProfileId:String} + ` + + const { data } = await clickhouse + .query({ + query, + query_params: { pid, anonProfileId }, + }) + .then((resultSet) => resultSet.json<{ userProfileId: string | null }>()) + + const userProfileId = data[0]?.userProfileId || null + + if (userProfileId) { + await redis.set(cacheKey, userProfileId, 'EX', 3600) + } else { + // Short TTL: a link created concurrently on another node (linkProfiles + // uses async_insert, so the row may not be SELECTable yet) must become + // visible shortly after this sentinel expires. + await redis.set(cacheKey, AnalyticsService.PROFILE_ALIAS_NONE, 'EX', 60) + } + + return userProfileId + } + + /** + * Links an anonymous (anon_) profile to an identified (usr_) profile. + * The link is created only once per anonymous profile - if it is already + * linked to a different user, the existing link is kept (first + * identification wins; e.g. a second person logging in on a shared device + * must not steal the anonymous history of the first). + * + * @returns whether the anonymous profile is linked to the supplied user + * profile after the call + */ + async linkProfiles( + pid: string, + anonProfileId: string, + userProfileId: string, + ): Promise { + const existing = await this.getUserProfileForAnon(pid, anonProfileId) + + if (existing) { + return existing === userProfileId + } + + try { + await clickhouse.insert({ + table: 'profile_aliases', + format: 'JSONEachRow', + values: [ + { + pid, + anonProfileId, + userProfileId, + created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'), + }, + ], + clickhouse_settings: { async_insert: 1 }, + }) + + await redis.set( + this.getProfileAliasKey(pid, anonProfileId), + userProfileId, + 'EX', + 3600, + ) + } catch (error) { + this.logger.error(`[linkProfiles] Failed to link profiles: ${error}`) + return false + } + + return true + } + + private getProfileTraitsKey(pid: string, profileId: string): string { + return `pft:${pid}:${profileId}` + } + + /** + * Stores the traits (arbitrary key/value metadata a site attaches to a user - + * email, plan, signup date, ...) of an identified profile. + * + * Traits are merged per key: a later call only overwrites the keys it + * carries, and an empty value removes a trait. Storing one row per key lets + * ClickHouse handle that without a read-modify-write cycle. + */ + async saveProfileTraits( + pid: string, + profileId: string, + traits: Record, + ): Promise { + const keys = _sortBy(_keys(traits)) + + if (_isEmpty(keys)) { + return + } + + // identify() is called on every page load, so the same traits arrive over + // and over again - skip the insert when nothing changed since the last one. + const cacheKey = this.getProfileTraitsKey(pid, profileId) + const fingerprint = this.hashToNumericString( + JSON.stringify(_map(keys, (key) => [key, traits[key]])), + ) + + try { + if ((await redis.get(cacheKey)) === fingerprint) { + return + } + + // Millisecond precision: `created` is the ReplacingMergeTree version, so + // two updates of the same trait within a second must still be ordered. + const created = dayjs.utc().format('YYYY-MM-DD HH:mm:ss.SSS') + + await clickhouse.insert({ + table: 'profile_traits', + format: 'JSONEachRow', + values: _map(keys, (key) => ({ + pid, + profileId, + key, + value: traits[key], + created, + })), + clickhouse_settings: { async_insert: 1 }, + }) + + await redis.set(cacheKey, fingerprint, 'EX', 3600) + } catch (error) { + this.logger.error(`[saveProfileTraits] Failed to save traits: ${error}`) + } + } + + /** + * Latest value of every trait set for the given profiles. Traits whose value + * was cleared are dropped. + */ + async getProfileTraits( + pid: string, + profileIds: string[], + ): Promise> { + if (_isEmpty(profileIds)) { + return {} + } + + const query = ` + SELECT key, argMax(value, created) AS value + FROM profile_traits + WHERE pid = {pid:FixedString(12)} + AND profileId IN {profileIds:Array(String)} + GROUP BY key + HAVING value != '' + ORDER BY key ASC + LIMIT {limit:UInt32} + ` + + try { + const { data } = await clickhouse + .query({ + query, + query_params: { + pid, + profileIds, + limit: AnalyticsService.MAX_PROFILE_TRAITS, + }, + }) + .then((resultSet) => resultSet.json<{ key: string; value: string }>()) + + return _reduce( + data, + (acc, { key, value }) => ({ ...acc, [key]: value }), + {} as Record, + ) + } catch (error) { + this.logger.error(`[getProfileTraits] Failed to load traits: ${error}`) + return {} + } + } + + /** + * Resolves the full identity of a profile: the canonical profile ID to + * display and every profile ID whose events belong to it. + * + * For an identified (usr_) profile this is the profile itself plus all + * anonymous profiles linked to it. For a linked anonymous profile the + * identity is canonicalised to the identified profile. For an unlinked + * anonymous profile it is just the profile itself. + */ + async resolveProfileIdentity( + pid: string, + profileId: string, + ): Promise<{ canonicalId: string; profileIds: string[] }> { + if (!this.isUserSuppliedProfile(profileId)) { + const userProfileId = await this.getUserProfileForAnon(pid, profileId) + + if (!userProfileId) { + return { canonicalId: profileId, profileIds: [profileId] } + } + + profileId = userProfileId + } + + const query = ` + SELECT anonProfileId + FROM ( + SELECT + anonProfileId, + argMin(userProfileId, created) AS userProfileId + FROM profile_aliases + WHERE pid = {pid:FixedString(12)} + GROUP BY anonProfileId + ) + WHERE userProfileId = {userProfileId:String} + ` + + const { data } = await clickhouse + .query({ + query, + query_params: { pid, userProfileId: profileId }, + }) + .then((resultSet) => resultSet.json<{ anonProfileId: string }>()) + + return { + canonicalId: profileId, + profileIds: [profileId, ..._map(data, 'anonProfileId')], + } + } + + /** + * CTE mapping anonymous profile IDs to the identified profiles they are + * linked to. LEFT JOIN it on profileId and coalesce(nullIf(pam.userProfileId, ''), + * profileId) to attribute pre-identification events to the identified + * profile. Requires a {pid:FixedString(12)} query param. + */ + private buildProfileAliasMapCTE(): string { + return ` + profile_alias_map AS ( + SELECT + anonProfileId, + argMin(userProfileId, created) AS userProfileId + FROM profile_aliases + WHERE pid = {pid:FixedString(12)} + GROUP BY anonProfileId + )` + } + private getSessionFirstSeenKey(pid: string, psid: string): string { return `ses:fs:${pid}:${psid}` } @@ -2472,7 +2832,8 @@ export class AnalyticsService { ) const query = ` - WITH funnel_qualified AS ( + WITH ${this.buildProfileAliasMapCTE()}, + funnel_qualified AS ( SELECT psid FROM ( SELECT @@ -2549,20 +2910,22 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration, - argMax(profileId, lastSeen) as profileId - FROM sessions + argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as profileId + FROM sessions AS s + LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId WHERE pid = {pid:FixedString(12)} AND psid IN (SELECT psid FROM funnel_qualified) GROUP BY psidCasted, pid ), first_session_per_profile AS ( SELECT - profileId, + coalesce(nullIf(pam.userProfileId, ''), s.profileId) AS profileId, argMin(CAST(psid, 'String'), firstSeen) AS firstPsid - FROM sessions FINAL + FROM sessions AS s FINAL + LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId WHERE pid = {pid:FixedString(12)} - AND profileId IS NOT NULL - AND profileId != '' + AND s.profileId IS NOT NULL + AND s.profileId != '' GROUP BY profileId ) SELECT @@ -2631,7 +2994,8 @@ export class AnalyticsService { : '' const query = ` - WITH journey_qualified AS ( + WITH ${this.buildProfileAliasMapCTE()}, + journey_qualified AS ( SELECT psid FROM ( SELECT @@ -2704,20 +3068,22 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration, - argMax(profileId, lastSeen) as profileId - FROM sessions + argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as profileId + FROM sessions AS s + LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId WHERE pid = {pid:FixedString(12)} AND psid IN (SELECT psid FROM journey_qualified) GROUP BY psidCasted, pid ), first_session_per_profile AS ( SELECT - profileId, + coalesce(nullIf(pam.userProfileId, ''), s.profileId) AS profileId, argMin(CAST(psid, 'String'), firstSeen) AS firstPsid - FROM sessions FINAL + FROM sessions AS s FINAL + LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId WHERE pid = {pid:FixedString(12)} - AND profileId IS NOT NULL - AND profileId != '' + AND s.profileId IS NOT NULL + AND s.profileId != '' GROUP BY profileId ) SELECT @@ -6092,6 +6458,19 @@ export class AnalyticsService { isLive = lastActivityTime.isAfter(liveThresholdTime) } + // If the session's profile was linked to an identified profile + // (via the identify API), display the identified profile + if (details?.profileId && !this.isUserSuppliedProfile(details.profileId)) { + const userProfileId = await this.getUserProfileForAnon( + pid, + details.profileId, + ) + + if (userProfileId) { + details.profileId = userProfileId + } + } + return { pages: this.processPageflow(pages), details: { @@ -6123,7 +6502,8 @@ export class AnalyticsService { ) const query = ` - WITH distinct_sessions_filtered AS ( + WITH ${this.buildProfileAliasMapCTE()}, + distinct_sessions_filtered AS ( SELECT psidCasted, pid, @@ -6171,19 +6551,21 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration, - argMax(profileId, lastSeen) as profileId - FROM sessions + argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as profileId + FROM sessions AS s + LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId WHERE pid = {pid:FixedString(12)} GROUP BY psidCasted, pid ), first_session_per_profile AS ( SELECT - profileId, + coalesce(nullIf(pam.userProfileId, ''), s.profileId) AS profileId, argMin(CAST(psid, 'String'), firstSeen) AS firstPsid - FROM sessions FINAL + FROM sessions AS s FINAL + LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId WHERE pid = {pid:FixedString(12)} - AND profileId IS NOT NULL - AND profileId != '' + AND s.profileId IS NOT NULL + AND s.profileId != '' GROUP BY profileId ), sessions_enriched AS ( @@ -7136,12 +7518,11 @@ export class AnalyticsService { private buildProfilesListDataCTE( filtersQuery: string, - profileTypeFilter: string, customEVFilterApplied: boolean, ): string { const scopedProfileFilter = customEVFilterApplied ? ` - AND profileId IN ( + AND events.profileId IN ( SELECT DISTINCT profileId FROM events WHERE pid = {pid:FixedString(12)} @@ -7149,16 +7530,17 @@ export class AnalyticsService { AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL AND profileId != '' - ${profileTypeFilter} ${filtersQuery} ) ` : filtersQuery + // Events recorded for an anonymous profile before the visitor was + // identified get attributed to the identified profile via profile_alias_map return ` all_profile_data AS ( SELECT - profileId, + coalesce(nullIf(pam.userProfileId, ''), events.profileId) AS profileId, psid, cc, os, @@ -7169,12 +7551,12 @@ export class AnalyticsService { if(type = 'custom_event', 1, 0) AS isEvent, if(type = 'error', 1, 0) AS isError FROM events + LEFT JOIN profile_alias_map pam ON events.profileId = pam.anonProfileId WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'error') AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND profileId IS NOT NULL - AND profileId != '' - ${profileTypeFilter} + AND events.profileId IS NOT NULL + AND events.profileId != '' ${scopedProfileFilter} )` } @@ -7189,21 +7571,23 @@ export class AnalyticsService { profileType: 'all' | 'anonymous' | 'identified' = 'all', customEVFilterApplied = false, ): Promise { + // The profile type filter is applied to the alias-resolved profileId, so + // anonymous history linked to an identified profile counts as identified let profileTypeFilter = '' if (profileType === 'anonymous') { - profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'` + profileTypeFilter = `WHERE profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'` } else if (profileType === 'identified') { - profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'` + profileTypeFilter = `WHERE profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'` } const allProfileDataCTE = this.buildProfilesListDataCTE( filtersQuery, - profileTypeFilter, customEVFilterApplied, ) const query = ` - WITH ${allProfileDataCTE}, + WITH ${this.buildProfileAliasMapCTE()}, + ${allProfileDataCTE}, profile_aggregated AS ( SELECT profileId, @@ -7218,6 +7602,7 @@ export class AnalyticsService { any(br) AS br_agg, any(dv) AS dv_agg FROM all_profile_data + ${profileTypeFilter} GROUP BY profileId ) SELECT @@ -7258,6 +7643,11 @@ export class AnalyticsService { profileId: string, _safeTimezone: string, ): Promise { + const { canonicalId, profileIds } = await this.resolveProfileIdentity( + pid, + profileId, + ) + // Query session count from sessions table const querySessionCount = ` SELECT @@ -7266,7 +7656,7 @@ export class AnalyticsService { max(sessions.lastSeen) AS lastSeen FROM sessions FINAL WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} ` // Prefer pageview timings, but fall back to sessions for event/error-only profiles. @@ -7281,7 +7671,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} AND psid IS NOT NULL GROUP BY psid HAVING session_duration > 0 @@ -7296,7 +7686,7 @@ export class AnalyticsService { dateDiff('second', min(firstSeen), max(lastSeen)) AS session_duration FROM sessions FINAL WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} GROUP BY psid HAVING session_duration > 0 ) @@ -7311,7 +7701,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} ` const queryEvents = ` @@ -7319,7 +7709,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} ` const queryErrors = ` @@ -7327,7 +7717,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type = 'error' - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} ` // Query for device/location details @@ -7344,11 +7734,11 @@ export class AnalyticsService { argMax(lc, created) AS lc FROM events WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} AND type IN ('pageview', 'custom_event', 'error') ` - const params = { pid, profileId } + const params = { pid, profileIds } const [ sessionCountResult, @@ -7357,6 +7747,7 @@ export class AnalyticsService { eventsResult, errorsResult, detailsResult, + traits, ] = await Promise.all([ clickhouse .query({ query: querySessionCount, query_params: params }) @@ -7376,6 +7767,7 @@ export class AnalyticsService { clickhouse .query({ query: queryDetails, query_params: params }) .then((resultSet) => resultSet.json()), + this.getProfileTraits(pid, profileIds), ]) const sessionCount = (sessionCountResult.data[0] || {}) as Record< @@ -7389,8 +7781,9 @@ export class AnalyticsService { const details = (detailsResult.data[0] || {}) as Record return { - profileId, - isIdentified: this.isUserSuppliedProfile(profileId), + profileId: canonicalId, + isIdentified: this.isUserSuppliedProfile(canonicalId), + traits, sessionsCount: sessionCount.sessionsCount || 0, pageviewsCount: pageviews.pageviewsCount || 0, eventsCount: events.eventsCount || 0, @@ -7407,6 +7800,8 @@ export class AnalyticsService { profileId: string, limit = 10, ): Promise<{ page: string; count: number }[]> { + const { profileIds } = await this.resolveProfileIdentity(pid, profileId) + const query = ` SELECT pg AS page, @@ -7414,7 +7809,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} GROUP BY pg ORDER BY count DESC LIMIT {limit:UInt32} @@ -7423,7 +7818,7 @@ export class AnalyticsService { const { data } = await clickhouse .query({ query, - query_params: { pid, profileId, limit: Number(limit) }, + query_params: { pid, profileIds, limit: Number(limit) }, }) .then((resultSet) => resultSet.json()) @@ -7435,6 +7830,8 @@ export class AnalyticsService { profileId: string, months = 4, ): Promise<{ date: string; pageviews: number; events: number }[]> { + const { profileIds } = await this.resolveProfileIdentity(pid, profileId) + const startDate = dayjs .utc() .subtract(months, 'month') @@ -7449,7 +7846,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event') - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} AND created >= {startDate:Date} GROUP BY date ORDER BY date ASC @@ -7458,7 +7855,7 @@ export class AnalyticsService { const { data } = await clickhouse .query({ query, - query_params: { pid, profileId, startDate }, + query_params: { pid, profileIds, startDate }, }) .then((resultSet) => resultSet.json()) @@ -7476,7 +7873,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} @@ -7489,7 +7886,6 @@ export class AnalyticsService { SELECT CAST(psid, 'String') AS psidCasted, pid, - profileId, cc, os, br, @@ -7497,7 +7893,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'error') - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${scopedSessionFilter} @@ -7508,7 +7904,7 @@ export class AnalyticsService { pid: string, psids: string[], safeTimezone: string, - profileId: string, + profileIds: string[], groupFrom: string, groupTo: string, ): Promise> { @@ -7556,7 +7952,7 @@ export class AnalyticsService { WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'error') - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} AND psid IS NOT NULL AND toString(psid) IN {psids:Array(String)} AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -7581,7 +7977,7 @@ export class AnalyticsService { pid, psids, timezone: safeTimezone, - profileId, + profileIds, groupFrom, groupTo, }, @@ -7616,6 +8012,8 @@ export class AnalyticsService { skip = 0, customEVFilterApplied = false, ): Promise { + const { profileIds } = await this.resolveProfileIdentity(pid, profileId) + const allProfileEventsCTE = this.buildProfileSessionsEventsCTE( filtersQuery, customEVFilterApplied, @@ -7627,14 +8025,13 @@ export class AnalyticsService { SELECT psidCasted, pid, - profileId, any(cc) AS cc_agg, any(os) AS os_agg, any(br) AS br_agg, min(created_tz) AS sessionStart, max(created_tz) AS lastActivity FROM all_profile_events - GROUP BY psidCasted, pid, profileId + GROUP BY psidCasted, pid ), pageview_counts AS ( SELECT @@ -7644,7 +8041,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid @@ -7657,7 +8054,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid @@ -7670,7 +8067,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND type = 'error' - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid @@ -7682,7 +8079,7 @@ export class AnalyticsService { dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration FROM sessions WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} + AND profileId IN {profileIds:Array(String)} GROUP BY psidCasted, pid ) SELECT @@ -7713,7 +8110,7 @@ export class AnalyticsService { query, query_params: { pid, - profileId, + profileIds, ...paramsData.params, timezone: safeTimezone, take, @@ -7727,7 +8124,7 @@ export class AnalyticsService { pid, sessions.map((session) => String(session.psid)).filter(Boolean), safeTimezone, - profileId, + profileIds, paramsData.params.groupFrom, paramsData.params.groupTo, ) diff --git a/backend/apps/community/src/analytics/bot-detection.service.ts b/backend/apps/community/src/analytics/bot-detection.service.ts index 4af374566..510e0bfbd 100644 --- a/backend/apps/community/src/analytics/bot-detection.service.ts +++ b/backend/apps/community/src/analytics/bot-detection.service.ts @@ -23,6 +23,7 @@ export type BotEndpoint = | 'error' | 'feature_flag' | 'heartbeat' + | 'identify' | 'noscript' interface BotDetectionInput { diff --git a/backend/apps/community/src/analytics/dto/get-profile.dto.ts b/backend/apps/community/src/analytics/dto/get-profile.dto.ts index 6962d2bac..52d1d6d88 100644 --- a/backend/apps/community/src/analytics/dto/get-profile.dto.ts +++ b/backend/apps/community/src/analytics/dto/get-profile.dto.ts @@ -13,6 +13,7 @@ import { import { DEFAULT_TIMEZONE } from '../../user/entities/user.entity' import { PID_REGEX } from '../../common/constants' import { GetDataDto } from './getData.dto' +import { MAX_STORED_PROFILE_ID_LENGTH } from './identify.dto' export class GetProfileDto { @ApiProperty({ @@ -27,11 +28,11 @@ export class GetProfileDto { @ApiProperty({ required: true, description: 'The profile ID', - maxLength: 256, + maxLength: MAX_STORED_PROFILE_ID_LENGTH, }) @IsNotEmpty() @IsString() - @MaxLength(256) + @MaxLength(MAX_STORED_PROFILE_ID_LENGTH) profileId: string @ApiProperty({ @@ -53,11 +54,11 @@ export class GetProfileSessionsDto extends PickType(GetDataDto, [ @ApiProperty({ required: true, description: 'The profile ID', - maxLength: 256, + maxLength: MAX_STORED_PROFILE_ID_LENGTH, }) @IsNotEmpty() @IsString() - @MaxLength(256) + @MaxLength(MAX_STORED_PROFILE_ID_LENGTH) profileId: string @ApiProperty({ required: false, default: 30 }) diff --git a/backend/apps/community/src/analytics/dto/identify.dto.ts b/backend/apps/community/src/analytics/dto/identify.dto.ts new file mode 100644 index 000000000..9d90d1744 --- /dev/null +++ b/backend/apps/community/src/analytics/dto/identify.dto.ts @@ -0,0 +1,164 @@ +import _keys from 'lodash/keys' +import _some from 'lodash/some' +import _values from 'lodash/values' +import { ApiProperty } from '@nestjs/swagger' +import { Transform, Type } from 'class-transformer' +import { + IsNotEmpty, + IsObject, + IsOptional, + IsString, + Matches, + MaxLength, + Validate, + ValidatorConstraint, + ValidatorConstraintInterface, +} from 'class-validator' +import { PID_REGEX } from '../../common/constants' + +// The raw identifier a site passes to identify() (or as `profileId` on an +// event). It is stored as provided, behind the `usr_` prefix. +export const MAX_USER_PROFILE_ID_LENGTH = 256 +export const MAX_STORED_PROFILE_ID_LENGTH = + MAX_USER_PROFILE_ID_LENGTH + 'usr_'.length + +const MAX_TRAITS_KEYS = 50 +const MAX_TRAIT_KEY_LENGTH = 128 +const MAX_TRAITS_TOTAL_LENGTH = 2000 + +@ValidatorConstraint() +class TraitsKeysQuantity implements ValidatorConstraintInterface { + validate(traits: Record) { + return _keys(traits).length <= MAX_TRAITS_KEYS + } +} + +// Control / format characters never appear in a genuine trait name or value, +// but would corrupt the dashboard rendering them (e.g. via a right-to-left +// override). +const UNPRINTABLE_REGEX = /[\p{Cc}\p{Cf}]/u + +@ValidatorConstraint() +class TraitsKeyFormat implements ValidatorConstraintInterface { + validate(traits: Record) { + return !_some( + _keys(traits), + (key) => + !key.trim() || + key.length > MAX_TRAIT_KEY_LENGTH || + UNPRINTABLE_REGEX.test(key), + ) + } +} + +@ValidatorConstraint() +class TraitsValueType implements ValidatorConstraintInterface { + validate(traits: Record) { + return !_some( + _values(traits), + (value) => typeof value !== 'string' || UNPRINTABLE_REGEX.test(value), + ) + } +} + +@ValidatorConstraint() +class TraitsSizeLimit implements ValidatorConstraintInterface { + validate(traits: Record) { + let totalSize = 0 + + for (const key of _keys(traits)) { + const value = traits[key] + totalSize += key.length + (typeof value === 'string' ? value.length : 0) + + if (totalSize > MAX_TRAITS_TOTAL_LENGTH) { + return false + } + } + + return true + } +} + +/** + * Traits are free-form, so accept the primitives a JSON payload naturally + * carries and store them as strings. `null` / `undefined` mean "remove this + * trait" and are normalised to an empty value. + */ +function transformTraits(value: any): any { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return value + } + + const transformed: Record = {} + + for (const key of _keys(value)) { + const trait = value[key] + + if (trait === null || trait === undefined) { + transformed[key.trim()] = '' + } else if (typeof trait === 'number' || typeof trait === 'boolean') { + transformed[key.trim()] = String(trait) + } else if (typeof trait === 'string') { + transformed[key.trim()] = trait.trim() + } else { + // Objects and arrays are left as-is for the validation below to reject + transformed[key.trim()] = trait + } + } + + return transformed +} + +export class IdentifyDto { + @ApiProperty({ + example: 'aUn1quEid-3', + required: true, + description: 'The project ID', + }) + @IsNotEmpty() + @Matches(PID_REGEX, { message: 'The provided Project ID (pid) is incorrect' }) + pid: string + + @ApiProperty({ + example: 'user_12345', + required: true, + description: + 'A unique, stable identifier of the user (e.g. an internal user ID), stored as provided. The current anonymous profile of the visitor gets linked to the resulting identified profile.', + }) + @IsNotEmpty() + @IsString() + @MaxLength(MAX_USER_PROFILE_ID_LENGTH) + profileId: string + + @ApiProperty({ + example: { + email: 'john@example.com', + plan: 'premium', + }, + required: false, + description: + 'Traits of the identified user, displayed on their profile. Values must be primitive JSON types and are stored as strings; a null value removes the trait.', + }) + @IsOptional() + @IsObject() + // Pins the target type of the nested object. Without it class-transformer + // guesses one from `value.constructor`, which a `{"constructor": "..."}` + // trait turns into a string and blows up with a TypeError (a 500 on this + // public endpoint) before any validation below runs. + @Type(() => Object) + @Transform(({ value }) => transformTraits(value)) + @Validate(TraitsKeysQuantity, { + message: `Traits object can't have more than ${MAX_TRAITS_KEYS} keys`, + }) + @Validate(TraitsKeyFormat, { + message: `Traits keys must be non-empty and no longer than ${MAX_TRAIT_KEY_LENGTH} characters`, + }) + @Validate(TraitsValueType, { + message: + 'All of traits object values must be primitive JSON values without control characters', + }) + @Validate(TraitsSizeLimit, { + message: `Traits object can't have keys and values with total length more than ${MAX_TRAITS_TOTAL_LENGTH} characters`, + }) + traits?: Record +} diff --git a/backend/apps/community/src/analytics/v2/dto/entities.dto.ts b/backend/apps/community/src/analytics/v2/dto/entities.dto.ts index b8579e767..1f1534853 100644 --- a/backend/apps/community/src/analytics/v2/dto/entities.dto.ts +++ b/backend/apps/community/src/analytics/v2/dto/entities.dto.ts @@ -9,10 +9,12 @@ import { IsOptional, IsString, Max, + MaxLength, Min, } from 'class-validator' import { TimeBucketType } from '../../dto/getData.dto' +import { MAX_STORED_PROFILE_ID_LENGTH } from '../../dto/identify.dto' import { V2BaseQueryDto, V2ProjectParamsDto } from './v2-base.dto' export const V2_MAX_ENTITY_LIMIT = 150 @@ -138,6 +140,7 @@ export class V2ProfileParamsDto extends V2ProjectParamsDto { @ApiProperty({ description: 'The profile identifier' }) @IsNotEmpty() @IsString() + @MaxLength(MAX_STORED_PROFILE_ID_LENGTH) profileId: string } diff --git a/backend/apps/community/src/project/project.controller.ts b/backend/apps/community/src/project/project.controller.ts index 1930a37df..74bcb7a40 100644 --- a/backend/apps/community/src/project/project.controller.ts +++ b/backend/apps/community/src/project/project.controller.ts @@ -1027,6 +1027,14 @@ export class ProjectController { query: `ALTER TABLE error_statuses DELETE WHERE pid={pid:FixedString(12)}`, query_params: { pid: id }, }) + await clickhouse.command({ + query: `ALTER TABLE profile_aliases DELETE WHERE pid={pid:FixedString(12)}`, + query_params: { pid: id }, + }) + await clickhouse.command({ + query: `ALTER TABLE profile_traits DELETE WHERE pid={pid:FixedString(12)}`, + query_params: { pid: id }, + }) await deleteProjectRedis(id) return 'Project deleted successfully' } catch (e) { diff --git a/backend/apps/community/src/user/user.controller.ts b/backend/apps/community/src/user/user.controller.ts index c984d862d..21223829f 100644 --- a/backend/apps/community/src/user/user.controller.ts +++ b/backend/apps/community/src/user/user.controller.ts @@ -184,6 +184,8 @@ export class UserController { const queries = [ 'ALTER TABLE events DELETE WHERE pid IN ({pids:Array(FixedString(12))})', 'ALTER TABLE error_statuses DELETE WHERE pid IN ({pids:Array(FixedString(12))})', + 'ALTER TABLE profile_aliases DELETE WHERE pid IN ({pids:Array(FixedString(12))})', + 'ALTER TABLE profile_traits DELETE WHERE pid IN ({pids:Array(FixedString(12))})', ] await deleteProjectsByUserIdClickhouse(id) const promises = _map(queries, async (query) => diff --git a/backend/migrations/clickhouse/2026_07_17_profile_aliases.js b/backend/migrations/clickhouse/2026_07_17_profile_aliases.js new file mode 100644 index 000000000..22a33e171 --- /dev/null +++ b/backend/migrations/clickhouse/2026_07_17_profile_aliases.js @@ -0,0 +1,18 @@ +const { queriesRunner, dbName } = require('./setup') + +const queries = [ + // Profile aliases table: maps anonymous (anon_) profile IDs to identified (usr_) + // profile IDs created via the identify API. Resolved at query time with + // argMin(userProfileId, created) so the first identification wins. + `CREATE TABLE IF NOT EXISTS ${dbName}.profile_aliases + ( + pid FixedString(12), + anonProfileId String CODEC(ZSTD(3)), + userProfileId String CODEC(ZSTD(3)), + created DateTime('UTC') CODEC(Delta(4), LZ4) + ) + ENGINE = ReplacingMergeTree() + ORDER BY (pid, anonProfileId, userProfileId);`, +] + +queriesRunner(queries) diff --git a/backend/migrations/clickhouse/2026_07_28_profile_traits.js b/backend/migrations/clickhouse/2026_07_28_profile_traits.js new file mode 100644 index 000000000..ecfa761a5 --- /dev/null +++ b/backend/migrations/clickhouse/2026_07_28_profile_traits.js @@ -0,0 +1,20 @@ +const { queriesRunner, dbName } = require('./setup') + +const queries = [ + // Traits (arbitrary key/value metadata - email, plan, ...) attached to + // identified profiles via the identify API. One row per key so traits merge + // across calls without a read-modify-write cycle; the latest value of each + // key wins and an empty value means the trait was removed. + `CREATE TABLE IF NOT EXISTS ${dbName}.profile_traits + ( + pid FixedString(12), + profileId String CODEC(ZSTD(3)), + key String CODEC(ZSTD(3)), + value String CODEC(ZSTD(3)), + created DateTime64(3, 'UTC') CODEC(Delta(4), LZ4) + ) + ENGINE = ReplacingMergeTree(created) + ORDER BY (pid, profileId, key);`, +] + +queriesRunner(queries) diff --git a/backend/migrations/clickhouse/initialise_database.js b/backend/migrations/clickhouse/initialise_database.js index cc079eae6..d60828387 100644 --- a/backend/migrations/clickhouse/initialise_database.js +++ b/backend/migrations/clickhouse/initialise_database.js @@ -83,6 +83,34 @@ const CLICKHOUSE_INIT_QUERIES = [ ORDER BY (pid, psid) PARTITION BY toYYYYMM(firstSeen);`, + // Profile aliases table: maps anonymous (anon_) profile IDs to identified (usr_) + // profile IDs created via the identify API. Resolved at query time with + // argMin(userProfileId, created) so the first identification wins. + `CREATE TABLE IF NOT EXISTS ${dbName}.profile_aliases + ( + pid FixedString(12), + anonProfileId String CODEC(ZSTD(3)), + userProfileId String CODEC(ZSTD(3)), + created DateTime('UTC') CODEC(Delta(4), LZ4) + ) + ENGINE = ReplacingMergeTree() + ORDER BY (pid, anonProfileId, userProfileId);`, + + // Profile traits table: arbitrary key/value metadata (email, plan, ...) set + // for identified profiles via the identify API. One row per key so traits + // merge across calls; the latest value of each key wins and an empty value + // means the trait was removed. + `CREATE TABLE IF NOT EXISTS ${dbName}.profile_traits + ( + pid FixedString(12), + profileId String CODEC(ZSTD(3)), + key String CODEC(ZSTD(3)), + value String CODEC(ZSTD(3)), + created DateTime64(3, 'UTC') CODEC(Delta(4), LZ4) + ) + ENGINE = ReplacingMergeTree(created) + ORDER BY (pid, profileId, key);`, + // Feature flag evaluations table `CREATE TABLE IF NOT EXISTS ${dbName}.feature_flag_evaluations ( diff --git a/docs/.gitignore b/docs/.gitignore index ca8600674..52dc280d6 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -2,3 +2,4 @@ node_modules/ .next/ .source/ out/ +.static-archive/ diff --git a/docs/content/docs/analytics-dashboard/profiles-and-sessions.mdx b/docs/content/docs/analytics-dashboard/profiles-and-sessions.mdx index f79987f8f..8d26a2ad0 100644 --- a/docs/content/docs/analytics-dashboard/profiles-and-sessions.mdx +++ b/docs/content/docs/analytics-dashboard/profiles-and-sessions.mdx @@ -13,7 +13,7 @@ The **Profiles** tab lists all unique visitors to your website within the select For each profile, you can see: -- **Profile ID**: A unique identifier for the user. If the user is identified (e.g., logged in), this might display their name or email if you have set up User Identification. +- **Profile ID**: A unique identifier for the user. Anonymous visitors get an `anon_…` ID derived from a monthly-rotating fingerprint; visitors identified via [`swetrix.identify()`](/swetrix-js-reference#identify) get a stable `usr_…` ID — the user ID your application supplied, stored as provided. - **Status**: - **Online**: User is currently active (green indicator). - **Recently Active**: User was active in the last 30 minutes (yellow indicator). @@ -39,6 +39,16 @@ You can filter the profiles list to find specific users or segments: This is particularly useful for finding users who experienced specific bugs (e.g., filter by "Browser: Safari" if you suspect a Safari-specific issue). +### Identified users and anonymous history + +When your application calls [`swetrix.identify()`](/swetrix-js-reference#identify) after a user logs in, Swetrix links the visitor's current anonymous profile to the identified one. From then on: + +- The anonymous profile disappears from the list — its sessions, pageviews, events and errors are attributed to the identified profile instead. +- The session during which the user logged in is shown as a single, identified session (including the pageviews from before the login). +- The **Identified** filter includes the linked anonymous history as well. + +See [visitor identification](/visitor-identification) for how the linking works and its limits. + ## Profile Details Clicking on any profile in the list opens the **Profile Details** view. This view provides a comprehensive history of that specific user's interaction with your site. @@ -52,6 +62,12 @@ The top section shows a summary of the user: - **Total Revenue**: If you are tracking revenue, the total amount generated by this user. - **Activity Calendar**: A heat map visualization (similar to GitHub contributions) showing the user's activity intensity over the last few months. This helps you quickly identify when the user is most active. +### User Traits + +If your application passes traits when calling [`swetrix.identify()`](/swetrix-js-reference#identify) — an email address, a name, a plan, or anything else you find useful — they're listed in a **User traits** section on the profile. The section is hidden for profiles that have none. + +Traits are merged per key across calls, so the profile always shows the latest value your application sent for each one. See [visitor identification](/visitor-identification#user-traits) for the limits and how to remove a trait. + ### Session History Below the overview, you will see a list of all **Sessions** for this user. A session represents a continuous period of activity. diff --git a/docs/content/docs/api/events.mdx b/docs/content/docs/api/events.mdx index 93272b79e..8bc3d07ff 100644 --- a/docs/content/docs/api/events.mdx +++ b/docs/content/docs/api/events.mdx @@ -104,6 +104,14 @@ All of the values are numbers in milliseconds. | ----- | -------- | -------- | ---------------------------------------------- | | `pid` | `string` | `true` | A project ID to record the heartbeat event for | +### Identify event structure + +| Name | Type | Required | Description | +| ----------- | -------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `pid` | `string` | `true` | A project ID to identify the visitor for | +| `profileId` | `string` | `true` | A unique, stable identifier of the user (e.g. an internal user ID), max 256 characters. It's stored as provided, behind a `usr_` prefix. | +| `traits` | `object` | `false` | A `key` / `value` pair of metadata describing the user, for example `{ email: 'john@example.com', plan: 'premium' }`. Values must be a primitive type (string, number, boolean, null) which will be converted to a string; `null` removes a trait. Max 50 keys, 128 characters per key, and 2000 characters for all keys and values combined. | + ### Revenue event structure @@ -218,6 +226,33 @@ socket.on("error", (err) => { {} ``` +### POST /log/identify + +This endpoint identifies the visitor with your own user ID (e.g. after they log in). It links the visitor's current anonymous profile — derived from the request's IP address and User-Agent — to the identified profile derived from the supplied `profileId`, so the activity recorded before identification is attributed to the identified profile in the dashboard. + +After calling it, keep sending the same `profileId` field with your pageview / custom event payloads so subsequent events are stamped with the identified profile directly. The [tracking script](/swetrix-js-reference#identify) does both automatically via `swetrix.identify()`. + +Notes: + +- An anonymous profile is linked to the **first** identified profile that claims it — repeated calls with the same `profileId` are idempotent, and calls with a different `profileId` won't re-link the anonymous history (but are otherwise harmless). +- Since anonymous profiles rotate monthly, the linking covers the visitor's recent anonymous activity — up to about a month back on that device and network. +- Values like `null`, `undefined`, `guest` or an empty string are rejected with a `400` error, as they would fuse unrelated visitors into a single profile. +- The optional `traits` object is stored against the identified profile and displayed on its dashboard page. Traits are merged per key, so a later call only overwrites the keys it carries and an empty / `null` value removes one. + +```bash title="Request" +curl -i -X POST https://api.swetrix.com/log/identify \ + -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/116.0" \ + -H "X-Client-IP-Address: 192.0.2.1" \ + -H "Content-Type: application/json" \ + -d '{"pid":"YOUR_PROJECT_ID","profileId":"user-12345","traits":{"email":"john@example.com","plan":"premium"}}' +``` + +```json title="Response (201 Created)" +{ "profileId": "usr_user-12345" } +``` + +The returned `profileId` is the identified profile ID your events are stored under — the same value you'd use for [revenue attribution](/analytics-dashboard/revenue-tracking#user-attribution). + ### POST /log/error This endpoint records error events. These error events are later aggregated in the dashboard under the Errors tab. diff --git a/docs/content/docs/script-reference.mdx b/docs/content/docs/script-reference.mdx index 310eb6480..07f71f43f 100644 --- a/docs/content/docs/script-reference.mdx +++ b/docs/content/docs/script-reference.mdx @@ -29,14 +29,14 @@ swetrix.init("YOUR_PROJECT_ID", { }); ``` -| Name | Description | Default value | -| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------- | -| devMode | When set to `true`, localhost events will be sent to server. | `false` | -| disabled | When set to true, the tracking library won't send any data to server.
Useful for development purposes when this value is set based on '.env' var. | `false` | -| respectDNT | By setting this flag to true, we will not collect ANY kind of data about the user with the DNT setting.
This setting is not true by default because our service anonymises all incoming data and does not pass it on to any third parties under any circumstances. | `false` | -| apiURL | If you use Swetrix Cloud, you don't need to change this parameter. If you're self-hosting Swetrix CE, you should put the URL of your API instance here, like `[BASE_URL]/backend/v1/log`. | `'https://api.swetrix.com/log'` | -| profileId | Optional profile ID for long-term user tracking (MAU/DAU). If set, it will be used for all pageviews and events unless overridden per-call. This allows you to track users across sessions and devices. | `undefined` | -| preloadSessionReplay | Set to `true` to preload the session replay recorder after `init()`, or pass `{ rrwebUrl: "https://example.com/rrweb.min.js" }` to load it from a custom URL. Recording only starts after you call `startSessionReplay()`. | `undefined` | +| Name | Description | Default value | +| -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------- | +| devMode | When set to `true`, localhost events will be sent to server. | `false` | +| disabled | When set to true, the tracking library won't send any data to server.
Useful for development purposes when this value is set based on '.env' var. | `false` | +| respectDNT | By setting this flag to true, we will not collect ANY kind of data about the user with the DNT setting.
This setting is not true by default because our service anonymises all incoming data and does not pass it on to any third parties under any circumstances. | `false` | +| apiURL | If you use Swetrix Cloud, you don't need to change this parameter. If you're self-hosting Swetrix CE, you should put the URL of your API instance here, like `[BASE_URL]/backend/v1/log`. | `'https://api.swetrix.com/log'` | +| profileId | Optional profile ID for long-term user tracking (MAU/DAU). If set, it will be used for all pageviews and events unless overridden per-call. This allows you to track users across sessions and devices. To also link the visitor's previous anonymous activity to this profile, use [identify()](#identify) instead. | `undefined` | +| preloadSessionReplay | Set to `true` to preload the session replay recorder after `init()`, or pass `{ rrwebUrl: "https://example.com/rrweb.min.js" }` to load it from a custom URL. Recording only starts after you call `startSessionReplay()`. | `undefined` | ## track() @@ -162,6 +162,71 @@ swetrix.pageview({ }); ``` +## identify() + +Identify the current visitor with your own user ID (e.g. after they log in). + +Calling this function does two things: + +1. **Links the anonymous history** — the visitor's current anonymous profile is linked to the identified profile server-side, so the activity they had before logging in (recent sessions, pageviews, events) is attributed to the identified profile in the dashboard. +2. **Stamps all future events** — every pageview, custom event, error, feature flag and experiment call sent after `identify()` is associated with the identified profile, exactly as if you had passed `profileId` to `init()`. + +```javascript +swetrix.init("YOUR_PROJECT_ID"); +swetrix.trackViews(); + +// After the user logs in (or on page load if they're already logged in) +swetrix.identify("user-12345"); +``` + +Swetrix stores nothing in the browser (no cookies, no `localStorage`), so the identification only lasts for the current page life. Call `identify()` on every page load while the user is logged in — repeated calls with the same ID are deduplicated and cost you nothing extra. + +You can also pass **traits** — arbitrary key / value metadata shown in the "User traits" section of the user's profile: + +```javascript +swetrix.identify("user-12345", { + email: "john@example.com", + name: "John Doe", + plan: "premium", +}); +``` + +Traits are merged per key, so a later call only overwrites the keys it carries and `null` removes a trait. Per call you can send up to 50 keys, keys of up to 128 characters, and 2000 characters for all keys and values combined; values must be strings, numbers, booleans or `null` and are stored as strings. + + + Use a unique, **stable** identifier — your application's internal user ID is the best choice. + Don't use values like `null`, `undefined` or `"guest"`: the API rejects those, as they would fuse + unrelated visitors into a single profile. Surrounding whitespace is trimmed; otherwise the ID you + pass is stored as provided (behind a `usr_` prefix) and shown in your dashboard, so if you want a + user's email on their profile, send it as a trait rather than using it as the ID. + + +A few things to know about how linking works: + +- An anonymous profile is linked to the **first** identified profile that claims it. If a second person logs in with a different account on the same device and network, their identity is stamped on their own events going forward, but the shared anonymous history stays with the first account. +- Anonymous profiles rotate monthly (see [visitor identification](/visitor-identification)), so `identify()` links the visitor's recent anonymous activity — up to about a month back on that device and network. +- Identification works across devices: call `identify()` with the same ID on any device and all of that activity is unified under one profile. + +## setTraits() + +Updates the traits of an already identified visitor without repeating their user ID. Only the keys you pass are touched; pass `null` to remove a trait. + +```javascript +swetrix.setTraits({ plan: "enterprise", trialEndsAt: null }); +``` + +Requires [`identify()`](#identify) (or a `profileId` on `init()`) to have been called first — otherwise there's no profile to attach the traits to and the call is a no-op. The same per-call limits as `identify()` apply. + +## reset() + +Resets the visitor's identity set via `identify()` (e.g. after they log out), so subsequent events are tracked anonymously again. + +```javascript +swetrix.reset(); +``` + +Always call this on logout if your users may share devices — otherwise the next visitor's events would be tracked under the previous user's identity. + ## trackErrors() This function is used to enable automatic client-side error monitoring. It adds an `error` event listener on your site and when an error happens - send it to our APIs. @@ -449,7 +514,7 @@ swetrix.clearExperimentsCache(); ## getProfileId() -Gets the anonymous profile ID for the current visitor. If `profileId` was set via `init`, returns that. Otherwise, requests server to generate one from IP/UA hash. +Gets the profile ID for the current visitor. If the visitor was identified via [identify()](#identify), returns the identified (`usr_`-prefixed) profile ID events are stored under. If `profileId` was set via `init`, returns that. Otherwise, requests server to generate an anonymous one from IP/UA hash. This ID is commonly used for [revenue attribution](/analytics-dashboard/revenue-tracking#user-attribution) — pass it to your payment provider (`swetrix_profile_id` in Stripe metadata or Paddle `customData`) or include it in the request body when posting to [`/log/revenue`](/events-api#post-logrevenue) to link transactions to specific visitors. diff --git a/docs/content/docs/visitor-identification.mdx b/docs/content/docs/visitor-identification.mdx index 43884adc3..3eb974afc 100644 --- a/docs/content/docs/visitor-identification.mdx +++ b/docs/content/docs/visitor-identification.mdx @@ -130,32 +130,68 @@ have an account, or even just have a stable client-side identifier you generate yourself — you can tell Swetrix about it and get **per‑user accuracy that doesn't depend on IP or User‑Agent at all.** -Pass the `profileId` to any of the tracking methods. The script -forwards it to the server, where it's salted and hashed with your -project ID before being stored — so we still never see your raw user -ID, and the ID stored in ClickHouse can't be reversed back to your -database. +The recommended way is `identify()`: ```javascript -// Set it once globally on init — every pageview, custom event, error, -// feature flag, and experiment call inherits it automatically. -swetrix.init("YOUR_PROJECT_ID", { - profileId: "user-12345", -}); - +swetrix.init("YOUR_PROJECT_ID"); swetrix.trackViews(); -// Or override per-call if you need to. -swetrix.track({ - ev: "Sign up", - profileId: "user-12345", -}); +// After the user logs in (or on page load if they're already logged in) +swetrix.identify("user-12345"); + +// On logout +swetrix.reset(); ``` -`profileId` is supported on `init()`, `track()`, `pageview()`, +Your user ID is stored as you send it — bar surrounding whitespace, +which is trimmed — behind a `usr_` prefix, so the profile that shows up +in your dashboard is `usr_user-12345`, and you can look a user up by the +same ID you use in your own database. Only send identifiers you're +comfortable seeing in the dashboard. + +Alternatively, you can pass `profileId` to individual tracking methods — +it's supported on `init()`, `track()`, `pageview()`, `getFeatureFlags()` / `getFeatureFlag()`, and `getExperiments()` / `getExperiment()`. See the [script reference](/swetrix-js-reference) -for the full list. +for the full list. The difference: a plain `profileId` only stamps the +events it's passed with, while `identify()` additionally links the +visitor's anonymous history, as described below. + +### How identify() links anonymous history + +A visitor usually browses your site anonymously before they sign in — +so by the time you know who they are, Swetrix has already recorded +their pageviews under an anonymous `anon_…` fingerprint. Without extra +care, identifying them would simply start a second, separate profile +and the pre-login activity would stay orphaned. + +`identify()` solves this: when called, the server links the visitor's +current anonymous profile to the identified one, and the dashboard +attributes the anonymous activity — sessions, pageviews, events, +errors — to the identified profile. Even the session during which the +user signed in stays intact: the pageviews before and after login show +up as one session under the identified profile. + +The rules, in short: + +- **First identification wins.** An anonymous profile can only ever be + linked to one identified profile. If a second person logs into a + different account on a shared device, their events are tracked under + their own identity going forward, but the shared anonymous history + stays with the first account. This mirrors how PostHog and other + analytics tools guard against runaway profile merging. +- **About a month of history.** Anonymous fingerprints rotate monthly + (see above), so the linking covers the visitor's recent anonymous + activity on that device and network. Each month they return + logged-in, the new anonymous fingerprint is linked too. +- **Cross-device works.** Call `identify()` with the same ID on any + device or browser, and all of that activity is unified under one + profile in the dashboard. +- **Nothing is stored client-side.** Swetrix remains cookieless — call + `identify()` on every page load while the user is logged in + (repeated calls are deduplicated), and call `reset()` on logout so + the next person on a shared device isn't tracked under the previous + user's identity. ### What you can use as a profile ID @@ -169,15 +205,52 @@ choices: works for anonymous users too, but because _you_ (not Swetrix) are now setting a persistent identifier on the device, you should review whether your local privacy regulations require user consent for it. -- **A hashed email or account identifier** if you already use one for - other systems. Just make sure the hash is stable for the same user. +- **A pseudonymous account identifier** if you already use one for other + systems. Just make sure it's stable for the same user. - **A device or installation ID** in mobile or desktop apps embedding a webview. - Don't pass raw email addresses, full names, phone numbers, or any other personal data as - `profileId`. Hash or pseudonymise it on your side first. Swetrix anonymises whatever it receives, - but the cleanest approach is to never send PII in the first place. + The `profileId` is stored as provided, apart from surrounding whitespace, which is trimmed — + Swetrix doesn't hash or otherwise transform it. Prefer an opaque internal ID over email addresses, + full names or phone numbers, and pseudonymise on your side if your privacy policy requires it. If + you do want a user's email on their profile, attach it as a [trait](#user-traits) instead of using + it as the ID. + + +### User traits + +An ID on its own tells you _that_ two sessions belong to the same +person, not _who_ they are. Pass **traits** alongside it and Swetrix +shows them in a "User traits" section on the profile page: + +```javascript +swetrix.identify("user-12345", { + email: "john@example.com", + name: "John Doe", + plan: "premium", + signupDate: "2026-01-14", +}); +``` + +Traits are free-form — any key / value pair you find useful. They're +merged per key across calls, so a later `identify()` (or +[`setTraits()`](/swetrix-js-reference#settraits)) only overwrites the +keys it carries, and passing `null` removes one: + +```javascript +// Later, without repeating the user ID +swetrix.setTraits({ plan: "enterprise", trialEndsAt: null }); +``` + +The limits per call: up to 50 keys, keys of up to 128 characters, and +2000 characters for all keys and values combined. Values must be +strings, numbers, booleans or `null`, and are stored as strings. + + + Traits are the one place where personal data legitimately ends up in Swetrix — an email address on + a profile is exactly what the feature is for. Send only what you actually need, and make sure your + privacy policy covers it. ### Notes on consent @@ -212,8 +285,9 @@ swetrix.trackViews(); // Upgrade to accurate per-user tracking once the user signs in // (or once the user grants consent in your banner, if applicable). +// This also links their pre-login anonymous activity to the profile. afterLogin((user) => { - swetrix.init("YOUR_PROJECT_ID", { profileId: user.id }); + swetrix.identify(user.id); }); ``` @@ -236,6 +310,7 @@ chosen. | Accuracy on a typical consumer site | High — collisions are rare | Per-user accurate (for signed-in users) | | Accuracy behind a school / office NAT | Lower — devices may collide | Per-user accurate | | Cross-device tracking | No | Yes (same `profileId` on each) | +| Pre-login activity linked | — | Yes (via `identify()`) | | Works without JavaScript | Yes (via the noscript pixel) | No (needs the JS tracker) | Pick the default for marketing sites, blogs, docs, and anywhere you diff --git a/packages/tracker-js/README.md b/packages/tracker-js/README.md index e34620fa0..d825fbaf8 100644 --- a/packages/tracker-js/README.md +++ b/packages/tracker-js/README.md @@ -260,6 +260,31 @@ await startSessionReplay({ }) ``` +### `identify(profileId, traits?)` / `setTraits(traits)` / `reset()` + +Identify the current visitor with your own user ID (e.g. after they log in). Their current anonymous profile gets linked to the identified profile server-side, so pre-login activity is attributed to it, and all subsequent events are tracked under the identified profile. + +```javascript +// After the user logs in (or on page load if they're already logged in) +identify('user-12345') + +// On logout +reset() +``` + +Use a unique, stable identifier (e.g. an internal user ID). The ID is stored as provided and shown in your dashboard, so don't pass values you wouldn't want to see there. Swetrix stores nothing in the browser, so call `identify()` on every page load while the user is logged in. + +Optionally pass **traits** — key / value metadata displayed on the user's profile: + +```javascript +identify('user-12345', { email: 'john@example.com', plan: 'premium' }) + +// Later, without repeating the user ID. null removes a trait +setTraits({ plan: 'enterprise', trialEndsAt: null }) +``` + +Traits are merged per key. Per call: max 50 keys, 128 characters per key, and 2000 characters for all keys and values combined. + ### Session & Profile IDs ```javascript diff --git a/packages/tracker-js/src/Lib.ts b/packages/tracker-js/src/Lib.ts index c06db66ac..dc78d0f1f 100644 --- a/packages/tracker-js/src/Lib.ts +++ b/packages/tracker-js/src/Lib.ts @@ -46,6 +46,12 @@ declare global { } } +/** + * Key / value metadata describing an identified user (email, plan, signup + * date, ...). Values are stored as strings; `null` removes a trait. + */ +export type Traits = Record + export interface LibOptions { /** * When set to `true`, localhost events will be sent to server. @@ -356,6 +362,10 @@ export class Lib { private rrwebLoader: Promise | null = null private sessionReplayActions: SessionReplayActions | null = null private sessionReplayInitPromise: Promise | null = null + // The server-side (usr_-prefixed) profile ID returned by the identify API + private identifiedProfileId: string | null = null + // The last profile ID sent to the identify API (to avoid duplicate requests) + private lastIdentifySent: string | null = null constructor(private projectID: string, private options?: LibOptions) { this.trackPathChange = this.trackPathChange.bind(this) @@ -646,6 +656,107 @@ export class Lib { this.cachedData = null } + /** + * Identify the current visitor with your own user ID (e.g. after they log in). + * + * The visitor's current anonymous profile gets linked to the identified + * profile server-side, so their pre-login activity is attributed to it. All + * events sent after this call are associated with the identified profile. + * + * Swetrix stores nothing in the browser, so call identify() on every page + * load while the user is logged in. Call reset() when they log out. + * + * @param profileId A unique, stable identifier of the user, e.g. an internal + * user ID. It's stored as you provide it, so don't pass values you wouldn't + * want to see in your dashboard. + * @param traits Optional key / value metadata to show on the user's profile, + * e.g. their email, plan or signup date. Traits are merged with the ones + * already stored; pass `null` to remove one. + */ + async identify(profileId: string, traits?: Traits): Promise { + if (typeof profileId !== 'string' || !profileId.trim()) { + console.error('[Swetrix] identify() expects a non-empty string profileId') + return + } + + const trimmed = profileId.trim() + + this.options = { + ...this.options, + profileId: trimmed, + } + + // Traits are part of the payload, so re-send when only they changed + const identifyKey = `${trimmed}:${traits ? JSON.stringify(traits) : ''}` + + if (!this.canTrack() || this.lastIdentifySent === identifyKey) { + return + } + + this.lastIdentifySent = identifyKey + + // The profile changed, so cached flags / experiments may no longer apply + this.clearFeatureFlagsCache() + + try { + const apiBase = this.getApiBase() + const response = await fetch(`${apiBase}/log/identify`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ pid: this.projectID, profileId: trimmed, traits }), + }) + + if (!response.ok) { + return + } + + const data = (await response.json()) as { profileId: string | null } + this.identifiedProfileId = data.profileId || null + } catch { + // Keep the profileId set locally even if the identify request failed - + // events will still be attributed to the identified profile + } + } + + /** + * Updates the traits of the already identified visitor without having to + * repeat their user ID. Only the keys you pass are touched; pass `null` to + * remove a trait. + */ + async setTraits(traits: Traits): Promise { + const profileId = this.options?.profileId + + if (!profileId) { + console.error('[Swetrix] setTraits() requires the visitor to be identified via identify() first') + return + } + + if (!traits || typeof traits !== 'object' || Array.isArray(traits)) { + console.error('[Swetrix] setTraits() expects an object of traits') + return + } + + await this.identify(profileId, traits) + } + + /** + * Resets the visitor's identity set via identify() (e.g. after they log + * out), so subsequent events are tracked anonymously again. Important on + * shared devices - otherwise the next visitor would be tracked under the + * previous user's identity. + */ + reset(): void { + if (this.options) { + delete this.options.profileId + } + + this.identifiedProfileId = null + this.lastIdentifySent = null + this.clearFeatureFlagsCache() + } + /** * Fetches variant assignments for running A/B test experiments returned by feature flag evaluation. * Results are cached for 5 minutes by default (shared cache with feature flags). @@ -747,6 +858,12 @@ export class Lib { * ``` */ async getProfileId(): Promise { + // If the visitor was identified, return the server-side (usr_-prefixed) + // profile ID, which is what events are stored under + if (this.identifiedProfileId) { + return this.identifiedProfileId + } + // If profileId is already set in options, return it if (this.options?.profileId) { return this.options.profileId diff --git a/packages/tracker-js/src/index.ts b/packages/tracker-js/src/index.ts index 1b6ae2527..3802eb611 100644 --- a/packages/tracker-js/src/index.ts +++ b/packages/tracker-js/src/index.ts @@ -14,6 +14,7 @@ import { SessionReplayOptions, SessionReplayActions, defaultSessionReplayActions, + Traits, } from './Lib.js' export let LIB_INSTANCE: Lib | null = null @@ -278,6 +279,69 @@ export function clearExperimentsCache(): void { LIB_INSTANCE.clearExperimentsCache() } +/** + * Identify the current visitor with your own user ID (e.g. after they log in). + * + * The visitor's current anonymous profile gets linked to the identified profile + * server-side, so their pre-login activity is attributed to it. All events sent + * after this call are associated with the identified profile. + * + * Swetrix stores nothing in the browser, so call identify() on every page load + * while the user is logged in. Call reset() when they log out. + * + * @param profileId A unique, stable identifier of the user, e.g. an internal + * user ID. It's stored as you provide it, so don't pass values you wouldn't + * want to see in your dashboard. + * @param traits Optional key / value metadata to show on the user's profile, + * e.g. their email, plan or signup date. Traits are merged with the ones + * already stored; pass `null` to remove one. + * + * @example + * ```typescript + * // After the user logs in (or on page load if they're already logged in) + * swetrix.identify('user-12345', { + * email: 'john@example.com', + * plan: 'premium', + * }) + * + * // On logout + * swetrix.reset() + * ``` + */ +export async function identify(profileId: string, traits?: Traits): Promise { + if (!LIB_INSTANCE) return + + await LIB_INSTANCE.identify(profileId, traits) +} + +/** + * Updates the traits of the already identified visitor without having to + * repeat their user ID. Only the keys you pass are touched; pass `null` to + * remove a trait. + * + * @example + * ```typescript + * swetrix.setTraits({ plan: 'enterprise', trialEndsAt: null }) + * ``` + */ +export async function setTraits(traits: Traits): Promise { + if (!LIB_INSTANCE) return + + await LIB_INSTANCE.setTraits(traits) +} + +/** + * Resets the visitor's identity set via identify() (e.g. after they log out), + * so subsequent events are tracked anonymously again. Important on shared + * devices - otherwise the next visitor would be tracked under the previous + * user's identity. + */ +export function reset(): void { + if (!LIB_INSTANCE) return + + LIB_INSTANCE.reset() +} + /** * Gets the anonymous profile ID for the current visitor. * If profileId was set via init options, returns that. @@ -346,4 +410,5 @@ export { IPageViewPayload, FeatureFlagsOptions, ExperimentOptions, + Traits, } diff --git a/packages/tracker-node/README.md b/packages/tracker-node/README.md index 85c8743fb..4fb6d6642 100644 --- a/packages/tracker-node/README.md +++ b/packages/tracker-node/README.md @@ -433,6 +433,29 @@ const profileId = await swetrix.getProfileId('192.155.52.12', 'Mozilla/5.0...') If you set a `profileId` in the constructor options, it will be returned directly instead of generating one. +## Identifying users + +Link a visitor's anonymous profile to your own user ID (e.g. after they log in), so their pre-login activity is attributed to the identified profile: + +```javascript +const identifiedProfileId = await swetrix.identify('192.155.52.12', 'Mozilla/5.0...', 'user-12345') +``` + +Use a unique, stable identifier (e.g. an internal user ID). Surrounding whitespace is trimmed; otherwise the ID is stored as provided and shown in your dashboard, so don't pass values you wouldn't want to see there. + +You can also pass **traits** — key / value metadata displayed on the user's profile. Traits are merged per key, and `null` removes one: + +```javascript +await swetrix.identify('192.155.52.12', 'Mozilla/5.0...', 'user-12345', { + email: 'john@example.com', + plan: 'premium', +}) +``` + +Per call: max 50 keys, 128 characters per key, and 2000 characters for all keys and values combined. + +Note: unlike the browser tracker, `identify()` does not set a default `profileId` for subsequent calls — a `Swetrix` instance is shared across all visitors of your server. Keep passing `profileId` per `track()` / `trackPageView()` call. + ## Session ID Get the current session ID for the visitor. Session IDs are generated server-side based on IP and user agent. diff --git a/packages/tracker-node/src/index.ts b/packages/tracker-node/src/index.ts index 5cd73fee4..30ead6f14 100644 --- a/packages/tracker-node/src/index.ts +++ b/packages/tracker-node/src/index.ts @@ -223,6 +223,12 @@ export interface ExperimentOptions { profileId?: string } +/** + * Key / value metadata describing an identified user (email, plan, signup + * date, ...). Values are stored as strings; `null` removes a trait. + */ +export type Traits = Record + const DEFAULT_API_HOST = 'https://api.swetrix.com/log' const DEFAULT_API_BASE = 'https://api.swetrix.com' @@ -557,6 +563,63 @@ export class Swetrix { } } + /** + * Identify a visitor with your own user ID (e.g. after they log in), + * implements https://docs.swetrix.com/events-api#post-logidentify. + * + * The visitor's current anonymous profile (derived from their IP and user + * agent) gets linked to the identified profile server-side, so their + * pre-login activity is attributed to it. + * + * Note: unlike the browser tracker, this does NOT set a default profileId + * for subsequent calls - a Swetrix instance is shared across all visitors + * of your server. Keep passing `profileId` per track / trackPageView call. + * + * @param ip IP address of the visitor + * @param userAgent User agent of the visitor + * @param profileId A unique, stable identifier of the user, e.g. an internal + * user ID. It's stored as you provide it, so don't pass values you wouldn't + * want to see in your dashboard. + * @param traits Optional key / value metadata to show on the user's profile, + * e.g. their email, plan or signup date. Traits are merged with the ones + * already stored; pass `null` to remove one. + * @returns A promise that resolves to the identified (usr_-prefixed) profile + * ID events are stored under, or null on error. + */ + public async identify(ip: string, userAgent: string, profileId: string, traits?: Traits): Promise { + if (!this.canTrack()) { + return null + } + + if (typeof profileId !== 'string' || !profileId.trim()) { + this.debug('identify() expects a non-empty string profileId', true) + return null + } + + try { + const apiBase = this.getApiBase() + const response = await fetch(`${apiBase}/log/identify`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Client-IP-Address': ip, + 'User-Agent': userAgent, + }, + body: JSON.stringify({ pid: this.projectID, profileId: profileId.trim(), traits }), + }) + + if (!response.ok) { + return null + } + + const data = (await response.json()) as { profileId: string | null } + return data.profileId + } catch (error) { + this.debug(`Error identifying profile: ${error}`, true) + return null + } + } + private async fetchFlagsAndExperiments( ip: string, userAgent: string, diff --git a/web/app/lib/models/Project.ts b/web/app/lib/models/Project.ts index 305e6459a..cd793300d 100644 --- a/web/app/lib/models/Project.ts +++ b/web/app/lib/models/Project.ts @@ -150,6 +150,7 @@ export interface Profile { export interface ProfileDetails extends Profile { avgDuration: number + traits?: Record region: string | null city: string | null locale: string | null diff --git a/web/app/pages/Project/tabs/Profiles/ProfileDetails.tsx b/web/app/pages/Project/tabs/Profiles/ProfileDetails.tsx index 23b0dce53..802c66233 100644 --- a/web/app/pages/Project/tabs/Profiles/ProfileDetails.tsx +++ b/web/app/pages/Project/tabs/Profiles/ProfileDetails.tsx @@ -609,6 +609,11 @@ export const ProfileDetails = ({ [details?.lastSeen], ) + const traits = useMemo( + () => Object.entries(details?.traits || {}), + [details?.traits], + ) + if (!details) return const avgDurationStr = details.avgDuration @@ -983,6 +988,20 @@ export const ProfileDetails = ({ + {_isEmpty(traits) ? null : ( + +
+ {traits.map(([key, value]) => ( + {value}} + /> + ))} +
+ + )} +