Merge pull request #1403 from supertokens/agent/issue-1397-emit-creation-disassoc

feat: emit user_creation and tenant_disassociation lifecycle events

Author: tamassoltesz

CHANGELOG.md

@@ -10,7 +10,13 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 - `ActiveUsers.updateLastActive` no longer writes `user_last_active` directly; it only appends the throttled `user_last_active` activity-log event, and the `RollupUserLastActive` cron is now the sole writer of the projection (so counts reflect activity within a rollup interval).
 - Added Phase-1 parity tests proving the last-active rollup derives the same `countUsersActiveSince` answer (and per-user projection) as the direct write, including link/unlink cases (`ActivityLogRollupParityTest`).
 - Adds an observability-only shadow audit to the approximate-user-count background refresh; discrepancies are logged and emitted as telemetry, never served.
+- The `allowApproximate` user-count path now serves the exact anchor plus a fold of lifecycle events since the anchor, making the approximate count exact for creations, deletions and account (un)linking (no API change).
+- From CDI 5.6 the default single-tenant, unfiltered `/users/count` path serves `anchor + fold` from the lifecycle-event ledger instead of an exact recompute per request, always returning the `approximate`/`asOf` fields and making the `allowApproximate` parameter a no-op; older CDI versions are unchanged.
+- Emits `user_creation` and `tenant_disassociation` lifecycle events atomically with the mutation from the interactive user-creation and tenant-removal paths.
+- Bulk import now emits lifecycle events atomically with the import: one `user_import` per imported user (counted toward user totals like `user_creation`, but under its own type so the last-active rollup can exclude imports) plus a `tenant_association` for each remaining tenant the user lands in.
 - The last-active rollup fold now skips activity for apps no longer present in `apps`, so a deleted app's retained `activity_log` rows can never resurrect a `user_last_active` projection row (which would violate its `apps` foreign key).
+- Replaced the synthetic `user_last_active` event with semantic activity events (`sign_in`, `token_refresh`, `session_create`, `sign_out`, `oauth_token_exchange`, `oauth_authorize`); the last-active fold now reads these plus the `user_creation` and `account_linking` lifecycle events, and a new protected config `activity_log_throttle_enabled` (boolean, default `true`, per connection URI domain) toggles the per-`(app, user)` write throttle on the throttled events.
+- Sign-up and account-linking now wake the last-active rollup, so a user with no other activity is folded into `user_last_active` on the next tick instead of waiting for the periodic backstop pass.
 
 ## [12.2.0]
 

config.yaml

@@ -215,6 +215,13 @@ core_config_version: 0
 # by a periodic cleanup. Must be the same for all apps/tenants under a connection URI domain.
 # activity_log_retention_days:
 
+# (OPTIONAL | Default: true) boolean value. If true, throttled activity events (token_refresh, session_create,
+# oauth_token_exchange, oauth_authorize) are collapsed to at most one activity_log write per (app, user) every 5
+# minutes, so a burst of refreshes does not turn into a per-request insert. sign_in and sign_out are never
+# throttled. Set to false to record every activity event as its own row (a complete audit trail) at the cost of
+# that write volume. Must be the same for all apps/tenants under a connection URI domain.
+# activity_log_throttle_enabled:
+
 # (DIFFERENT_ACROSS_APPS | OPTIONAL | Default: 3600000) long value. Time in milliseconds for how long a webauthn
 # account recovery token is valid for.
 # webauthn_recover_account_token_lifetime:

devConfig.yaml

@@ -214,6 +214,13 @@ bcrypt_log_rounds: 4
 # by a periodic cleanup. Must be the same for all apps/tenants under a connection URI domain.
 # activity_log_retention_days:
 
+# (OPTIONAL | Default: true) boolean value. If true, throttled activity events (token_refresh, session_create,
+# oauth_token_exchange, oauth_authorize) are collapsed to at most one activity_log write per (app, user) every 5
+# minutes, so a burst of refreshes does not turn into a per-request insert. sign_in and sign_out are never
+# throttled. Set to false to record every activity event as its own row (a complete audit trail) at the cost of
+# that write volume. Must be the same for all apps/tenants under a connection URI domain.
+# activity_log_throttle_enabled:
+
 # (DIFFERENT_ACROSS_APPS | OPTIONAL | Default: 3600000) long value. Time in milliseconds for how long a webauthn
 # account recovery token is valid for.
 # webauthn_recover_account_token_lifetime:

src/main/java/io/supertokens/ActiveUsers.java

@@ -1,10 +1,12 @@
 package io.supertokens;
 
 import io.supertokens.auditlog.AuditLog;
+import io.supertokens.config.Config;
 import io.supertokens.cronjobs.rollupUserLastActive.RollupDirtySignal;
 import io.supertokens.pluginInterface.ActiveUsersSQLStorage;
 import io.supertokens.pluginInterface.Storage;
 import io.supertokens.pluginInterface.StorageUtils;
+import io.supertokens.pluginInterface.auditlog.ActivityEventType;
 import io.supertokens.pluginInterface.auditlog.AuditLogEvent;
 import io.supertokens.pluginInterface.exceptions.StorageQueryException;
 import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException;
@@ -14,17 +16,34 @@
 import io.supertokens.storageLayer.StorageLayer;
 import org.jetbrains.annotations.TestOnly;
 
+import java.util.EnumSet;
 import java.util.concurrent.ConcurrentHashMap;
 import io.supertokens.auditlog.UnauditedTransaction;
 
 public class ActiveUsers {
 
-    // Skip the user_last_active upsert if we already wrote one for this (app, userId) within
-    // this window. The table feeds daily/monthly active-user counts, so a few minutes of
-    // staleness is invisible — but at refresh-token rates the unthrottled upsert dominates
-    // commit waits on the database.
+    // Skip appending a throttled activity event if we already wrote one for this (app, userId) within
+    // this window. The activity log feeds daily/monthly active-user counts (via the fold), so a few
+    // minutes of staleness is invisible — but at refresh-token rates an unthrottled insert dominates
+    // commit waits on the database. Unthrottled activity classes (sign_in, sign_out) bypass this.
     private static final long THROTTLE_MS = 5 * 60 * 1000L;
 
+    // Throttle policy for the shared plugin-interface {@link ActivityEventType} vocabulary. The vocabulary
+    // deliberately carries no throttle flag — its javadoc keeps throttling core-side — so which classes are
+    // throttled is decided here: sign_in / sign_out are low-volume, user-initiated and audit-meaningful, so
+    // they always emit; every other activity class is high-volume and shares the throttle.
+    private static final EnumSet<ActivityEventType> UNTHROTTLED_EVENTS =
+            EnumSet.of(ActivityEventType.SIGN_IN, ActivityEventType.SIGN_OUT);
+
+    /**
+     * @return whether emits of {@code eventType} are subject to the shared 5-minute per-{@code (app, user)}
+     * throttle. {@code sign_in} / {@code sign_out} return {@code false} (always emitted); the rest return
+     * {@code true}. Core-side policy over the plugin-interface {@link ActivityEventType} vocabulary.
+     */
+    public static boolean isThrottled(ActivityEventType eventType) {
+        return !UNTHROTTLED_EVENTS.contains(eventType);
+    }
+
     // Hard cap on cache size. Beyond this we sweep expired entries; if still over we clear.
     // Extra upserts for a window are acceptable; unbounded memory growth is not.
     private static final int MAX_CACHE_ENTRIES = 200_000;
@@ -72,45 +91,106 @@ public static void markRecentlyActive(AppIdentifier appIdentifier, String userId
         recordActiveAt(cacheKey(appIdentifier, userId), System.currentTimeMillis());
     }
 
-    public static void updateLastActive(AppIdentifier appIdentifier, Main main, String userId)
+    /**
+     * Records a unit of user activity of the given {@code eventType}, emitting it into the request's tenant.
+     * The last-active rollup cron is the sole writer of the {@code user_last_active} projection (PLAN-011
+     * cutover); here we only append the activity-log event — the fold's source — and mark the storage dirty
+     * so the next rollup pass folds it. When the {@code activity_log_throttle_enabled} config is on (the
+     * default), throttled activity classes ({@link #isThrottled(ActivityEventType)}) skip the append when
+     * this (app, user) was seen within the throttle window and unthrottled classes always append — either way
+     * the recency cache is refreshed. When the config is off, the throttle and its cache are bypassed and
+     * every activity is recorded as its own row (a complete audit trail). The projection updates
+     * asynchronously (within a rollup interval).
+     *
+     * <p>The append is best-effort: {@link AuditLog#emit} swallows its own write failures, so a failed
+     * activity write never fails the caller's request. That matters because these events are emitted after an
+     * already-committed (and, for OAuth, externally non-reversible) auth operation — a transient activity-log
+     * error must not turn a succeeded sign-in / refresh / session-create / sign-out / oauth call into a 500.
+     * Dropped rows self-heal for active-user counting: the next event for the user re-credits them, and the
+     * reliable recency anchors are the transactional {@code user_creation} / {@code account_linking} lifecycle
+     * events.
+     */
+    public static void updateLastActive(TenantIdentifier tenantIdentifier, Main main, String userId,
+                                        ActivityEventType eventType)
             throws TenantOrAppNotFoundException {
+        AppIdentifier appIdentifier = tenantIdentifier.toAppIdentifier();
         long now = System.currentTimeMillis();
         String key = cacheKey(appIdentifier, userId);
-        if (!Main.isTesting && isRecentlyActive(key, now)) {
-            return;
+        boolean throttleEnabled = Config.getConfig(appIdentifier.getAsPublicTenantIdentifier(), main)
+                .getActivityLogThrottleEnabled();
+        if (throttleEnabled && !Main.isTesting) {
+            if (isThrottled(eventType) && isRecentlyActive(key, now)) {
+                return;
+            }
+            // Refresh the recency cache so a subsequent throttled event (and wasRecentlyActive) sees this
+            // activity. Only meaningful while throttling is on; when off we never touch the cache, so
+            // wasRecentlyActive stays false and every activity is recorded.
+            recordActiveAt(key, now);
         }
+        // The activity log and its projection live on the app's public-tenant storage — as before, so the
+        // fold (which groups by app_id) and the count read see the same rows. The request's tenant is written
+        // into the tenant_id column for provenance only.
         Storage storage = StorageLayer.getStorage(appIdentifier.getAsPublicTenantIdentifier(), main);
-        // The last-active rollup cron is the sole writer of user_last_active (PLAN-011 cutover). Here we only
-        // append the throttled user_last_active activity-log event — the fold's source — and mark the storage
-        // dirty so the next rollup pass folds it. The 5-minute throttle now caps activity-log insert volume
-        // instead of direct-upsert volume. The projection updates asynchronously (within a rollup interval).
-        recordActiveAt(key, now);
-        emitLastActiveAuditLog(main, storage, appIdentifier, userId, now);
+        emitActivityAuditLog(main, storage, tenantIdentifier, userId, eventType, now);
     }
 
     /**
-     * Records a {@code user_last_active} entry in the activity_log table. Mirrors every successful
-     * user_last_active write so the audit log captures user activity. Best-effort: {@link AuditLog#emit}
-     * swallows its own failures, so a failed audit write never affects the active-users update.
+     * Overload for callers that only have the app on hand (no request tenant): the event is emitted into the
+     * app's public tenant — today's behavior for every activity emit before per-tenant provenance was added.
      */
-    private static void emitLastActiveAuditLog(Main main, Storage storage, AppIdentifier appIdentifier,
-                                               String userId, long now) {
-        TenantIdentifier tenantIdentifier = appIdentifier.getAsPublicTenantIdentifier();
+    public static void updateLastActive(AppIdentifier appIdentifier, Main main, String userId,
+                                        ActivityEventType eventType)
+            throws TenantOrAppNotFoundException {
+        updateLastActive(appIdentifier.getAsPublicTenantIdentifier(), main, userId, eventType);
+    }
+
+    /**
+     * Appends an activity event to the activity_log so the last-active fold captures the user's activity.
+     * Best-effort: {@link AuditLog#emit} swallows its own failures, so a failed audit write never affects the
+     * request. {@code tenant_id} carries the request's tenant; {@code event_type} is {@code eventType}'s value.
+     */
+    private static void emitActivityAuditLog(Main main, Storage storage, TenantIdentifier tenantIdentifier,
+                                             String userId, ActivityEventType eventType, long now) {
         AuditLog.emit(main, storage, tenantIdentifier, new AuditLogEvent(
-                appIdentifier.getAppId(), tenantIdentifier.getTenantId(),
+                tenantIdentifier.getAppId(), tenantIdentifier.getTenantId(),
                 userId, userId,
-                "user_last_active", "success", null, null,
+                eventType.getValue(), "success", null, null,
                 now, null));
         // Signal the last-active rollup cron that this storage now has unfolded activity, so its next tick
         // folds instead of skipping.
         RollupDirtySignal.getInstance(main).markDirty(storage.getUserPoolId());
     }
 
+    /**
+     * Wakes the last-active rollup for a user whose fold credit comes from a transactional lifecycle event —
+     * {@code user_creation} on sign-up, {@code account_linking} on link (the two lifecycle members of the
+     * fold set, see {@code RollupEventTypes#FOLD_SET}) — rather than from {@link #updateLastActive}. Those events
+     * are written on the mutation's own connection via {@code startAuditedTransaction}, which — unlike
+     * {@code updateLastActive} / {@link #emitActivityAuditLog} — does not touch the rollup dirty signal.
+     *
+     * <p>Without this nudge a user who only signs up (or is only linked) and produces no other activity would
+     * not be folded into {@code user_last_active} until the periodic backstop pass — up to a backstop
+     * interval — a promptness regression versus the pre-semantic-event behaviour where sign-up went through
+     * {@code updateLastActive → markDirty} and folded on the next rollup tick.
+     *
+     * <p>Call after the lifecycle event's transaction has committed. Marking dirty only signals <em>that</em>
+     * there is something to fold, never the fold window, so it is idempotent and safe to over-signal; a lost
+     * signal is corrected by the cron's periodic backstop.
+     */
+    public static void markLastActiveRollupDirty(Main main, AppIdentifier appIdentifier)
+            throws TenantOrAppNotFoundException {
+        // The projection and its dirty flag are keyed by the app's public-tenant storage pool — the same
+        // storage updateLastActive marks dirty — so a fold-relevant lifecycle event written on any tenant in
+        // the pool wakes the one rollup pass that folds it.
+        Storage storage = StorageLayer.getStorage(appIdentifier.getAsPublicTenantIdentifier(), main);
+        RollupDirtySignal.getInstance(main).markDirty(storage.getUserPoolId());
+    }
+
     @TestOnly
     public static void updateLastActive(Main main, String userId) {
         try {
             ActiveUsers.updateLastActive(ResourceDistributor.getAppForTesting().toAppIdentifier(),
-                    main, userId);
+                    main, userId, ActivityEventType.SIGN_IN);
         } catch (TenantOrAppNotFoundException e) {
             throw new IllegalStateException(e);
         }
@@ -138,20 +218,20 @@ public static void updateLastActiveAfterLinking(Main main, AppIdentifier appIden
         // Latency optimization only: the rollup's reconcile — driven by the account_linking event that
         // AuthRecipe.linkAccounts emits atomically with the mapping change — is the source of truth for
         // dropping the recipe user's now-stale projection row. Deleting it here just makes the merge visible
-        // before the next rollup pass instead of after it.
+        // before the next rollup pass instead of after it. The primary user's refreshed recency comes from
+        // the same account_linking event (the fold credits primary_or_recipe_user_id), so no activity ping is
+        // emitted here.
         activeUsersStorage.startTransaction(con -> {
             activeUsersStorage.deleteUserActive_Transaction(con, appIdentifier, recipeUserId);
             return null;
         });
         recentlyActiveCache.remove(cacheKey(appIdentifier, recipeUserId));
 
-        // Bypass throttle: linking merges two users into primaryUserId, so its timestamp must
-        // be refreshed to "now" regardless of cache state — it now represents the merged
-        // activity and an undercounted timestamp would lose the recipeUser's recency. Emitted into the
-        // activity log (not written directly) so the rollup — the sole user_last_active writer — folds it.
-        long now = System.currentTimeMillis();
-        recordActiveAt(cacheKey(appIdentifier, primaryUserId), now);
-        emitLastActiveAuditLog(main, activeUsersStorage, appIdentifier, primaryUserId, now);
+        // The primary user's refreshed recency is credited by the account_linking lifecycle event
+        // AuthRecipe.linkAccounts emitted transactionally — but that event, written via startAuditedTransaction,
+        // does not mark the rollup dirty. Wake the rollup here so a link with no other activity folds on the
+        // next tick rather than waiting for the periodic backstop.
+        RollupDirtySignal.getInstance(main).markDirty(activeUsersStorage.getUserPoolId());
     }
 
     @TestOnly

src/main/java/io/supertokens/auditlog/lifecycle/CountDeltaInterpreter.java

@@ -34,7 +34,9 @@
  * derived here from those lists, per {@link LifecycleEventType}:
  *
  * <ul>
- *   <li>{@code USER_CREATION} — {@code +1} for the created-in tenant (a new standalone group).</li>
+ *   <li>{@code USER_CREATION} / {@code USER_IMPORT} — {@code +1} for the tenant the (new or imported) user
+ *       lands in. The two are folded identically here; the distinction is for the last-active rollup, not the
+ *       user count.</li>
  *   <li>{@code USER_GROUP_DELETION} — {@code -1} for every tenant the group was present in.</li>
  *   <li>{@code USER_DELETION} — {@code after − before} per tenant: {@code -1} for each tenant the group left
  *       (its last member there was deleted) and {@code +1} for any it newly appears in (not expected for a
@@ -147,6 +149,10 @@ public static long computeDeltaForTenant(List<LifecycleEventPayload> events, Str
     private static void applyEvent(LifecycleEventPayload event, Map<String, Long> deltas) {
         switch (event.type) {
             case USER_CREATION:
+            case USER_IMPORT:
+                // A bulk-imported user is counted toward totals exactly like an interactively created one:
+                // a +1 in the tenant it lands in. (The type distinction matters only to the last-active
+                // rollup, which folds user_creation but excludes user_import, and never enters this fold.)
                 add(deltas, event.tenantId, 1);
                 break;
             case USER_GROUP_DELETION: