fix(dashboard): cache org stats in Redis with versioned invalidation (#883)

* fix(dashboard): cache org stats in Redis with versioned invalidation

Replace the never-expiring process-local cache in the org dashboard page
load with API-side Redis caching keyed per org. A version stamp prevents
slow reads from writing pre-purge data back into the cache.

Invalidate on course create/update/delete, student enroll/remove, certificate
claim, and progress reset. Remove the dashboard SSR module cache so every
page load reads through the API cache layer.

Fixes #852

Co-authored-by: Rotimi Best <[email protected]>

* fix(dashboard): address PR review on stats cache invalidation

- Validate bust via ZDashStats instead of raw query access
- Use import type for OrganisationAnalytics
- Resolve orgId before course delete so invalidation is reliable
- Invalidate immediately after persisted writes, before fallible side effects
- Invalidate on member role changes and program course access enrollment

Co-authored-by: Rotimi Best <[email protected]>

* fix(dashboard): complete stats cache invalidation coverage

Wire org stats cache purge on lesson completion (manual toggle and
video-watch auto-complete) so topCourses completion percentages stay
fresh. Harden bulk member adds with sequential inserts and a finally
invalidation boundary, and only purge org invite enrollments when new
group memberships were actually inserted.

Co-authored-by: Rotimi Best <[email protected]>

* fix(core): make lesson completion stats purge best-effort

Wrap org stats cache invalidation in try/catch so a lookup or Redis
failure cannot turn a successful lesson completion into a 500.

Co-authored-by: Rotimi Best <[email protected]>

* fix(dashboard): keep org stats cache through lesson completion

Lesson complete is a high-frequency event and usually does not change
the rounded dashboard percentages. Rely on a 10 minute TTL instead of
purging the snapshot on every completion.

Co-authored-by: Rotimi Best <[email protected]>

* docs: explain how admin dashboard numbers are calculated

Add a user-facing guide covering headline counts, top courses,
certificates, login activity, and when the 10 minute snapshot refreshes.

Co-authored-by: Rotimi Best <[email protected]>

---------

apps/api/src/__tests__/dash-org-stats-cache.test.ts

@@ -0,0 +1,133 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const redisMocks = vi.hoisted(() => ({
+  mGet: vi.fn(),
+  setEx: vi.fn(),
+  incr: vi.fn(),
+  del: vi.fn()
+}));
+
+vi.mock('@cio/core/config/env', () => ({
+  env: { REDIS_URL: 'redis://localhost:6379' }
+}));
+
+vi.mock('@cio/core/utils/redis/redis', () => ({
+  redis: redisMocks,
+  logRedisUnavailableOnce: vi.fn()
+}));
+
+vi.mock('@cio/db/queries/dash', () => ({
+  getDashOrgStats: vi.fn(),
+  getCourseStats: vi.fn(),
+  getRecentCertifications: vi.fn(),
+  getTotalCertificatesIssued: vi.fn(),
+  getOrgStudentLoginsByDayOfWeek: vi.fn(),
+  getUserLoginStreak: vi.fn()
+}));
+
+vi.mock('@cio/db/queries', () => ({
+  getOrgIdBySiteName: vi.fn()
+}));
+
+import {
+  getCourseStats,
+  getDashOrgStats,
+  getRecentCertifications,
+  getTotalCertificatesIssued
+} from '@cio/db/queries/dash';
+import { getOrganisationAnalytics } from '@api/services/dash';
+
+const sampleAnalytics = {
+  totalCertificates: 2,
+  numberOfCourses: 8,
+  totalStudents: 10,
+  topCourses: [],
+  recentCertifications: []
+};
+
+describe('getOrganisationAnalytics cache', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    redisMocks.mGet.mockResolvedValue([null, null]);
+    vi.mocked(getDashOrgStats).mockResolvedValue([{ orgId: 'org-1', noOfCourses: 8, enrolledStudents: 10 }]);
+    vi.mocked(getCourseStats).mockResolvedValue([]);
+    vi.mocked(getRecentCertifications).mockResolvedValue([]);
+    vi.mocked(getTotalCertificatesIssued).mockResolvedValue([{ count: 2 }]);
+  });
+
+  it('computes from the database on cache miss and writes the payload', async () => {
+    const result = await getOrganisationAnalytics('org-1');
+
+    expect(result.numberOfCourses).toBe(8);
+    expect(getDashOrgStats).toHaveBeenCalledTimes(1);
+    expect(redisMocks.setEx).toHaveBeenCalledWith(
+      'dash:stats:v1:org-1',
+      600,
+      JSON.stringify({ version: '0', data: sampleAnalytics })
+    );
+  });
+
+  it('returns cached payload when version matches', async () => {
+    redisMocks.mGet.mockResolvedValue([
+      '3',
+      JSON.stringify({ version: '3', data: { ...sampleAnalytics, numberOfCourses: 5 } })
+    ]);
+
+    const result = await getOrganisationAnalytics('org-1');
+
+    expect(result.numberOfCourses).toBe(5);
+    expect(getDashOrgStats).not.toHaveBeenCalled();
+    expect(redisMocks.setEx).not.toHaveBeenCalled();
+  });
+
+  it('treats a version mismatch as a cache miss', async () => {
+    redisMocks.mGet.mockResolvedValue([
+      '4',
+      JSON.stringify({ version: '3', data: { ...sampleAnalytics, numberOfCourses: 5 } })
+    ]);
+
+    const result = await getOrganisationAnalytics('org-1');
+
+    expect(result.numberOfCourses).toBe(8);
+    expect(getDashOrgStats).toHaveBeenCalledTimes(1);
+  });
+
+  it('skips cache read when bustCache is true but still writes', async () => {
+    redisMocks.mGet.mockResolvedValue([
+      '1',
+      JSON.stringify({ version: '1', data: { ...sampleAnalytics, numberOfCourses: 5 } })
+    ]);
+
+    const result = await getOrganisationAnalytics('org-1', undefined, true);
+
+    expect(result.numberOfCourses).toBe(8);
+    expect(getDashOrgStats).toHaveBeenCalledTimes(1);
+    expect(redisMocks.setEx).toHaveBeenCalled();
+  });
+
+  it('falls back to the database when redis read fails', async () => {
+    redisMocks.mGet.mockRejectedValue(new Error('redis down'));
+
+    const result = await getOrganisationAnalytics('org-1');
+
+    expect(result.numberOfCourses).toBe(8);
+    expect(getDashOrgStats).toHaveBeenCalledTimes(1);
+  });
+
+  it('returns live data when redis write fails', async () => {
+    redisMocks.setEx.mockRejectedValue(new Error('redis down'));
+
+    const result = await getOrganisationAnalytics('org-1');
+
+    expect(result.numberOfCourses).toBe(8);
+  });
+
+  it('treats malformed cache payload as a miss', async () => {
+    redisMocks.mGet.mockResolvedValue(['1', '{not-json']);
+
+    const result = await getOrganisationAnalytics('org-1');
+
+    expect(result.numberOfCourses).toBe(8);
+    expect(getDashOrgStats).toHaveBeenCalledTimes(1);
+  });
+});

apps/api/src/routes/dash/stats.ts

@@ -41,9 +41,9 @@ export const dashAnalyticsRouter = new Hono()
   })
   .get('/stats', authMiddleware, orgMemberMiddleware, zValidator('query', ZDashStats), async (c) => {
     try {
-      const { orgId, siteName } = c.req.valid('query');
+      const { orgId, siteName, bust } = c.req.valid('query');
 
-      const result = await getOrganisationAnalytics(orgId, siteName);
+      const result = await getOrganisationAnalytics(orgId, siteName, bust);
 
       return c.json({ success: true, data: result }, 200);
     } catch (error) {

apps/api/src/services/course/clone.ts

@@ -23,6 +23,7 @@ import {
 } from '@db/queries';
 
 import { ROLE } from '@cio/utils/constants';
+import { invalidateOrgStats } from '@cio/core/utils/redis/org-stats-cache';
 import { QUESTION_TYPE_IDS } from '@cio/question-types';
 
 async function cloneLessonLanguages(newLessons: TLesson[], oldLessons: TLesson[]): Promise<void> {
@@ -237,6 +238,9 @@ export async function cloneCourse(
     type: course.type
   });
 
+  const statsOrgId = organizationId ?? newGroup.organizationId;
+  await invalidateOrgStats(statsOrgId);
+
   // 4. add group member
   await addGroupMember({
     profileId: userId,

apps/api/src/services/course/completion.ts

@@ -19,6 +19,7 @@ import { claimMemberCertificateEarned } from '@cio/db/queries/course/people';
 import { getActiveOrganizationPlan } from '@cio/db/queries/organization';
 import { PLAN } from '@cio/utils/plans';
 import { env } from '@cio/core/config/env';
+import { invalidateOrgStats } from '@cio/core/utils/redis/org-stats-cache';
 import { trackServerEvent, SERVER_EVENTS } from '@cio/analytics';
 
 export type CertificationBlocker = {
@@ -238,6 +239,8 @@ export async function evaluateCourseCertification(
       if (didClaimCertificate) {
         isNewCompletion = true;
 
+        await invalidateOrgStats(courseRow.orgId);
+
         scheduleCertificationCompletionWork({
           courseId,
           profileId,

apps/api/src/services/course/invite.ts

@@ -27,6 +27,7 @@ import type { TCreateCourseInvite } from '@cio/utils/validation/course/invite';
 import type { TNewCourseInviteAudit } from '@db/types';
 import crypto from 'node:crypto';
 import { getDashboardBaseUrl } from '@cio/core/config/dashboard-url';
+import { invalidateOrgStats } from '@cio/core/utils/redis/org-stats-cache';
 import { getCourseTeachers } from '@cio/db/queries/course/people';
 import { getProfileById } from '@cio/db/queries/auth';
 import { buildEmailFromName, buildEmailBranding, type EmailBranding } from '@cio/email';
@@ -693,6 +694,8 @@ export async function enrollInCourse(
     email: normalizedEmail
   });
 
+  await invalidateOrgStats(org.id);
+
   await ensureComplianceEnrollmentRecordsForProfiles([courseId], [user.id]);
 
   await sendStudentJoinEmails({
@@ -1065,6 +1068,10 @@ export async function acceptStudentInvite(token: string, user: TAuthUser, contex
     };
   });
 
+  if (!result.alreadyJoined) {
+    await invalidateOrgStats(result.organizationId);
+  }
+
   await ensureComplianceEnrollmentRecordsForProfiles([result.courseId], [user.id]);
 
   if (!result.alreadyJoined) {

apps/api/src/services/course/people.ts

@@ -13,7 +13,8 @@ import { resetStudentCourseProgress } from '@cio/db/queries/course/reset-progres
 import type { TAddCourseMembers } from '@cio/utils/validation/course/people';
 import type { TGroupmember } from '@cio/db/types';
 import { getDashboardBaseUrl } from '@cio/core/config/dashboard-url';
-import { getCourseWithOrgData } from '@cio/db/queries/course';
+import { invalidateOrgStats } from '@cio/core/utils/redis/org-stats-cache';
+import { getCourseWithOrgData, getOrgIdByCourseId } from '@cio/db/queries/course';
 import { getProfileById } from '@cio/db/queries/auth';
 import { buildEmailFromName, buildEmailBranding } from '@cio/email';
 import { enqueueTransactionalEmail } from '@api/services/jobs';
@@ -57,6 +58,11 @@ export async function addMember(
 
     const addedMember = await addCourseMember(courseId, data);
 
+    if (data.roleId === ROLE.STUDENT) {
+      const statsOrgId = await getOrgIdByCourseId(courseId);
+      await invalidateOrgStats(statsOrgId);
+    }
+
     if (data.roleId === ROLE.STUDENT && data.profileId) {
       await ensureComplianceEnrollmentRecordsForProfiles([courseId], [data.profileId]);
     }
@@ -201,8 +207,25 @@ export async function addMembers(courseId: string, members: TAddCourseMembers) {
       theme: courseOrgData.orgTheme
     });
 
-    // Add all members
-    const addedMembers = await Promise.all(members.map((member) => addCourseMember(courseId, member)));
+    // Add members sequentially so partial failures still invalidate stats for successful inserts.
+    const addedMembers = [];
+    let addedStudentMember = false;
+
+    try {
+      for (const member of members) {
+        const addedMember = await addCourseMember(courseId, member);
+        addedMembers.push(addedMember);
+
+        if (member.roleId === ROLE.STUDENT) {
+          addedStudentMember = true;
+        }
+      }
+    } finally {
+      if (addedStudentMember) {
+        await invalidateOrgStats(courseOrgData.orgId);
+      }
+    }
+
     const studentProfileIds = members
       .filter((member) => member.roleId === ROLE.STUDENT && member.profileId)
       .map((member) => member.profileId!);
@@ -279,10 +302,26 @@ export async function addMembers(courseId: string, members: TAddCourseMembers) {
  */
 export async function updateMember(courseId: string, memberId: string, data: Partial<TGroupmember>) {
   try {
+    const existingMember = await getCourseMember(courseId, memberId);
+    if (!existingMember) {
+      throw new AppError('Course member not found', ErrorCodes.NOT_FOUND, 404);
+    }
+
     const updated = await updateCourseMember(courseId, memberId, data);
     if (!updated) {
       throw new AppError('Course member not found', ErrorCodes.NOT_FOUND, 404);
     }
+
+    const previousRoleId = existingMember.roleId;
+    const nextRoleId = data.roleId ?? previousRoleId;
+    const studentRoleChanged =
+      data.roleId !== undefined && (previousRoleId === ROLE.STUDENT || nextRoleId === ROLE.STUDENT);
+
+    if (studentRoleChanged) {
+      const statsOrgId = await getOrgIdByCourseId(courseId);
+      await invalidateOrgStats(statsOrgId);
+    }
+
     return updated;
   } catch (error) {
     if (error instanceof AppError) {
@@ -308,6 +347,12 @@ export async function deleteMember(courseId: string, memberId: string) {
     if (!deleted) {
       throw new AppError('Course member not found', ErrorCodes.NOT_FOUND, 404);
     }
+
+    if (deleted.roleId === ROLE.STUDENT) {
+      const statsOrgId = await getOrgIdByCourseId(courseId);
+      await invalidateOrgStats(statsOrgId);
+    }
+
     return deleted;
   } catch (error) {
     if (error instanceof AppError) {
@@ -348,6 +393,9 @@ export async function resetMemberCourseProgress(courseId: string, memberId: stri
       summary
     });
 
+    const statsOrgId = await getOrgIdByCourseId(courseId);
+    await invalidateOrgStats(statsOrgId);
+
     return summary;
   } catch (error) {
     if (error instanceof AppError) {

apps/api/src/services/dash.ts

@@ -10,19 +10,52 @@ import {
 } from '@cio/db/queries/dash';
 import { dashLoginActivityKey } from '@api/utils/redis/key-generators';
 import { logRedisUnavailableOnce, redis } from '@cio/core/utils/redis/redis';
+import {
+  invalidateOrgStats,
+  readOrgStatsVersionAndCache,
+  writeOrgStatsCache
+} from '@cio/core/utils/redis/org-stats-cache';
 
-import { OrganisationAnalytics } from '@api/types';
+import type { OrganisationAnalytics } from '@api/types';
 import { getOrgIdBySiteName } from '@cio/db/queries';
 
-export async function getOrganisationAnalytics(orgId?: string, siteName?: string): Promise<OrganisationAnalytics> {
+async function loadOrganisationAnalyticsFromDatabase(orgId: string): Promise<OrganisationAnalytics> {
+  const [stats, topCourses, recentCertificationRows, certificateCountRows] = await Promise.all([
+    getDashOrgStats(orgId),
+    getCourseStats(orgId),
+    getRecentCertifications(orgId),
+    getTotalCertificatesIssued(orgId)
+  ]);
+
   const analytics: OrganisationAnalytics = {
-    totalCertificates: 0,
-    numberOfCourses: 0,
-    totalStudents: 0,
-    topCourses: [],
-    recentCertifications: []
+    totalCertificates: certificateCountRows[0]?.count ?? 0,
+    numberOfCourses: stats?.[0]?.noOfCourses ?? 0,
+    totalStudents: stats?.[0]?.enrolledStudents ?? 0,
+    topCourses: topCourses.map((course) => ({
+      id: course.courseId,
+      title: course.courseTitle,
+      enrollments: course.totalStudents,
+      completion: course.completionPercentage,
+      certification: course.certificationPercentage
+    })),
+    recentCertifications: recentCertificationRows.map((row) => ({
+      id: row.profileId,
+      avatarUrl: row.avatarUrl,
+      name: row.fullname,
+      courseId: row.courseId,
+      course: row.courseTitle,
+      date: row.earnedAt ?? ''
+    }))
   };
 
+  return analytics;
+}
+
+export async function getOrganisationAnalytics(
+  orgId?: string,
+  siteName?: string,
+  bustCache = false
+): Promise<OrganisationAnalytics> {
   let resolvedOrgId = orgId;
 
   if (!resolvedOrgId && siteName) {
@@ -35,37 +68,26 @@ export async function getOrganisationAnalytics(orgId?: string, siteName?: string
     }
   }
 
+  if (!resolvedOrgId) {
+    throw new AppError('Organization not found', ErrorCodes.ORG_NOT_FOUND, 404);
+  }
+
   try {
-    const [stats, topCourses, recentCertificationRows, certificateCountRows] = await Promise.all([
-      getDashOrgStats(resolvedOrgId!),
-      getCourseStats(resolvedOrgId!),
-      getRecentCertifications(resolvedOrgId!),
-      getTotalCertificatesIssued(resolvedOrgId!)
-    ]);
+    const { version, data: cached } = await readOrgStatsVersionAndCache<OrganisationAnalytics>(resolvedOrgId);
 
-    analytics.totalCertificates = certificateCountRows[0]?.count ?? 0;
-    analytics.numberOfCourses = stats?.[0]?.noOfCourses ?? 0;
-    analytics.totalStudents = stats?.[0]?.enrolledStudents ?? 0;
+    if (!bustCache && cached) {
+      return cached;
+    }
 
-    analytics.topCourses = topCourses.map((c) => ({
-      id: c.courseId,
-      title: c.courseTitle,
-      enrollments: c.totalStudents,
-      completion: c.completionPercentage,
-      certification: c.certificationPercentage
-    }));
-
-    analytics.recentCertifications = recentCertificationRows.map((row) => ({
-      id: row.profileId,
-      avatarUrl: row.avatarUrl,
-      name: row.fullname,
-      courseId: row.courseId,
-      course: row.courseTitle,
-      date: row.earnedAt ?? ''
-    }));
+    const analytics = await loadOrganisationAnalyticsFromDatabase(resolvedOrgId);
+    await writeOrgStatsCache(resolvedOrgId, version, analytics);
 
     return analytics;
   } catch (error) {
+    if (error instanceof AppError) {
+      throw error;
+    }
+
     console.error('Failed to load organisation analytics:', error);
     throw new AppError('Failed to load organisation analytics', ErrorCodes.ORG_ANALYTICS_FETCH_FAILED, 500);
   }
@@ -168,3 +190,5 @@ export async function getCurrentUserLoginStreak(userId: string) {
     throw new AppError('Failed to load login streak', ErrorCodes.ORG_ANALYTICS_FETCH_FAILED, 500);
   }
 }
+
+export { invalidateOrgStats };

apps/api/src/services/organization/audience.ts

@@ -5,6 +5,7 @@ import type {
   TImportAudienceMembers
 } from '@cio/utils/validation/organization';
 import { addGroupMembers, enrollUsersInCourseGroups, getExistingGroupMembers } from '@cio/db/queries/group';
+import { invalidateOrgStats } from '@cio/core/utils/redis/org-stats-cache';
 import { buildEmailFromName, buildEmailBranding } from '@cio/email';
 import { enqueueTransactionalEmail } from '@api/services/jobs';
 import { addCohortMember, getExistingCohortMembers, getCohortsByOrg } from '@cio/db/queries/cohort';
@@ -194,6 +195,8 @@ async function enrollAudienceStudentProfilesInCourses(
         email: profileEmailMap.get(p.profileId) || undefined
       }))
     );
+
+    await invalidateOrgStats(orgId);
   }
 
   if (validProfiles.length > 0) {
@@ -551,6 +554,7 @@ export async function importAudienceMembers(orgId: string, data: TImportAudience
         if (profiles.length > 0) {
           const users = profiles.map((p) => ({ profileId: p.id, email: p.email ?? undefined }));
           await enrollUsersInCourseGroups(validGroupIds, users, ROLE.STUDENT);
+          await invalidateOrgStats(orgId);
           await ensureComplianceEnrollmentRecordsForProfiles(
             courseIds,
             profiles.map((profile) => profile.id)

apps/api/src/services/organization/invite.ts

@@ -22,6 +22,7 @@ import {
 } from '@cio/db/queries/organization';
 import { getCourseGroupIds } from '@cio/db/queries/course';
 import { enrollUsersInCourseGroups } from '@cio/db/queries/group';
+import { invalidateOrgStats } from '@cio/core/utils/redis/org-stats-cache';
 import { addCohortMember, getExistingCohortMembers } from '@cio/db/queries/cohort';
 
 import { ROLE } from '@cio/utils/constants';
@@ -418,11 +419,16 @@ export async function acceptOrganizationInvite(token: string, user: TAuthUser, c
         // Same as importAudienceMembers after course ids are known: resolve groups from course rows only.
         const courseGroupMappings = await getCourseGroupIds(courseIds);
         const validGroupIds = courseGroupMappings.map((m) => m.groupId).filter(Boolean) as string[];
-        await enrollUsersInCourseGroups(
+        const enrolledCount = await enrollUsersInCourseGroups(
           validGroupIds,
           [{ profileId: user.id, email: normalizedEmail }],
           invite.invite.roleId
         );
+
+        if (enrolledCount > 0 && invite.invite.roleId === ROLE.STUDENT) {
+          await invalidateOrgStats(invite.invite.organizationId);
+        }
+
         await ensureComplianceEnrollmentRecordsForProfiles(courseIds, [user.id]);
       } catch (error) {
         console.error('acceptOrganizationInvite course enrollment error:', error);
@@ -710,11 +716,16 @@ export async function acceptOrganizationInviteById(
     try {
       const courseGroupMappings = await getCourseGroupIds(courseIds);
       const validGroupIds = courseGroupMappings.map((m) => m.groupId).filter(Boolean) as string[];
-      await enrollUsersInCourseGroups(
+      const enrolledCount = await enrollUsersInCourseGroups(
         validGroupIds,
         [{ profileId: user.id, email: normalizedEmail }],
         result.invite.roleId
       );
+
+      if (enrolledCount > 0 && result.invite.roleId === ROLE.STUDENT) {
+        await invalidateOrgStats(result.organization.id);
+      }
+
       await ensureComplianceEnrollmentRecordsForProfiles(courseIds, [user.id]);
     } catch (error) {
       console.error('acceptOrganizationInviteById course enrollment error:', error);