feat: let AI chat read lesson video transcripts on demand
Add get_lesson_transcript (teacher) and read_lesson_transcript (student) tools that fetch a lesson's uploaded-video transcript from media_transcript by the video assetIds. The agents call them only when a question is about the video's spoken content, so text-only chats keep a cacheable prefix.
This commit is contained in:
@@ -181,6 +181,7 @@ You have read-only, course-scoped tools:
|
||||
|
||||
- \`list_course_outline\` — see all sections, lessons, and exercises in this course.
|
||||
- \`read_lesson\` — fetch the body of a specific lesson by ID.
|
||||
- \`read_lesson_transcript\` — fetch the transcript of a lesson's uploaded video. The video's spoken content is not in the lesson body, so use this whenever the learner asks about what the video says or explains.
|
||||
- \`read_exercise\` — fetch an exercise prompt (no answer keys).
|
||||
- \`search_course\` — keyword search across this course\u2019s lessons and exercise prompts.
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ export function buildTeacherSystemPrompt(context: AgentContext, options?: BuildT
|
||||
|
||||
## Your Capabilities
|
||||
|
||||
You have access to specific tools listed below. Use them to read course content, create sections and lessons, update existing course-outline section metadata via update_section, update existing lesson metadata via update_lesson, write lesson content, create exercises with questions, create exercise question blocks via create_exercise_section, edit exercise-level metadata (title, description, linked lesson, course section placement, order, due date, lock state, allow-multiple-attempts) via update_exercise, edit each existing in-exercise question block heading and intro via update_exercise_section (ids from get_exercise_details sections array — not course section ids), add or update questions inside an existing exercise (use add_questions with exerciseSectionId from that sections array when the exercise groups questions into blocks), update course landing-page copy/settings, check go-live readiness, and publish the course when it is ready. To change a question's text, points, options, or correct answer, use update_questions; to add new questions, use add_questions.
|
||||
You have access to specific tools listed below. Use them to read course content, read a lesson's uploaded video transcript via get_lesson_transcript (the video's spoken content is not in the lesson body — use this whenever a question is about what the video says or explains), create sections and lessons, update existing course-outline section metadata via update_section, update existing lesson metadata via update_lesson, write lesson content, create exercises with questions, create exercise question blocks via create_exercise_section, edit exercise-level metadata (title, description, linked lesson, course section placement, order, due date, lock state, allow-multiple-attempts) via update_exercise, edit each existing in-exercise question block heading and intro via update_exercise_section (ids from get_exercise_details sections array — not course section ids), add or update questions inside an existing exercise (use add_questions with exerciseSectionId from that sections array when the exercise groups questions into blocks), update course landing-page copy/settings, check go-live readiness, and publish the course when it is ready. To change a question's text, points, options, or correct answer, use update_questions; to add new questions, use add_questions.
|
||||
|
||||
## Question Types
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import { ZCourseLandingPageUpdate, ZCourseLandingPageMetadataUpdate } from '@cio
|
||||
|
||||
export const emptyParam = z.object({});
|
||||
export const lessonReadParam = z.object({ lessonId: z.string(), locale: z.string().default('en') });
|
||||
export const lessonTranscriptParam = z.object({ lessonId: z.string() });
|
||||
export const exerciseReadParam = z.object({ exerciseId: z.string() });
|
||||
export const createSectionParam = z.object({ title: z.string().min(1), order: z.number().int().min(0) });
|
||||
export const updateSectionParam = z
|
||||
|
||||
@@ -27,6 +27,7 @@ import {
|
||||
} from '../exercise/exercise';
|
||||
import { reorderCourseContent } from '../course/content';
|
||||
import { normalizeAgentLessonContent } from './lesson-content';
|
||||
import { getLessonVideoTranscript } from './lesson-transcript';
|
||||
import { buildUpdatedQuestions } from './question-update';
|
||||
import { updateCourseLandingPageService } from '../course/landing-page';
|
||||
import { getCourseGoLiveReadiness, publishCourseWhenReady } from '../course/go-live-readiness';
|
||||
@@ -50,6 +51,7 @@ import {
|
||||
fetchDocumentationUrlParam,
|
||||
goLiveParam,
|
||||
lessonReadParam,
|
||||
lessonTranscriptParam,
|
||||
reorderContentParam,
|
||||
updateContentParam,
|
||||
updateCourseLandingPageParam,
|
||||
@@ -487,6 +489,19 @@ export function buildAgentTools(
|
||||
}
|
||||
}),
|
||||
|
||||
get_lesson_transcript: tool({
|
||||
description:
|
||||
"Get the transcript of a lesson's uploaded video(s). The spoken content of a video is NOT part of the lesson's HTML content, so call this whenever a question is about what the video says, explains, or demonstrates. Only uploaded videos are transcribed — embedded links (YouTube, etc.) return no transcript.",
|
||||
inputSchema: lessonTranscriptParam,
|
||||
execute: async (args) => {
|
||||
return executeAgentTool('get_lesson_transcript', { orgId, userId, courseId, args }, async () => {
|
||||
await verifyLessonBelongsToCourse(args.lessonId, courseId);
|
||||
|
||||
return getLessonVideoTranscript(args.lessonId, orgId);
|
||||
});
|
||||
}
|
||||
}),
|
||||
|
||||
get_exercise_details: tool({
|
||||
description:
|
||||
'Get an exercise with sections, questions, and answer options from this course. Each sections entry has an id — use it with update_exercise_section to change that question block title or description.',
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { listMediaTranscriptsByAssetIds } from '@cio/db/queries/media-transcript';
|
||||
import { getLesson } from '../lesson/lesson';
|
||||
|
||||
export interface LessonVideoTranscriptResult {
|
||||
lessonId: string;
|
||||
title: string;
|
||||
hasTranscript: boolean;
|
||||
transcript: string | null;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Assembles the transcript text for a lesson's uploaded video(s).
|
||||
*
|
||||
* Lesson `videos[]` carry an `assetId` for uploaded videos; the transcription
|
||||
* pipeline stores Whisper output in `media_transcript` keyed by that assetId.
|
||||
* YouTube/embedded links have no assetId and are never transcribed.
|
||||
*/
|
||||
export async function getLessonVideoTranscript(lessonId: string, orgId: string): Promise<LessonVideoTranscriptResult> {
|
||||
const lesson = await getLesson(lessonId);
|
||||
const lessonWithVideos = lesson as {
|
||||
id: string;
|
||||
title: string;
|
||||
videos?: Array<{ assetId?: string }> | null;
|
||||
};
|
||||
|
||||
const assetIds = (lessonWithVideos.videos ?? [])
|
||||
.map((video) => video.assetId)
|
||||
.filter((assetId): assetId is string => typeof assetId === 'string' && assetId.length > 0);
|
||||
|
||||
if (assetIds.length === 0) {
|
||||
return {
|
||||
lessonId: lesson.id,
|
||||
title: lesson.title,
|
||||
hasTranscript: false,
|
||||
transcript: null,
|
||||
message:
|
||||
'This lesson has no uploaded video with a transcript. Only uploaded videos are transcribed — embedded links (YouTube, Google Drive, etc.) are not.'
|
||||
};
|
||||
}
|
||||
|
||||
const transcripts = await listMediaTranscriptsByAssetIds(assetIds, orgId);
|
||||
const transcript = transcripts
|
||||
.map((row) => row.text?.trim())
|
||||
.filter((text): text is string => Boolean(text))
|
||||
.join('\n\n');
|
||||
|
||||
if (!transcript) {
|
||||
return {
|
||||
lessonId: lesson.id,
|
||||
title: lesson.title,
|
||||
hasTranscript: false,
|
||||
transcript: null,
|
||||
message:
|
||||
'This lesson has an uploaded video, but no transcript is available yet — transcription may still be processing.'
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
lessonId: lesson.id,
|
||||
title: lesson.title,
|
||||
hasTranscript: true,
|
||||
transcript
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user