feat: certificates, landing pages, ai tutor, website update, compliance

This commit is contained in:
Rotimi Best
2026-05-18 03:47:22 +01:00
parent c31ad63955
commit ac3a2c14a7
644 changed files with 56050 additions and 5439 deletions
+21
View File
@@ -0,0 +1,21 @@
'use strict';
Object.defineProperty(exports, '__esModule', { value: true });
require('dotenv/config');
var env_1 = require('./config/env');
var ffmpeg_1 = require('./utils/ffmpeg');
var logger_1 = require('./utils/logger');
void (0, ffmpeg_1.warnIfFfmpegMissing)();
if (!env_1.env.REDIS_URL) {
throw new Error('REDIS_URL is required to start a worker process.');
}
logger_1.log.info('worker-bootstrap', {
nodeEnv: env_1.env.NODE_ENV,
pid: process.pid,
hasOpenAi: Boolean(env_1.env.OPENAI_API_KEY)
});
process.on('uncaughtException', function (error) {
logger_1.log.error('uncaught-exception', { message: error.message, stack: error.stack });
});
process.on('unhandledRejection', function (reason) {
logger_1.log.error('unhandled-rejection', { reason: reason instanceof Error ? reason.message : String(reason) });
});
+25
View File
@@ -0,0 +1,25 @@
import 'dotenv/config';
import { env } from './config/env';
import { warnIfFfmpegMissing } from './utils/ffmpeg';
import { log } from './utils/logger';
void warnIfFfmpegMissing();
if (!env.REDIS_URL) {
throw new Error('REDIS_URL is required to start a worker process.');
}
log.info('worker-bootstrap', {
nodeEnv: env.NODE_ENV,
pid: process.pid,
hasOpenAi: Boolean(env.OPENAI_API_KEY)
});
process.on('uncaughtException', (error) => {
log.error('uncaught-exception', { message: error.message, stack: error.stack });
});
process.on('unhandledRejection', (reason) => {
log.error('unhandled-rejection', { reason: reason instanceof Error ? reason.message : String(reason) });
});
View File
+41
View File
@@ -0,0 +1,41 @@
import * as z from 'zod';
const envSchema = z.object({
NODE_ENV: z.enum(['development', 'production', 'test']).default('development'),
REDIS_URL: z.string().min(1, 'REDIS_URL is required for the BullMQ worker'),
DATABASE_URL: z.string().optional(),
PRIVATE_DATABASE_URL: z.string().optional(),
/** Object storage (mirrors apps/api/src/config/env). Workers stream object bytes. */
OBJECT_STORAGE_ENDPOINT: z.string().optional(),
OBJECT_STORAGE_REGION: z.string().optional(),
OBJECT_STORAGE_ACCESS_KEY_ID: z.string().optional(),
OBJECT_STORAGE_SECRET_ACCESS_KEY: z.string().optional(),
OBJECT_STORAGE_FORCE_PATH_STYLE: z.string().optional(),
OBJECT_STORAGE_BUCKET_VIDEOS: z.string().optional(),
OBJECT_STORAGE_BUCKET_DOCUMENTS: z.string().optional(),
OBJECT_STORAGE_BUCKET_MEDIA: z.string().optional(),
OBJECT_STORAGE_MEDIA_PUBLIC_BASE_URL: z.string().optional(),
OBJECT_STORAGE_PUBLIC_ENDPOINT: z.string().optional(),
/** Cloudflare R2 fallback. */
CLOUDFLARE_ACCOUNT_ID: z.string().optional(),
CLOUDFLARE_ACCESS_KEY: z.string().optional(),
CLOUDFLARE_SECRET_ACCESS_KEY: z.string().optional(),
CLOUDFLARE_IMAGE_BUCKET_DOMAIN: z.string().optional(),
/** Concurrency tuning per worker process. */
MEDIA_WORKER_CONCURRENCY: z.string().optional(),
TRANSCRIBE_WORKER_CONCURRENCY: z.string().optional(),
EMAIL_WORKER_CONCURRENCY: z.string().optional(),
/** Optional OpenAI Whisper key — when unset, transcribe-audio jobs no-op. */
OPENAI_API_KEY: z.string().optional()
});
export const env = envSchema.parse(process.env);
/** Worker DB connection string (DATABASE_URL preferred, falls back to PRIVATE_DATABASE_URL). */
export function resolveDatabaseUrl(): string {
return env.DATABASE_URL ?? env.PRIVATE_DATABASE_URL ?? '';
}
+78
View File
@@ -0,0 +1,78 @@
'use strict';
Object.defineProperty(exports, '__esModule', { value: true });
exports.getStorageConfig = getStorageConfig;
exports.getS3Client = getS3Client;
var client_s3_1 = require('@aws-sdk/client-s3');
var env_1 = require('./env');
/**
* Worker-side S3 client. Mirrors apps/api/src/config/storage.ts so workers
* can stream the same buckets the API uploads to. Kept self-contained so
* apps/jobs has no runtime dependency on apps/api.
*/
var BUCKET_DEFAULTS = {
VIDEOS: 'videos',
DOCUMENTS: 'documents',
MEDIA: 'media'
};
function useS3CompatibleConfig() {
return Boolean(
env_1.env.OBJECT_STORAGE_ENDPOINT &&
env_1.env.OBJECT_STORAGE_ACCESS_KEY_ID &&
env_1.env.OBJECT_STORAGE_SECRET_ACCESS_KEY
);
}
function isCloudflareConfigured() {
return Boolean(
env_1.env.CLOUDFLARE_ACCOUNT_ID && env_1.env.CLOUDFLARE_ACCESS_KEY && env_1.env.CLOUDFLARE_SECRET_ACCESS_KEY
);
}
var cachedConfig = null;
function getStorageConfig() {
if (cachedConfig) return cachedConfig;
if (useS3CompatibleConfig()) {
cachedConfig = {
endpoint: env_1.env.OBJECT_STORAGE_ENDPOINT,
region: env_1.env.OBJECT_STORAGE_REGION || 'us-east-1',
accessKeyId: env_1.env.OBJECT_STORAGE_ACCESS_KEY_ID,
secretAccessKey: env_1.env.OBJECT_STORAGE_SECRET_ACCESS_KEY,
forcePathStyle:
env_1.env.OBJECT_STORAGE_FORCE_PATH_STYLE === 'true' || env_1.env.OBJECT_STORAGE_FORCE_PATH_STYLE === '1',
bucketVideos: env_1.env.OBJECT_STORAGE_BUCKET_VIDEOS || BUCKET_DEFAULTS.VIDEOS,
bucketDocuments: env_1.env.OBJECT_STORAGE_BUCKET_DOCUMENTS || BUCKET_DEFAULTS.DOCUMENTS,
bucketMedia: env_1.env.OBJECT_STORAGE_BUCKET_MEDIA || BUCKET_DEFAULTS.MEDIA,
mediaPublicBaseUrl: env_1.env.OBJECT_STORAGE_MEDIA_PUBLIC_BASE_URL || null
};
} else if (isCloudflareConfigured()) {
cachedConfig = {
endpoint: 'https://'.concat(env_1.env.CLOUDFLARE_ACCOUNT_ID, '.r2.cloudflarestorage.com'),
region: 'auto',
accessKeyId: env_1.env.CLOUDFLARE_ACCESS_KEY,
secretAccessKey: env_1.env.CLOUDFLARE_SECRET_ACCESS_KEY,
forcePathStyle: false,
bucketVideos: BUCKET_DEFAULTS.VIDEOS,
bucketDocuments: BUCKET_DEFAULTS.DOCUMENTS,
bucketMedia: BUCKET_DEFAULTS.MEDIA,
mediaPublicBaseUrl: env_1.env.CLOUDFLARE_IMAGE_BUCKET_DOMAIN || null
};
} else {
throw new Error(
'Object storage not configured. Set OBJECT_STORAGE_* (MinIO/S3) or CLOUDFLARE_* (R2) env vars on apps/jobs.'
);
}
return cachedConfig;
}
var s3Client = null;
function getS3Client() {
if (s3Client) return s3Client;
var config = getStorageConfig();
s3Client = new client_s3_1.S3Client({
region: config.region,
endpoint: config.endpoint,
credentials: {
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey
},
forcePathStyle: config.forcePathStyle
});
return s3Client;
}
+94
View File
@@ -0,0 +1,94 @@
import { S3Client } from '@aws-sdk/client-s3';
import { env } from './env';
/**
* Worker-side S3 client. Mirrors apps/api/src/config/storage.ts so workers
* can stream the same buckets the API uploads to. Kept self-contained so
* apps/jobs has no runtime dependency on apps/api.
*/
const BUCKET_DEFAULTS = {
VIDEOS: 'videos',
DOCUMENTS: 'documents',
MEDIA: 'media'
} as const;
export interface StorageConfig {
endpoint: string;
region: string;
accessKeyId: string;
secretAccessKey: string;
forcePathStyle: boolean;
bucketVideos: string;
bucketDocuments: string;
bucketMedia: string;
mediaPublicBaseUrl: string | null;
}
function useS3CompatibleConfig(): boolean {
return Boolean(
env.OBJECT_STORAGE_ENDPOINT && env.OBJECT_STORAGE_ACCESS_KEY_ID && env.OBJECT_STORAGE_SECRET_ACCESS_KEY
);
}
function isCloudflareConfigured(): boolean {
return Boolean(env.CLOUDFLARE_ACCOUNT_ID && env.CLOUDFLARE_ACCESS_KEY && env.CLOUDFLARE_SECRET_ACCESS_KEY);
}
let cachedConfig: StorageConfig | null = null;
export function getStorageConfig(): StorageConfig {
if (cachedConfig) return cachedConfig;
if (useS3CompatibleConfig()) {
cachedConfig = {
endpoint: env.OBJECT_STORAGE_ENDPOINT!,
region: env.OBJECT_STORAGE_REGION || 'us-east-1',
accessKeyId: env.OBJECT_STORAGE_ACCESS_KEY_ID!,
secretAccessKey: env.OBJECT_STORAGE_SECRET_ACCESS_KEY!,
forcePathStyle: env.OBJECT_STORAGE_FORCE_PATH_STYLE === 'true' || env.OBJECT_STORAGE_FORCE_PATH_STYLE === '1',
bucketVideos: env.OBJECT_STORAGE_BUCKET_VIDEOS || BUCKET_DEFAULTS.VIDEOS,
bucketDocuments: env.OBJECT_STORAGE_BUCKET_DOCUMENTS || BUCKET_DEFAULTS.DOCUMENTS,
bucketMedia: env.OBJECT_STORAGE_BUCKET_MEDIA || BUCKET_DEFAULTS.MEDIA,
mediaPublicBaseUrl: env.OBJECT_STORAGE_MEDIA_PUBLIC_BASE_URL || null
};
} else if (isCloudflareConfigured()) {
cachedConfig = {
endpoint: `https://${env.CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com`,
region: 'auto',
accessKeyId: env.CLOUDFLARE_ACCESS_KEY!,
secretAccessKey: env.CLOUDFLARE_SECRET_ACCESS_KEY!,
forcePathStyle: false,
bucketVideos: BUCKET_DEFAULTS.VIDEOS,
bucketDocuments: BUCKET_DEFAULTS.DOCUMENTS,
bucketMedia: BUCKET_DEFAULTS.MEDIA,
mediaPublicBaseUrl: env.CLOUDFLARE_IMAGE_BUCKET_DOMAIN || null
};
} else {
throw new Error(
'Object storage not configured. Set OBJECT_STORAGE_* (MinIO/S3) or CLOUDFLARE_* (R2) env vars on apps/jobs.'
);
}
return cachedConfig;
}
let s3Client: S3Client | null = null;
export function getS3Client(): S3Client {
if (s3Client) return s3Client;
const config = getStorageConfig();
s3Client = new S3Client({
region: config.region,
endpoint: config.endpoint,
credentials: {
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey
},
forcePathStyle: config.forcePathStyle
});
return s3Client;
}
+11
View File
@@ -0,0 +1,11 @@
'use strict';
Object.defineProperty(exports, '__esModule', { value: true });
require('./bootstrap');
require('./workers/media');
require('./workers/media-transcribe');
require('./workers/emails');
require('./workers/maintenance');
var logger_1 = require('./utils/logger');
logger_1.log.info('all-workers-running', {
workers: ['media', 'media-transcribe', 'emails', 'maintenance']
});
+12
View File
@@ -0,0 +1,12 @@
import './bootstrap';
import './workers/media';
import './workers/media-transcribe';
import './workers/emails';
import './workers/maintenance';
import { log } from './utils/logger';
log.info('all-workers-running', {
workers: ['media', 'media-transcribe', 'emails', 'maintenance']
});
+29
View File
@@ -0,0 +1,29 @@
'use strict';
var __createBinding =
(this && this.__createBinding) ||
(Object.create
? function (o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ('get' in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = {
enumerable: true,
get: function () {
return m[k];
}
};
}
Object.defineProperty(o, k2, desc);
}
: function (o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
});
var __exportStar =
(this && this.__exportStar) ||
function (m, exports) {
for (var p in m)
if (p !== 'default' && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
};
Object.defineProperty(exports, '__esModule', { value: true });
__exportStar(require('./send'), exports);
+1
View File
@@ -0,0 +1 @@
export * from './send';
+217
View File
@@ -0,0 +1,217 @@
'use strict';
var __awaiter =
(this && this.__awaiter) ||
function (thisArg, _arguments, P, generator) {
function adopt(value) {
return value instanceof P
? value
: new P(function (resolve) {
resolve(value);
});
}
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) {
try {
step(generator.next(value));
} catch (e) {
reject(e);
}
}
function rejected(value) {
try {
step(generator['throw'](value));
} catch (e) {
reject(e);
}
}
function step(result) {
result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected);
}
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator =
(this && this.__generator) ||
function (thisArg, body) {
var _ = {
label: 0,
sent: function () {
if (t[0] & 1) throw t[1];
return t[1];
},
trys: [],
ops: []
},
f,
y,
t,
g = Object.create((typeof Iterator === 'function' ? Iterator : Object).prototype);
return (
(g.next = verb(0)),
(g['throw'] = verb(1)),
(g['return'] = verb(2)),
typeof Symbol === 'function' &&
(g[Symbol.iterator] = function () {
return this;
}),
g
);
function verb(n) {
return function (v) {
return step([n, v]);
};
}
function step(op) {
if (f) throw new TypeError('Generator is already executing.');
while ((g && ((g = 0), op[0] && (_ = 0)), _))
try {
if (
((f = 1),
y &&
(t = op[0] & 2 ? y['return'] : op[0] ? y['throw'] || ((t = y['return']) && t.call(y), 0) : y.next) &&
!(t = t.call(y, op[1])).done)
)
return t;
if (((y = 0), t)) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0:
case 1:
t = op;
break;
case 4:
_.label++;
return { value: op[1], done: false };
case 5:
_.label++;
y = op[1];
op = [0];
continue;
case 7:
op = _.ops.pop();
_.trys.pop();
continue;
default:
if (!((t = _.trys), (t = t.length > 0 && t[t.length - 1])) && (op[0] === 6 || op[0] === 2)) {
_ = 0;
continue;
}
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) {
_.label = op[1];
break;
}
if (op[0] === 6 && _.label < t[1]) {
_.label = t[1];
t = op;
break;
}
if (t && _.label < t[2]) {
_.label = t[2];
_.ops.push(op);
break;
}
if (t[2]) _.ops.pop();
_.trys.pop();
continue;
}
op = body.call(thisArg, _);
} catch (e) {
op = [6, e];
y = 0;
} finally {
f = t = 0;
}
if (op[0] & 5) throw op[1];
return { value: op[0] ? op[1] : void 0, done: true };
}
};
Object.defineProperty(exports, '__esModule', { value: true });
exports.processSendEmail = processSendEmail;
var email_1 = require('@cio/email');
var jobs_1 = require('@cio/jobs');
var logger_1 = require('../../utils/logger');
/**
* Process an `emails:send` job. Dispatches based on payload `kind`, calls the
* provider via `@cio/email`, and returns a provider message id (BullMQ stores
* this as the job return value).
*/
function processSendEmail(rawPayload) {
return __awaiter(this, void 0, void 0, function () {
var payload, responses_1, result_1, responses, result;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
payload = jobs_1.ZSendEmailPayload.parse(rawPayload);
if (!(payload.kind === 'template')) return [3 /*break*/, 2];
return [
4 /*yield*/,
(0, email_1.sendEmail)(payload.template, {
to: payload.to,
// Re-validated by `sendEmail` against the registered template schema; the
// generic record was already validated in the API helper before enqueue.
fields: payload.fields,
from: payload.from,
replyTo: payload.replyTo
})
];
case 1:
responses_1 = _a.sent();
result_1 = extractProviderId(responses_1);
logger_1.log.info('email-sent', {
kind: 'template',
template: payload.template,
recipient: payload.to,
providerId: result_1.providerId
});
return [2 /*return*/, result_1];
case 2:
return [
4 /*yield*/,
(0, email_1.deliverEmail)([
{
to: payload.to,
subject: payload.subject,
content: payload.content,
from: payload.from,
replyTo: payload.replyTo
}
])
];
case 3:
responses = _a.sent();
result = extractProviderId(responses);
logger_1.log.info('email-sent', { kind: 'raw', recipient: payload.to, providerId: result.providerId });
return [2 /*return*/, result];
}
});
});
}
function extractProviderId(responses) {
var _a, _b, _c, _d;
var failed = responses.find(function (response) {
return !response.success;
});
if (failed) {
throw new Error(
(_a = failed.error) !== null && _a !== void 0 ? _a : 'email provider returned an unsuccessful response'
);
}
for (var _i = 0, responses_2 = responses; _i < responses_2.length; _i++) {
var response = responses_2[_i];
var details = response.details;
var candidate =
(_c =
(_b = details === null || details === void 0 ? void 0 : details.messageId) !== null && _b !== void 0
? _b
: details === null || details === void 0
? void 0
: details.request_id) !== null && _c !== void 0
? _c
: (_d = details === null || details === void 0 ? void 0 : details.data) === null || _d === void 0
? void 0
: _d.request_id;
if (candidate) {
return { providerId: String(candidate) };
}
}
return { providerId: '' };
}
+70
View File
@@ -0,0 +1,70 @@
import { deliverEmail, sendEmail, type EmailId } from '@cio/email';
import { ZSendEmailPayload } from '@cio/jobs';
import { log } from '../../utils/logger';
interface SendResult {
providerId: string;
}
/**
* Process an `emails:send` job. Dispatches based on payload `kind`, calls the
* provider via `@cio/email`, and returns a provider message id (BullMQ stores
* this as the job return value).
*/
export async function processSendEmail(rawPayload: unknown): Promise<SendResult> {
const payload = ZSendEmailPayload.parse(rawPayload);
if (payload.kind === 'template') {
const responses = await sendEmail(payload.template as EmailId, {
to: payload.to,
// Re-validated by `sendEmail` against the registered template schema; the
// generic record was already validated in the API helper before enqueue.
fields: payload.fields as never,
from: payload.from,
replyTo: payload.replyTo
});
const result = extractProviderId(responses);
log.info('email-sent', {
kind: 'template',
template: payload.template,
recipient: payload.to,
providerId: result.providerId
});
return result;
}
const responses = await deliverEmail([
{
to: payload.to,
subject: payload.subject,
content: payload.content,
from: payload.from,
replyTo: payload.replyTo
}
]);
const result = extractProviderId(responses);
log.info('email-sent', { kind: 'raw', recipient: payload.to, providerId: result.providerId });
return result;
}
function extractProviderId(
responses: ReadonlyArray<{ success: boolean; details?: unknown; error?: string }>
): SendResult {
const failed = responses.find((response) => !response.success);
if (failed) {
throw new Error(failed.error ?? 'email provider returned an unsuccessful response');
}
for (const response of responses) {
const details = response.details as
| { messageId?: string; request_id?: string; data?: { request_id?: string } }
| undefined;
const candidate = details?.messageId ?? details?.request_id ?? details?.data?.request_id;
if (candidate) {
return { providerId: String(candidate) };
}
}
return { providerId: '' };
}
@@ -0,0 +1,309 @@
'use strict';
var __awaiter =
(this && this.__awaiter) ||
function (thisArg, _arguments, P, generator) {
function adopt(value) {
return value instanceof P
? value
: new P(function (resolve) {
resolve(value);
});
}
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) {
try {
step(generator.next(value));
} catch (e) {
reject(e);
}
}
function rejected(value) {
try {
step(generator['throw'](value));
} catch (e) {
reject(e);
}
}
function step(result) {
result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected);
}
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator =
(this && this.__generator) ||
function (thisArg, body) {
var _ = {
label: 0,
sent: function () {
if (t[0] & 1) throw t[1];
return t[1];
},
trys: [],
ops: []
},
f,
y,
t,
g = Object.create((typeof Iterator === 'function' ? Iterator : Object).prototype);
return (
(g.next = verb(0)),
(g['throw'] = verb(1)),
(g['return'] = verb(2)),
typeof Symbol === 'function' &&
(g[Symbol.iterator] = function () {
return this;
}),
g
);
function verb(n) {
return function (v) {
return step([n, v]);
};
}
function step(op) {
if (f) throw new TypeError('Generator is already executing.');
while ((g && ((g = 0), op[0] && (_ = 0)), _))
try {
if (
((f = 1),
y &&
(t = op[0] & 2 ? y['return'] : op[0] ? y['throw'] || ((t = y['return']) && t.call(y), 0) : y.next) &&
!(t = t.call(y, op[1])).done)
)
return t;
if (((y = 0), t)) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0:
case 1:
t = op;
break;
case 4:
_.label++;
return { value: op[1], done: false };
case 5:
_.label++;
y = op[1];
op = [0];
continue;
case 7:
op = _.ops.pop();
_.trys.pop();
continue;
default:
if (!((t = _.trys), (t = t.length > 0 && t[t.length - 1])) && (op[0] === 6 || op[0] === 2)) {
_ = 0;
continue;
}
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) {
_.label = op[1];
break;
}
if (op[0] === 6 && _.label < t[1]) {
_.label = t[1];
t = op;
break;
}
if (t && _.label < t[2]) {
_.label = t[2];
_.ops.push(op);
break;
}
if (t[2]) _.ops.pop();
_.trys.pop();
continue;
}
op = body.call(thisArg, _);
} catch (e) {
op = [6, e];
y = 0;
} finally {
f = t = 0;
}
if (op[0] & 5) throw op[1];
return { value: op[0] ? op[1] : void 0, done: true };
}
};
Object.defineProperty(exports, '__esModule', { value: true });
exports.processExtractAudio = processExtractAudio;
var promises_1 = require('node:fs/promises');
var node_path_1 = require('node:path');
var node_os_1 = require('node:os');
var queries_1 = require('@cio/db/queries');
var media_1 = require('@cio/jobs/payloads/media');
var storage_1 = require('../../utils/storage');
var cancel_1 = require('../../utils/cancel');
var ffmpeg_1 = require('../../utils/ffmpeg');
var logger_1 = require('../../utils/logger');
var STEP_KEY = 'extract-audio';
var DOMAIN = 'media';
function transcodeToMp3(inputPath, outputPath) {
return (0, ffmpeg_1.ffmpegRun)([
'-y',
'-i',
inputPath,
'-vn',
'-acodec',
'libmp3lame',
'-b:a',
'32k',
'-ac',
'1',
'-ar',
'16000',
'-f',
'mp3',
outputPath
]);
}
/**
* Extract a 16 kHz mono MP3 (32 kbps) from the source video for Whisper (25 MB
* API limit). The transcribe step reads the object from the media bucket.
*/
function processExtractAudio(rawData) {
return __awaiter(this, void 0, void 0, function () {
var payload,
mediaJobId,
assetId,
storageKey,
existing,
inputPath,
outputPath,
dir,
audioKey,
fileStat,
result,
error_1;
var _a;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
payload = media_1.ZExtractAudioPayload.parse(rawData);
((mediaJobId = payload.mediaJobId), (assetId = payload.assetId), (storageKey = payload.storageKey));
return [4 /*yield*/, (0, queries_1.getJobStep)(DOMAIN, mediaJobId, STEP_KEY)];
case 1:
existing = _b.sent();
if (
(existing === null || existing === void 0 ? void 0 : existing.status) === 'completed' &&
existing.result
) {
logger_1.log.info('extract-audio-skip', { mediaJobId: mediaJobId, reason: 'ledger-completed' });
return [2 /*return*/, existing.result];
}
return [4 /*yield*/, (0, cancel_1.throwIfCancelRequested)(mediaJobId)];
case 2:
_b.sent();
return [
4 /*yield*/,
(0, queries_1.updateMediaJob)(mediaJobId, { stage: 'extracting-audio', progressPercent: 70 })
];
case 3:
_b.sent();
return [
4 /*yield*/,
(0, queries_1.upsertJobStep)({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'running',
startedAt: new Date().toISOString(),
attempt:
((_a = existing === null || existing === void 0 ? void 0 : existing.attempt) !== null && _a !== void 0
? _a
: 0) + 1
})
];
case 4:
_b.sent();
_b.label = 5;
case 5:
_b.trys.push([5, 14, 16, 19]);
return [
4 /*yield*/,
(0, storage_1.downloadObjectToTempFile)((0, storage_1.videosBucket)(), storageKey, 'audio-source.bin')
];
case 6:
inputPath = _b.sent();
dir = node_path_1.default.join((0, node_os_1.tmpdir)(), 'cio-jobs', 'audio');
return [4 /*yield*/, (0, promises_1.mkdir)(dir, { recursive: true })];
case 7:
_b.sent();
outputPath = node_path_1.default.join(dir, ''.concat(mediaJobId, '.mp3'));
return [4 /*yield*/, (0, cancel_1.throwIfCancelRequested)(mediaJobId)];
case 8:
_b.sent();
return [4 /*yield*/, transcodeToMp3(inputPath, outputPath)];
case 9:
_b.sent();
audioKey = 'audio/'.concat(assetId, '/').concat(mediaJobId, '.mp3');
return [
4 /*yield*/,
(0, storage_1.uploadFileToBucket)(
(0, storage_1.mediaBucket)(),
audioKey,
outputPath,
'audio/mpeg',
'private, max-age=0'
)
];
case 10:
_b.sent();
return [4 /*yield*/, (0, promises_1.stat)(outputPath)];
case 11:
fileStat = _b.sent();
result = { bucket: (0, storage_1.mediaBucket)(), key: audioKey, fileSizeBytes: fileStat.size };
return [
4 /*yield*/,
(0, queries_1.upsertJobStep)({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'completed',
finishedAt: new Date().toISOString(),
result: result
})
];
case 12:
_b.sent();
return [
4 /*yield*/,
(0, queries_1.updateMediaJob)(mediaJobId, { stage: 'audio-ready', progressPercent: 80 })
];
case 13:
_b.sent();
logger_1.log.info('extract-audio-done', { mediaJobId: mediaJobId, audioKey: audioKey });
return [2 /*return*/, result];
case 14:
error_1 = _b.sent();
logger_1.log.error('extract-audio-failed', {
mediaJobId: mediaJobId,
error: (0, cancel_1.errorMessage)(error_1)
});
return [
4 /*yield*/,
(0, queries_1.upsertJobStep)({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'failed',
finishedAt: new Date().toISOString(),
error: { code: 'EXTRACT_AUDIO_FAILED', message: (0, cancel_1.errorMessage)(error_1) }
})
];
case 15:
_b.sent();
throw error_1;
case 16:
return [4 /*yield*/, (0, storage_1.safeUnlink)(inputPath)];
case 17:
_b.sent();
return [4 /*yield*/, (0, storage_1.safeUnlink)(outputPath)];
case 18:
_b.sent();
return [7 /*endfinally*/];
case 19:
return [2 /*return*/];
}
});
});
}
@@ -0,0 +1,118 @@
import { mkdir, stat } from 'node:fs/promises';
import path from 'node:path';
import { tmpdir } from 'node:os';
import { getJobStep, updateMediaJob, upsertJobStep } from '@cio/db/queries';
import { ZExtractAudioPayload, type TExtractAudioPayload } from '@cio/jobs/payloads/media';
import {
downloadObjectToTempFile,
mediaBucket,
safeUnlink,
uploadFileToBucket,
videosBucket
} from '../../utils/storage';
import { errorMessage, throwIfCancelRequested } from '../../utils/cancel';
import { ffmpegRun } from '../../utils/ffmpeg';
import { log } from '../../utils/logger';
const STEP_KEY = 'extract-audio';
const DOMAIN = 'media';
interface ExtractAudioResult {
bucket: string;
key: string;
fileSizeBytes: number;
}
function transcodeToMp3(inputPath: string, outputPath: string): Promise<void> {
return ffmpegRun([
'-y',
'-i',
inputPath,
'-vn',
'-acodec',
'libmp3lame',
'-b:a',
'32k',
'-ac',
'1',
'-ar',
'16000',
'-f',
'mp3',
outputPath
]);
}
/**
* Extract a 16 kHz mono MP3 (32 kbps) from the source video for Whisper (25 MB
* API limit). The transcribe step reads the object from the media bucket.
*/
export async function processExtractAudio(rawData: unknown): Promise<ExtractAudioResult> {
const payload: TExtractAudioPayload = ZExtractAudioPayload.parse(rawData);
const { mediaJobId, assetId, storageKey } = payload;
const existing = await getJobStep(DOMAIN, mediaJobId, STEP_KEY);
if (existing?.status === 'completed' && existing.result) {
log.info('extract-audio-skip', { mediaJobId, reason: 'ledger-completed' });
return existing.result as unknown as ExtractAudioResult;
}
await throwIfCancelRequested(mediaJobId);
await updateMediaJob(mediaJobId, { stage: 'extracting-audio', progressPercent: 70 });
await upsertJobStep({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'running',
startedAt: new Date().toISOString(),
attempt: (existing?.attempt ?? 0) + 1
});
let inputPath: string | undefined;
let outputPath: string | undefined;
try {
inputPath = await downloadObjectToTempFile(videosBucket(), storageKey, 'audio-source.bin');
const dir = path.join(tmpdir(), 'cio-jobs', 'audio');
await mkdir(dir, { recursive: true });
outputPath = path.join(dir, `${mediaJobId}.mp3`);
await throwIfCancelRequested(mediaJobId);
await transcodeToMp3(inputPath, outputPath);
const audioKey = `audio/${assetId}/${mediaJobId}.mp3`;
await uploadFileToBucket(mediaBucket(), audioKey, outputPath, 'audio/mpeg', 'private, max-age=0');
const fileStat = await stat(outputPath);
const result: ExtractAudioResult = { bucket: mediaBucket(), key: audioKey, fileSizeBytes: fileStat.size };
await upsertJobStep({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'completed',
finishedAt: new Date().toISOString(),
result: result as unknown as Record<string, unknown>
});
await updateMediaJob(mediaJobId, { stage: 'audio-ready', progressPercent: 80 });
log.info('extract-audio-done', { mediaJobId, audioKey });
return result;
} catch (error) {
log.error('extract-audio-failed', { mediaJobId, error: errorMessage(error) });
await upsertJobStep({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'failed',
finishedAt: new Date().toISOString(),
error: { code: 'EXTRACT_AUDIO_FAILED', message: errorMessage(error) }
});
throw error;
} finally {
await safeUnlink(inputPath);
await safeUnlink(outputPath);
}
}
@@ -0,0 +1,338 @@
'use strict';
var __awaiter =
(this && this.__awaiter) ||
function (thisArg, _arguments, P, generator) {
function adopt(value) {
return value instanceof P
? value
: new P(function (resolve) {
resolve(value);
});
}
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) {
try {
step(generator.next(value));
} catch (e) {
reject(e);
}
}
function rejected(value) {
try {
step(generator['throw'](value));
} catch (e) {
reject(e);
}
}
function step(result) {
result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected);
}
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator =
(this && this.__generator) ||
function (thisArg, body) {
var _ = {
label: 0,
sent: function () {
if (t[0] & 1) throw t[1];
return t[1];
},
trys: [],
ops: []
},
f,
y,
t,
g = Object.create((typeof Iterator === 'function' ? Iterator : Object).prototype);
return (
(g.next = verb(0)),
(g['throw'] = verb(1)),
(g['return'] = verb(2)),
typeof Symbol === 'function' &&
(g[Symbol.iterator] = function () {
return this;
}),
g
);
function verb(n) {
return function (v) {
return step([n, v]);
};
}
function step(op) {
if (f) throw new TypeError('Generator is already executing.');
while ((g && ((g = 0), op[0] && (_ = 0)), _))
try {
if (
((f = 1),
y &&
(t = op[0] & 2 ? y['return'] : op[0] ? y['throw'] || ((t = y['return']) && t.call(y), 0) : y.next) &&
!(t = t.call(y, op[1])).done)
)
return t;
if (((y = 0), t)) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0:
case 1:
t = op;
break;
case 4:
_.label++;
return { value: op[1], done: false };
case 5:
_.label++;
y = op[1];
op = [0];
continue;
case 7:
op = _.ops.pop();
_.trys.pop();
continue;
default:
if (!((t = _.trys), (t = t.length > 0 && t[t.length - 1])) && (op[0] === 6 || op[0] === 2)) {
_ = 0;
continue;
}
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) {
_.label = op[1];
break;
}
if (op[0] === 6 && _.label < t[1]) {
_.label = t[1];
t = op;
break;
}
if (t && _.label < t[2]) {
_.label = t[2];
_.ops.push(op);
break;
}
if (t[2]) _.ops.pop();
_.trys.pop();
continue;
}
op = body.call(thisArg, _);
} catch (e) {
op = [6, e];
y = 0;
} finally {
f = t = 0;
}
if (op[0] & 5) throw op[1];
return { value: op[0] ? op[1] : void 0, done: true };
}
};
Object.defineProperty(exports, '__esModule', { value: true });
exports.processGenerateThumbnail = processGenerateThumbnail;
var promises_1 = require('node:fs/promises');
var node_path_1 = require('node:path');
var node_os_1 = require('node:os');
var queries_1 = require('@cio/db/queries');
var media_1 = require('@cio/jobs/payloads/media');
var storage_1 = require('../../utils/storage');
var cancel_1 = require('../../utils/cancel');
var storage_2 = require('../../config/storage');
var ffmpeg_1 = require('../../utils/ffmpeg');
var logger_1 = require('../../utils/logger');
var STEP_KEY = 'generate-thumbnail';
var DOMAIN = 'media';
function extractThumbnail(localPath, atSeconds, outputPath) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
return [
4 /*yield*/,
(0, ffmpeg_1.ffmpegRun)([
'-y',
'-ss',
String(atSeconds),
'-i',
localPath,
'-vframes',
'1',
'-vf',
'scale=640:-1',
outputPath
])
];
case 1:
_a.sent();
return [2 /*return*/];
}
});
});
}
/**
* Render a single JPEG frame to use as the asset thumbnail. Replaces the
* browser-side thumbnail as the source of truth on the asset row.
*/
function processGenerateThumbnail(rawData) {
return __awaiter(this, void 0, void 0, function () {
var payload,
mediaJobId,
assetId,
storageKey,
actorContext,
_a,
atSeconds,
existing,
localPath,
outputPath,
dir,
buffer,
thumbKey,
baseUrl,
publicUrl,
result,
error_1;
var _b;
return __generator(this, function (_c) {
switch (_c.label) {
case 0:
payload = media_1.ZGenerateThumbnailPayload.parse(rawData);
((mediaJobId = payload.mediaJobId),
(assetId = payload.assetId),
(storageKey = payload.storageKey),
(actorContext = payload.actorContext),
(_a = payload.atSeconds),
(atSeconds = _a === void 0 ? 1.0 : _a));
return [4 /*yield*/, (0, queries_1.getJobStep)(DOMAIN, mediaJobId, STEP_KEY)];
case 1:
existing = _c.sent();
if (
(existing === null || existing === void 0 ? void 0 : existing.status) === 'completed' &&
existing.result
) {
logger_1.log.info('generate-thumbnail-skip', { mediaJobId: mediaJobId, reason: 'ledger-completed' });
return [2 /*return*/, existing.result];
}
return [4 /*yield*/, (0, cancel_1.throwIfCancelRequested)(mediaJobId)];
case 2:
_c.sent();
return [
4 /*yield*/,
(0, queries_1.updateMediaJob)(mediaJobId, { stage: 'thumbnailing', progressPercent: 35 })
];
case 3:
_c.sent();
return [
4 /*yield*/,
(0, queries_1.upsertJobStep)({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'running',
startedAt: new Date().toISOString(),
attempt:
((_b = existing === null || existing === void 0 ? void 0 : existing.attempt) !== null && _b !== void 0
? _b
: 0) + 1
})
];
case 4:
_c.sent();
_c.label = 5;
case 5:
_c.trys.push([5, 15, 17, 20]);
return [
4 /*yield*/,
(0, storage_1.downloadObjectToTempFile)((0, storage_1.videosBucket)(), storageKey, 'thumb-source.bin')
];
case 6:
localPath = _c.sent();
dir = node_path_1.default.join((0, node_os_1.tmpdir)(), 'cio-jobs', 'thumbs');
return [4 /*yield*/, (0, promises_1.mkdir)(dir, { recursive: true })];
case 7:
_c.sent();
outputPath = node_path_1.default.join(dir, ''.concat(mediaJobId, '.jpg'));
return [4 /*yield*/, (0, cancel_1.throwIfCancelRequested)(mediaJobId)];
case 8:
_c.sent();
return [4 /*yield*/, extractThumbnail(localPath, atSeconds, outputPath)];
case 9:
_c.sent();
return [4 /*yield*/, (0, promises_1.readFile)(outputPath)];
case 10:
buffer = _c.sent();
thumbKey = 'thumbnails/'.concat(assetId, '/').concat(Date.now(), '.jpg');
return [
4 /*yield*/,
(0, storage_1.uploadBufferToBucket)(
(0, storage_1.mediaBucket)(),
thumbKey,
buffer,
'image/jpeg',
'public, max-age=31536000'
)
];
case 11:
_c.sent();
baseUrl = (0, storage_2.getStorageConfig)().mediaPublicBaseUrl;
publicUrl = baseUrl ? ''.concat(baseUrl.replace(/\/$/, ''), '/').concat(thumbKey) : null;
result = { bucket: (0, storage_1.mediaBucket)(), key: thumbKey, publicUrl: publicUrl };
return [
4 /*yield*/,
(0, queries_1.updateAsset)(assetId, actorContext.organizationId, {
thumbnailUrl: publicUrl !== null && publicUrl !== void 0 ? publicUrl : undefined
})
];
case 12:
_c.sent();
return [
4 /*yield*/,
(0, queries_1.upsertJobStep)({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'completed',
finishedAt: new Date().toISOString(),
result: result
})
];
case 13:
_c.sent();
return [
4 /*yield*/,
(0, queries_1.updateMediaJob)(mediaJobId, { stage: 'thumbnailed', progressPercent: 60 })
];
case 14:
_c.sent();
logger_1.log.info('generate-thumbnail-done', { mediaJobId: mediaJobId, thumbKey: thumbKey });
return [2 /*return*/, result];
case 15:
error_1 = _c.sent();
logger_1.log.error('generate-thumbnail-failed', {
mediaJobId: mediaJobId,
error: (0, cancel_1.errorMessage)(error_1)
});
return [
4 /*yield*/,
(0, queries_1.upsertJobStep)({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'failed',
finishedAt: new Date().toISOString(),
error: { code: 'THUMBNAIL_FAILED', message: (0, cancel_1.errorMessage)(error_1) }
})
];
case 16:
_c.sent();
throw error_1;
case 17:
return [4 /*yield*/, (0, storage_1.safeUnlink)(localPath)];
case 18:
_c.sent();
return [4 /*yield*/, (0, storage_1.safeUnlink)(outputPath)];
case 19:
_c.sent();
return [7 /*endfinally*/];
case 20:
return [2 /*return*/];
}
});
});
}
@@ -0,0 +1,123 @@
import { mkdir, readFile } from 'node:fs/promises';
import path from 'node:path';
import { tmpdir } from 'node:os';
import { getJobStep, updateAsset, updateMediaJob, upsertJobStep } from '@cio/db/queries';
import { ZGenerateThumbnailPayload, type TGenerateThumbnailPayload } from '@cio/jobs/payloads/media';
import {
downloadObjectToTempFile,
mediaBucket,
safeUnlink,
uploadBufferToBucket,
videosBucket
} from '../../utils/storage';
import { errorMessage, throwIfCancelRequested } from '../../utils/cancel';
import { getStorageConfig } from '../../config/storage';
import { ffmpegRun } from '../../utils/ffmpeg';
import { log } from '../../utils/logger';
const STEP_KEY = 'generate-thumbnail';
const DOMAIN = 'media';
interface ThumbnailResult {
bucket: string;
key: string;
publicUrl: string | null;
}
async function extractThumbnail(localPath: string, atSeconds: number, outputPath: string): Promise<void> {
await ffmpegRun([
'-y',
'-ss',
String(atSeconds),
'-i',
localPath,
'-vframes',
'1',
'-vf',
'scale=640:-1',
outputPath
]);
}
/**
* Render a single JPEG frame to use as the asset thumbnail. Replaces the
* browser-side thumbnail as the source of truth on the asset row.
*/
export async function processGenerateThumbnail(rawData: unknown): Promise<ThumbnailResult> {
const payload: TGenerateThumbnailPayload = ZGenerateThumbnailPayload.parse(rawData);
const { mediaJobId, assetId, storageKey, actorContext, atSeconds = 1.0 } = payload;
const existing = await getJobStep(DOMAIN, mediaJobId, STEP_KEY);
if (existing?.status === 'completed' && existing.result) {
log.info('generate-thumbnail-skip', { mediaJobId, reason: 'ledger-completed' });
return existing.result as unknown as ThumbnailResult;
}
await throwIfCancelRequested(mediaJobId);
await updateMediaJob(mediaJobId, { stage: 'thumbnailing', progressPercent: 35 });
await upsertJobStep({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'running',
startedAt: new Date().toISOString(),
attempt: (existing?.attempt ?? 0) + 1
});
let localPath: string | undefined;
let outputPath: string | undefined;
try {
localPath = await downloadObjectToTempFile(videosBucket(), storageKey, 'thumb-source.bin');
const dir = path.join(tmpdir(), 'cio-jobs', 'thumbs');
await mkdir(dir, { recursive: true });
outputPath = path.join(dir, `${mediaJobId}.jpg`);
await throwIfCancelRequested(mediaJobId);
await extractThumbnail(localPath, atSeconds, outputPath);
const buffer = await readFile(outputPath);
const thumbKey = `thumbnails/${assetId}/${Date.now()}.jpg`;
await uploadBufferToBucket(mediaBucket(), thumbKey, buffer, 'image/jpeg', 'public, max-age=31536000');
const baseUrl = getStorageConfig().mediaPublicBaseUrl;
const publicUrl = baseUrl ? `${baseUrl.replace(/\/$/, '')}/${thumbKey}` : null;
const result: ThumbnailResult = { bucket: mediaBucket(), key: thumbKey, publicUrl };
await updateAsset(assetId, actorContext.organizationId, {
thumbnailUrl: publicUrl ?? undefined
});
await upsertJobStep({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'completed',
finishedAt: new Date().toISOString(),
result: result as unknown as Record<string, unknown>
});
await updateMediaJob(mediaJobId, { stage: 'thumbnailed', progressPercent: 60 });
log.info('generate-thumbnail-done', { mediaJobId, thumbKey });
return result;
} catch (error) {
log.error('generate-thumbnail-failed', { mediaJobId, error: errorMessage(error) });
await upsertJobStep({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'failed',
finishedAt: new Date().toISOString(),
error: { code: 'THUMBNAIL_FAILED', message: errorMessage(error) }
});
throw error;
} finally {
await safeUnlink(localPath);
await safeUnlink(outputPath);
}
}
+32
View File
@@ -0,0 +1,32 @@
'use strict';
var __createBinding =
(this && this.__createBinding) ||
(Object.create
? function (o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ('get' in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = {
enumerable: true,
get: function () {
return m[k];
}
};
}
Object.defineProperty(o, k2, desc);
}
: function (o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
});
var __exportStar =
(this && this.__exportStar) ||
function (m, exports) {
for (var p in m)
if (p !== 'default' && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
};
Object.defineProperty(exports, '__esModule', { value: true });
__exportStar(require('./probe-metadata'), exports);
__exportStar(require('./generate-thumbnail'), exports);
__exportStar(require('./extract-audio'), exports);
__exportStar(require('./transcribe-audio'), exports);
+4
View File
@@ -0,0 +1,4 @@
export * from './probe-metadata';
export * from './generate-thumbnail';
export * from './extract-audio';
export * from './transcribe-audio';
@@ -0,0 +1,311 @@
'use strict';
var __awaiter =
(this && this.__awaiter) ||
function (thisArg, _arguments, P, generator) {
function adopt(value) {
return value instanceof P
? value
: new P(function (resolve) {
resolve(value);
});
}
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) {
try {
step(generator.next(value));
} catch (e) {
reject(e);
}
}
function rejected(value) {
try {
step(generator['throw'](value));
} catch (e) {
reject(e);
}
}
function step(result) {
result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected);
}
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator =
(this && this.__generator) ||
function (thisArg, body) {
var _ = {
label: 0,
sent: function () {
if (t[0] & 1) throw t[1];
return t[1];
},
trys: [],
ops: []
},
f,
y,
t,
g = Object.create((typeof Iterator === 'function' ? Iterator : Object).prototype);
return (
(g.next = verb(0)),
(g['throw'] = verb(1)),
(g['return'] = verb(2)),
typeof Symbol === 'function' &&
(g[Symbol.iterator] = function () {
return this;
}),
g
);
function verb(n) {
return function (v) {
return step([n, v]);
};
}
function step(op) {
if (f) throw new TypeError('Generator is already executing.');
while ((g && ((g = 0), op[0] && (_ = 0)), _))
try {
if (
((f = 1),
y &&
(t = op[0] & 2 ? y['return'] : op[0] ? y['throw'] || ((t = y['return']) && t.call(y), 0) : y.next) &&
!(t = t.call(y, op[1])).done)
)
return t;
if (((y = 0), t)) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0:
case 1:
t = op;
break;
case 4:
_.label++;
return { value: op[1], done: false };
case 5:
_.label++;
y = op[1];
op = [0];
continue;
case 7:
op = _.ops.pop();
_.trys.pop();
continue;
default:
if (!((t = _.trys), (t = t.length > 0 && t[t.length - 1])) && (op[0] === 6 || op[0] === 2)) {
_ = 0;
continue;
}
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) {
_.label = op[1];
break;
}
if (op[0] === 6 && _.label < t[1]) {
_.label = t[1];
t = op;
break;
}
if (t && _.label < t[2]) {
_.label = t[2];
_.ops.push(op);
break;
}
if (t[2]) _.ops.pop();
_.trys.pop();
continue;
}
op = body.call(thisArg, _);
} catch (e) {
op = [6, e];
y = 0;
} finally {
f = t = 0;
}
if (op[0] & 5) throw op[1];
return { value: op[0] ? op[1] : void 0, done: true };
}
};
Object.defineProperty(exports, '__esModule', { value: true });
exports.processProbeMetadata = processProbeMetadata;
var promises_1 = require('node:fs/promises');
var queries_1 = require('@cio/db/queries');
var media_1 = require('@cio/jobs/payloads/media');
var storage_1 = require('../../utils/storage');
var cancel_1 = require('../../utils/cancel');
var ffmpeg_1 = require('../../utils/ffmpeg');
var logger_1 = require('../../utils/logger');
var STEP_KEY = 'probe-metadata';
var DOMAIN = 'media';
function parseProbe(metadata, fileSizeBytes) {
var _a, _b;
var videoStream = metadata.streams.find(function (stream) {
return stream.codec_type === 'video';
});
var audioStream = metadata.streams.find(function (stream) {
return stream.codec_type === 'audio';
});
var duration = Number(
(_b =
(_a = metadata.format.duration) !== null && _a !== void 0
? _a
: videoStream === null || videoStream === void 0
? void 0
: videoStream.duration) !== null && _b !== void 0
? _b
: 0
);
return {
durationSeconds: Number.isFinite(duration) ? Math.max(0, Math.round(duration)) : 0,
width: videoStream === null || videoStream === void 0 ? void 0 : videoStream.width,
height: videoStream === null || videoStream === void 0 ? void 0 : videoStream.height,
videoCodec: videoStream === null || videoStream === void 0 ? void 0 : videoStream.codec_name,
audioCodec: audioStream === null || audioStream === void 0 ? void 0 : audioStream.codec_name,
hasAudio: Boolean(audioStream),
fileSizeBytes: fileSizeBytes
};
}
/**
* Probe authoritative metadata (duration, codecs, dimensions) using ffprobe.
* Replaces the browser-derived duration on the asset row.
*/
function processProbeMetadata(rawData) {
return __awaiter(this, void 0, void 0, function () {
var payload, mediaJobId, assetId, storageKey, actorContext, existing, localPath, fileStat, probe, result, error_1;
var _a;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
payload = media_1.ZProbeMetadataPayload.parse(rawData);
((mediaJobId = payload.mediaJobId),
(assetId = payload.assetId),
(storageKey = payload.storageKey),
(actorContext = payload.actorContext));
return [4 /*yield*/, (0, queries_1.getJobStep)(DOMAIN, mediaJobId, STEP_KEY)];
case 1:
existing = _b.sent();
if (
(existing === null || existing === void 0 ? void 0 : existing.status) === 'completed' &&
existing.result
) {
logger_1.log.info('probe-metadata-skip', { mediaJobId: mediaJobId, reason: 'ledger-completed' });
return [2 /*return*/, existing.result];
}
return [4 /*yield*/, (0, cancel_1.throwIfCancelRequested)(mediaJobId)];
case 2:
_b.sent();
return [
4 /*yield*/,
(0, queries_1.updateMediaJob)(mediaJobId, { status: 'running', stage: 'probing', progressPercent: 5 })
];
case 3:
_b.sent();
return [
4 /*yield*/,
(0, queries_1.upsertJobStep)({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'running',
startedAt: new Date().toISOString(),
attempt:
((_a = existing === null || existing === void 0 ? void 0 : existing.attempt) !== null && _a !== void 0
? _a
: 0) + 1
})
];
case 4:
_b.sent();
_b.label = 5;
case 5:
_b.trys.push([5, 13, 15, 17]);
return [
4 /*yield*/,
(0, storage_1.downloadObjectToTempFile)((0, storage_1.videosBucket)(), storageKey, 'source.bin')
];
case 6:
localPath = _b.sent();
return [4 /*yield*/, (0, promises_1.stat)(localPath)];
case 7:
fileStat = _b.sent();
return [4 /*yield*/, (0, cancel_1.throwIfCancelRequested)(mediaJobId)];
case 8:
_b.sent();
return [4 /*yield*/, (0, ffmpeg_1.ffprobeJson)(localPath)];
case 9:
probe = _b.sent();
result = parseProbe(probe, fileStat.size);
return [
4 /*yield*/,
(0, queries_1.updateAsset)(assetId, actorContext.organizationId, {
durationSeconds: result.durationSeconds,
aspectRatio: result.width && result.height ? ''.concat(result.width, ':').concat(result.height) : null,
metadata: {
probe: {
durationSeconds: result.durationSeconds,
width: result.width,
height: result.height,
videoCodec: result.videoCodec,
audioCodec: result.audioCodec,
hasAudio: result.hasAudio,
fileSizeBytes: result.fileSizeBytes,
probedAt: new Date().toISOString()
}
}
})
];
case 10:
_b.sent();
return [
4 /*yield*/,
(0, queries_1.upsertJobStep)({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'completed',
finishedAt: new Date().toISOString(),
result: result
})
];
case 11:
_b.sent();
return [
4 /*yield*/,
(0, queries_1.updateMediaJob)(mediaJobId, {
stage: 'probed',
progressPercent: 25
})
];
case 12:
_b.sent();
logger_1.log.info('probe-metadata-done', { mediaJobId: mediaJobId, durationSeconds: result.durationSeconds });
return [2 /*return*/, result];
case 13:
error_1 = _b.sent();
logger_1.log.error('probe-metadata-failed', {
mediaJobId: mediaJobId,
error: (0, cancel_1.errorMessage)(error_1)
});
return [
4 /*yield*/,
(0, queries_1.upsertJobStep)({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'failed',
finishedAt: new Date().toISOString(),
error: { code: 'PROBE_FAILED', message: (0, cancel_1.errorMessage)(error_1) }
})
];
case 14:
_b.sent();
throw error_1;
case 15:
return [4 /*yield*/, (0, storage_1.safeUnlink)(localPath)];
case 16:
_b.sent();
return [7 /*endfinally*/];
case 17:
return [2 /*return*/];
}
});
});
}
@@ -0,0 +1,122 @@
import { stat } from 'node:fs/promises';
import { getJobStep, updateAsset, updateMediaJob, upsertJobStep } from '@cio/db/queries';
import { ZProbeMetadataPayload, type TProbeMetadataPayload } from '@cio/jobs/payloads/media';
import { downloadObjectToTempFile, safeUnlink, videosBucket } from '../../utils/storage';
import { errorMessage, throwIfCancelRequested } from '../../utils/cancel';
import { ffprobeJson, type FfprobeData } from '../../utils/ffmpeg';
import { log } from '../../utils/logger';
const STEP_KEY = 'probe-metadata';
const DOMAIN = 'media';
interface ProbeResult {
durationSeconds: number;
width?: number;
height?: number;
videoCodec?: string;
audioCodec?: string;
hasAudio: boolean;
fileSizeBytes: number;
}
function parseProbe(metadata: FfprobeData, fileSizeBytes: number): ProbeResult {
const videoStream = metadata.streams.find((stream) => stream.codec_type === 'video');
const audioStream = metadata.streams.find((stream) => stream.codec_type === 'audio');
const duration = Number(metadata.format.duration ?? videoStream?.duration ?? 0);
return {
durationSeconds: Number.isFinite(duration) ? Math.max(0, Math.round(duration)) : 0,
width: videoStream?.width,
height: videoStream?.height,
videoCodec: videoStream?.codec_name,
audioCodec: audioStream?.codec_name,
hasAudio: Boolean(audioStream),
fileSizeBytes
};
}
/**
* Probe authoritative metadata (duration, codecs, dimensions) using ffprobe.
* Replaces the browser-derived duration on the asset row.
*/
export async function processProbeMetadata(rawData: unknown): Promise<ProbeResult> {
const payload: TProbeMetadataPayload = ZProbeMetadataPayload.parse(rawData);
const { mediaJobId, assetId, storageKey, actorContext } = payload;
const existing = await getJobStep(DOMAIN, mediaJobId, STEP_KEY);
if (existing?.status === 'completed' && existing.result) {
log.info('probe-metadata-skip', { mediaJobId, reason: 'ledger-completed' });
return existing.result as unknown as ProbeResult;
}
await throwIfCancelRequested(mediaJobId);
await updateMediaJob(mediaJobId, { status: 'running', stage: 'probing', progressPercent: 5 });
await upsertJobStep({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'running',
startedAt: new Date().toISOString(),
attempt: (existing?.attempt ?? 0) + 1
});
let localPath: string | undefined;
try {
localPath = await downloadObjectToTempFile(videosBucket(), storageKey, 'source.bin');
const fileStat = await stat(localPath);
await throwIfCancelRequested(mediaJobId);
const probe = await ffprobeJson(localPath);
const result = parseProbe(probe, fileStat.size);
await updateAsset(assetId, actorContext.organizationId, {
durationSeconds: result.durationSeconds,
aspectRatio: result.width && result.height ? `${result.width}:${result.height}` : null,
metadata: {
probe: {
durationSeconds: result.durationSeconds,
width: result.width,
height: result.height,
videoCodec: result.videoCodec,
audioCodec: result.audioCodec,
hasAudio: result.hasAudio,
fileSizeBytes: result.fileSizeBytes,
probedAt: new Date().toISOString()
}
}
});
await upsertJobStep({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'completed',
finishedAt: new Date().toISOString(),
result: result as unknown as Record<string, unknown>
});
await updateMediaJob(mediaJobId, {
stage: 'probed',
progressPercent: 25
});
log.info('probe-metadata-done', { mediaJobId, durationSeconds: result.durationSeconds });
return result;
} catch (error) {
log.error('probe-metadata-failed', { mediaJobId, error: errorMessage(error) });
await upsertJobStep({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'failed',
finishedAt: new Date().toISOString(),
error: { code: 'PROBE_FAILED', message: errorMessage(error) }
});
throw error;
} finally {
await safeUnlink(localPath);
}
}
@@ -0,0 +1,538 @@
'use strict';
var __awaiter =
(this && this.__awaiter) ||
function (thisArg, _arguments, P, generator) {
function adopt(value) {
return value instanceof P
? value
: new P(function (resolve) {
resolve(value);
});
}
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) {
try {
step(generator.next(value));
} catch (e) {
reject(e);
}
}
function rejected(value) {
try {
step(generator['throw'](value));
} catch (e) {
reject(e);
}
}
function step(result) {
result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected);
}
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator =
(this && this.__generator) ||
function (thisArg, body) {
var _ = {
label: 0,
sent: function () {
if (t[0] & 1) throw t[1];
return t[1];
},
trys: [],
ops: []
},
f,
y,
t,
g = Object.create((typeof Iterator === 'function' ? Iterator : Object).prototype);
return (
(g.next = verb(0)),
(g['throw'] = verb(1)),
(g['return'] = verb(2)),
typeof Symbol === 'function' &&
(g[Symbol.iterator] = function () {
return this;
}),
g
);
function verb(n) {
return function (v) {
return step([n, v]);
};
}
function step(op) {
if (f) throw new TypeError('Generator is already executing.');
while ((g && ((g = 0), op[0] && (_ = 0)), _))
try {
if (
((f = 1),
y &&
(t = op[0] & 2 ? y['return'] : op[0] ? y['throw'] || ((t = y['return']) && t.call(y), 0) : y.next) &&
!(t = t.call(y, op[1])).done)
)
return t;
if (((y = 0), t)) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0:
case 1:
t = op;
break;
case 4:
_.label++;
return { value: op[1], done: false };
case 5:
_.label++;
y = op[1];
op = [0];
continue;
case 7:
op = _.ops.pop();
_.trys.pop();
continue;
default:
if (!((t = _.trys), (t = t.length > 0 && t[t.length - 1])) && (op[0] === 6 || op[0] === 2)) {
_ = 0;
continue;
}
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) {
_.label = op[1];
break;
}
if (op[0] === 6 && _.label < t[1]) {
_.label = t[1];
t = op;
break;
}
if (t && _.label < t[2]) {
_.label = t[2];
_.ops.push(op);
break;
}
if (t[2]) _.ops.pop();
_.trys.pop();
continue;
}
op = body.call(thisArg, _);
} catch (e) {
op = [6, e];
y = 0;
} finally {
f = t = 0;
}
if (op[0] & 5) throw op[1];
return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __spreadArray =
(this && this.__spreadArray) ||
function (to, from, pack) {
if (pack || arguments.length === 2)
for (var i = 0, l = from.length, ar; i < l; i++) {
if (ar || !(i in from)) {
if (!ar) ar = Array.prototype.slice.call(from, 0, i);
ar[i] = from[i];
}
}
return to.concat(ar || Array.prototype.slice.call(from));
};
Object.defineProperty(exports, '__esModule', { value: true });
exports.processTranscribeAudio = processTranscribeAudio;
var promises_1 = require('node:fs/promises');
var queries_1 = require('@cio/db/queries');
var media_1 = require('@cio/jobs/payloads/media');
var env_1 = require('../../config/env');
var openai_1 = require('../../services/transcription/openai');
var vtt_1 = require('../../services/transcription/vtt');
var storage_1 = require('../../utils/storage');
var cancel_1 = require('../../utils/cancel');
var logger_1 = require('../../utils/logger');
var STEP_KEY = 'transcribe-audio';
var DOMAIN = 'media';
var EXTRACT_STEP_KEY = 'extract-audio';
function transcriptionCostCents(durationSeconds) {
return Math.ceil((durationSeconds / 60) * 0.6);
}
function mergeJobWarnings(mediaJobId, extra) {
return __awaiter(this, void 0, void 0, function () {
var job, prior;
var _a;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
return [4 /*yield*/, (0, queries_1.getMediaJobById)(mediaJobId)];
case 1:
job = _b.sent();
prior = (_a = job === null || job === void 0 ? void 0 : job.warnings) !== null && _a !== void 0 ? _a : [];
return [
4 /*yield*/,
(0, queries_1.updateMediaJob)(mediaJobId, {
warnings: __spreadArray(__spreadArray([], prior, true), extra, true)
})
];
case 2:
_b.sent();
return [2 /*return*/];
}
});
});
}
/**
* OpenAI Whisper (`whisper-1`) with verbose segment timestamps; persists VTT +
* `media_transcript` row. Skips cleanly when `OPENAI_API_KEY` is unset or when
* extracted audio exceeds the 25 MB API limit.
*/
function processTranscribeAudio(rawData) {
return __awaiter(this, void 0, void 0, function () {
var payload,
mediaJobId,
assetId,
actorContext,
existing,
result,
extractStep,
extractResult,
message,
localPath,
localStat,
whisper,
normalizedSegments,
vttBody,
vttKey,
bucket,
transcriptCost,
jobRow,
priorCost,
result,
error_1,
error_2;
var _a, _b;
return __generator(this, function (_c) {
switch (_c.label) {
case 0:
payload = media_1.ZTranscribeAudioPayload.parse(rawData);
((mediaJobId = payload.mediaJobId), (assetId = payload.assetId), (actorContext = payload.actorContext));
return [4 /*yield*/, (0, queries_1.getJobStep)(DOMAIN, mediaJobId, STEP_KEY)];
case 1:
existing = _c.sent();
if (
(existing === null || existing === void 0 ? void 0 : existing.status) === 'completed' &&
existing.result
) {
logger_1.log.info('transcribe-audio-skip', { mediaJobId: mediaJobId, reason: 'ledger-completed' });
return [2 /*return*/, existing.result];
}
return [4 /*yield*/, (0, cancel_1.throwIfCancelRequested)(mediaJobId)];
case 2:
_c.sent();
return [
4 /*yield*/,
(0, queries_1.updateMediaJob)(mediaJobId, { stage: 'transcribing', progressPercent: 90 })
];
case 3:
_c.sent();
return [
4 /*yield*/,
(0, queries_1.upsertJobStep)({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'running',
startedAt: new Date().toISOString(),
attempt:
((_a = existing === null || existing === void 0 ? void 0 : existing.attempt) !== null && _a !== void 0
? _a
: 0) + 1
})
];
case 4:
_c.sent();
_c.label = 5;
case 5:
_c.trys.push([5, 35, , 37]);
if (!!env_1.env.OPENAI_API_KEY) return [3 /*break*/, 9];
result = { provider: 'skipped', text: '' };
return [
4 /*yield*/,
(0, queries_1.upsertJobStep)({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'completed',
finishedAt: new Date().toISOString(),
result: result
})
];
case 6:
_c.sent();
return [4 /*yield*/, mergeJobWarnings(mediaJobId, ['transcription-skipped-no-openai-key'])];
case 7:
_c.sent();
return [
4 /*yield*/,
(0, queries_1.updateMediaJob)(mediaJobId, {
status: 'completed',
stage: 'done',
progressPercent: 100
})
];
case 8:
_c.sent();
logger_1.log.info('transcribe-audio-skipped', { mediaJobId: mediaJobId, reason: 'no-openai-key' });
return [2 /*return*/, result];
case 9:
return [4 /*yield*/, (0, queries_1.getJobStep)(DOMAIN, mediaJobId, EXTRACT_STEP_KEY)];
case 10:
extractStep = _c.sent();
extractResult = extractStep === null || extractStep === void 0 ? void 0 : extractStep.result;
if (
!(
!(extractResult === null || extractResult === void 0 ? void 0 : extractResult.bucket) ||
!(extractResult === null || extractResult === void 0 ? void 0 : extractResult.key)
)
)
return [3 /*break*/, 12];
message = 'extract-audio step missing or incomplete';
logger_1.log.error('transcribe-audio-failed', { mediaJobId: mediaJobId, error: message });
return [
4 /*yield*/,
(0, queries_1.upsertJobStep)({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'failed',
finishedAt: new Date().toISOString(),
error: { code: 'EXTRACT_AUDIO_MISSING', message: message }
})
];
case 11:
_c.sent();
throw new Error(message);
case 12:
localPath = void 0;
_c.label = 13;
case 13:
_c.trys.push([13, 27, 32, 34]);
return [
4 /*yield*/,
(0, storage_1.downloadObjectToTempFile)(extractResult.bucket, extractResult.key, 'whisper-input.mp3')
];
case 14:
localPath = _c.sent();
return [4 /*yield*/, (0, promises_1.stat)(localPath)];
case 15:
localStat = _c.sent();
if (!(localStat.size > openai_1.WHISPER_MAX_FILE_BYTES)) return [3 /*break*/, 19];
return [
4 /*yield*/,
(0, queries_1.upsertJobStep)({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'failed',
finishedAt: new Date().toISOString(),
error: {
code: 'AUDIO_TOO_LARGE',
message: 'Audio '.concat(localStat.size, ' bytes exceeds ').concat(openai_1.WHISPER_MAX_FILE_BYTES)
}
})
];
case 16:
_c.sent();
return [4 /*yield*/, mergeJobWarnings(mediaJobId, ['transcription-skipped-audio-too-large'])];
case 17:
_c.sent();
return [
4 /*yield*/,
(0, queries_1.updateMediaJob)(mediaJobId, {
status: 'completed',
stage: 'done',
progressPercent: 100
})
];
case 18:
_c.sent();
logger_1.log.warn('transcribe-audio-skipped-audio-too-large', {
mediaJobId: mediaJobId,
fileSizeBytes: localStat.size
});
return [2 /*return*/, { provider: 'skipped', text: '' }];
case 19:
return [4 /*yield*/, (0, cancel_1.throwIfCancelRequested)(mediaJobId)];
case 20:
_c.sent();
return [
4 /*yield*/,
(0, openai_1.transcribeAudioFile)({
apiKey: env_1.env.OPENAI_API_KEY,
localPath: localPath
})
];
case 21:
whisper = _c.sent();
normalizedSegments = whisper.segments.map(function (segment) {
return {
start: segment.start,
end: segment.end,
text: segment.text
};
});
vttBody = (0, vtt_1.segmentsToWebVtt)(normalizedSegments);
vttKey = 'transcripts/'.concat(assetId, '/').concat(mediaJobId, '.vtt');
bucket = (0, storage_1.mediaBucket)();
return [
4 /*yield*/,
(0, storage_1.uploadBufferToBucket)(
bucket,
vttKey,
Buffer.from(vttBody, 'utf8'),
'text/vtt; charset=utf-8',
'private, max-age=0'
)
];
case 22:
_c.sent();
transcriptCost = transcriptionCostCents(whisper.durationSeconds);
return [4 /*yield*/, (0, queries_1.getMediaJobById)(mediaJobId)];
case 23:
jobRow = _c.sent();
priorCost =
(_b = jobRow === null || jobRow === void 0 ? void 0 : jobRow.costCents) !== null && _b !== void 0 ? _b : 0;
return [
4 /*yield*/,
(0, queries_1.upsertMediaTranscript)({
organizationId: actorContext.organizationId,
assetId: assetId,
mediaJobId: mediaJobId,
language: whisper.language,
provider: 'openai',
model: 'whisper-1',
text:
whisper.fullText ||
normalizedSegments
.map(function (segment) {
return segment.text;
})
.join(' ')
.trim(),
segments: normalizedSegments,
vttStorageKey: vttKey,
vttBucket: bucket,
durationSeconds: Math.round(whisper.durationSeconds),
costCents: transcriptCost
})
];
case 24:
_c.sent();
result = {
provider: 'openai',
text: whisper.fullText,
language: whisper.language,
durationSeconds: whisper.durationSeconds,
costCents: transcriptCost
};
return [
4 /*yield*/,
(0, queries_1.upsertJobStep)({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'completed',
finishedAt: new Date().toISOString(),
result: {
language: whisper.language,
segmentCount: normalizedSegments.length,
vttKey: vttKey,
costCents: transcriptCost
},
providerId: null
})
];
case 25:
_c.sent();
return [
4 /*yield*/,
(0, queries_1.updateMediaJob)(mediaJobId, {
status: 'completed',
stage: 'done',
progressPercent: 100,
costCents: priorCost + transcriptCost
})
];
case 26:
_c.sent();
logger_1.log.info('transcribe-audio-done', {
mediaJobId: mediaJobId,
language: whisper.language,
segmentCount: normalizedSegments.length
});
return [2 /*return*/, result];
case 27:
error_1 = _c.sent();
if (!(error_1 instanceof openai_1.AudioFileTooLargeError)) return [3 /*break*/, 31];
return [
4 /*yield*/,
(0, queries_1.upsertJobStep)({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'failed',
finishedAt: new Date().toISOString(),
error: { code: 'AUDIO_TOO_LARGE', message: error_1.message }
})
];
case 28:
_c.sent();
return [4 /*yield*/, mergeJobWarnings(mediaJobId, ['transcription-skipped-audio-too-large'])];
case 29:
_c.sent();
return [
4 /*yield*/,
(0, queries_1.updateMediaJob)(mediaJobId, {
status: 'completed',
stage: 'done',
progressPercent: 100
})
];
case 30:
_c.sent();
logger_1.log.warn('transcribe-audio-skipped-audio-too-large', {
mediaJobId: mediaJobId,
fileSizeBytes: error_1.fileSizeBytes
});
return [2 /*return*/, { provider: 'skipped', text: '' }];
case 31:
throw error_1;
case 32:
return [4 /*yield*/, (0, storage_1.safeUnlink)(localPath)];
case 33:
_c.sent();
return [7 /*endfinally*/];
case 34:
return [3 /*break*/, 37];
case 35:
error_2 = _c.sent();
logger_1.log.error('transcribe-audio-failed', {
mediaJobId: mediaJobId,
error: (0, cancel_1.errorMessage)(error_2)
});
return [
4 /*yield*/,
(0, queries_1.upsertJobStep)({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'failed',
finishedAt: new Date().toISOString(),
error: { code: 'TRANSCRIBE_FAILED', message: (0, cancel_1.errorMessage)(error_2) }
})
];
case 36:
_c.sent();
throw error_2;
case 37:
return [2 /*return*/];
}
});
});
}
@@ -0,0 +1,254 @@
import { stat } from 'node:fs/promises';
import { getJobStep, getMediaJobById, updateMediaJob, upsertJobStep, upsertMediaTranscript } from '@cio/db/queries';
import { ZTranscribeAudioPayload, type TTranscribeAudioPayload } from '@cio/jobs/payloads/media';
import { env } from '../../config/env';
import {
AudioFileTooLargeError,
WHISPER_MAX_FILE_BYTES,
transcribeAudioFile,
type TranscriptionSegment
} from '../../services/transcription/openai';
import { segmentsToWebVtt } from '../../services/transcription/vtt';
import { downloadObjectToTempFile, mediaBucket, safeUnlink, uploadBufferToBucket } from '../../utils/storage';
import { errorMessage, throwIfCancelRequested } from '../../utils/cancel';
import { log } from '../../utils/logger';
const STEP_KEY = 'transcribe-audio';
const DOMAIN = 'media';
const EXTRACT_STEP_KEY = 'extract-audio';
interface TranscribeResult {
provider: 'openai' | 'assemblyai' | 'deepgram' | 'skipped';
text: string;
language?: string;
durationSeconds?: number;
costCents?: number;
}
function transcriptionCostCents(durationSeconds: number): number {
return Math.ceil((durationSeconds / 60) * 0.6);
}
async function mergeJobWarnings(mediaJobId: string, extra: string[]): Promise<void> {
const job = await getMediaJobById(mediaJobId);
const prior = job?.warnings ?? [];
await updateMediaJob(mediaJobId, { warnings: [...prior, ...extra] });
}
/**
* OpenAI Whisper (`whisper-1`) with verbose segment timestamps; persists VTT +
* `media_transcript` row. Skips cleanly when `OPENAI_API_KEY` is unset or when
* extracted audio exceeds the 25 MB API limit.
*/
export async function processTranscribeAudio(rawData: unknown): Promise<TranscribeResult> {
const payload: TTranscribeAudioPayload = ZTranscribeAudioPayload.parse(rawData);
const { mediaJobId, assetId, actorContext } = payload;
const existing = await getJobStep(DOMAIN, mediaJobId, STEP_KEY);
if (existing?.status === 'completed' && existing.result) {
log.info('transcribe-audio-skip', { mediaJobId, reason: 'ledger-completed' });
return existing.result as unknown as TranscribeResult;
}
await throwIfCancelRequested(mediaJobId);
await updateMediaJob(mediaJobId, { stage: 'transcribing', progressPercent: 90 });
await upsertJobStep({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'running',
startedAt: new Date().toISOString(),
attempt: (existing?.attempt ?? 0) + 1
});
try {
if (!env.OPENAI_API_KEY) {
const result: TranscribeResult = { provider: 'skipped', text: '' };
await upsertJobStep({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'completed',
finishedAt: new Date().toISOString(),
result: result as unknown as Record<string, unknown>
});
await mergeJobWarnings(mediaJobId, ['transcription-skipped-no-openai-key']);
await updateMediaJob(mediaJobId, {
status: 'completed',
stage: 'done',
progressPercent: 100
});
log.info('transcribe-audio-skipped', { mediaJobId, reason: 'no-openai-key' });
return result;
}
const extractStep = await getJobStep(DOMAIN, mediaJobId, EXTRACT_STEP_KEY);
const extractResult = extractStep?.result as { bucket?: string; key?: string; fileSizeBytes?: number } | null;
if (!extractResult?.bucket || !extractResult?.key) {
const message = 'extract-audio step missing or incomplete';
log.error('transcribe-audio-failed', { mediaJobId, error: message });
await upsertJobStep({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'failed',
finishedAt: new Date().toISOString(),
error: { code: 'EXTRACT_AUDIO_MISSING', message }
});
throw new Error(message);
}
let localPath: string | undefined;
try {
localPath = await downloadObjectToTempFile(extractResult.bucket, extractResult.key, 'whisper-input.mp3');
const localStat = await stat(localPath);
if (localStat.size > WHISPER_MAX_FILE_BYTES) {
await upsertJobStep({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'failed',
finishedAt: new Date().toISOString(),
error: {
code: 'AUDIO_TOO_LARGE',
message: `Audio ${localStat.size} bytes exceeds ${WHISPER_MAX_FILE_BYTES}`
}
});
await mergeJobWarnings(mediaJobId, ['transcription-skipped-audio-too-large']);
await updateMediaJob(mediaJobId, {
status: 'completed',
stage: 'done',
progressPercent: 100
});
log.warn('transcribe-audio-skipped-audio-too-large', { mediaJobId, fileSizeBytes: localStat.size });
return { provider: 'skipped', text: '' };
}
await throwIfCancelRequested(mediaJobId);
const whisper = await transcribeAudioFile({
apiKey: env.OPENAI_API_KEY,
localPath
});
const normalizedSegments: TranscriptionSegment[] = whisper.segments.map((segment) => ({
start: segment.start,
end: segment.end,
text: segment.text
}));
const vttBody = segmentsToWebVtt(normalizedSegments);
const vttKey = `transcripts/${assetId}/${mediaJobId}.vtt`;
const bucket = mediaBucket();
await uploadBufferToBucket(
bucket,
vttKey,
Buffer.from(vttBody, 'utf8'),
'text/vtt; charset=utf-8',
'private, max-age=0'
);
const transcriptCost = transcriptionCostCents(whisper.durationSeconds);
const jobRow = await getMediaJobById(mediaJobId);
const priorCost = jobRow?.costCents ?? 0;
await upsertMediaTranscript({
organizationId: actorContext.organizationId,
assetId,
mediaJobId,
language: whisper.language,
provider: 'openai',
model: 'whisper-1',
text:
whisper.fullText ||
normalizedSegments
.map((segment) => segment.text)
.join(' ')
.trim(),
segments: normalizedSegments,
vttStorageKey: vttKey,
vttBucket: bucket,
durationSeconds: Math.round(whisper.durationSeconds),
costCents: transcriptCost
});
const result: TranscribeResult = {
provider: 'openai',
text: whisper.fullText,
language: whisper.language,
durationSeconds: whisper.durationSeconds,
costCents: transcriptCost
};
await upsertJobStep({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'completed',
finishedAt: new Date().toISOString(),
result: {
language: whisper.language,
segmentCount: normalizedSegments.length,
vttKey,
costCents: transcriptCost
} as Record<string, unknown>,
providerId: null
});
await updateMediaJob(mediaJobId, {
status: 'completed',
stage: 'done',
progressPercent: 100,
costCents: priorCost + transcriptCost
});
log.info('transcribe-audio-done', {
mediaJobId,
language: whisper.language,
segmentCount: normalizedSegments.length
});
return result;
} catch (error) {
if (error instanceof AudioFileTooLargeError) {
await upsertJobStep({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'failed',
finishedAt: new Date().toISOString(),
error: { code: 'AUDIO_TOO_LARGE', message: error.message }
});
await mergeJobWarnings(mediaJobId, ['transcription-skipped-audio-too-large']);
await updateMediaJob(mediaJobId, {
status: 'completed',
stage: 'done',
progressPercent: 100
});
log.warn('transcribe-audio-skipped-audio-too-large', { mediaJobId, fileSizeBytes: error.fileSizeBytes });
return { provider: 'skipped', text: '' };
}
throw error;
} finally {
await safeUnlink(localPath);
}
} catch (error) {
log.error('transcribe-audio-failed', { mediaJobId, error: errorMessage(error) });
await upsertJobStep({
domain: DOMAIN,
runId: mediaJobId,
stepKey: STEP_KEY,
status: 'failed',
finishedAt: new Date().toISOString(),
error: { code: 'TRANSCRIBE_FAILED', message: errorMessage(error) }
});
throw error;
}
}
@@ -0,0 +1,245 @@
'use strict';
var __extends =
(this && this.__extends) ||
(function () {
var extendStatics = function (d, b) {
extendStatics =
Object.setPrototypeOf ||
({ __proto__: [] } instanceof Array &&
function (d, b) {
d.__proto__ = b;
}) ||
function (d, b) {
for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p];
};
return extendStatics(d, b);
};
return function (d, b) {
if (typeof b !== 'function' && b !== null)
throw new TypeError('Class extends value ' + String(b) + ' is not a constructor or null');
extendStatics(d, b);
function __() {
this.constructor = d;
}
d.prototype = b === null ? Object.create(b) : ((__.prototype = b.prototype), new __());
};
})();
var __assign =
(this && this.__assign) ||
function () {
__assign =
Object.assign ||
function (t) {
for (var s, i = 1, n = arguments.length; i < n; i++) {
s = arguments[i];
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) t[p] = s[p];
}
return t;
};
return __assign.apply(this, arguments);
};
var __awaiter =
(this && this.__awaiter) ||
function (thisArg, _arguments, P, generator) {
function adopt(value) {
return value instanceof P
? value
: new P(function (resolve) {
resolve(value);
});
}
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) {
try {
step(generator.next(value));
} catch (e) {
reject(e);
}
}
function rejected(value) {
try {
step(generator['throw'](value));
} catch (e) {
reject(e);
}
}
function step(result) {
result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected);
}
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator =
(this && this.__generator) ||
function (thisArg, body) {
var _ = {
label: 0,
sent: function () {
if (t[0] & 1) throw t[1];
return t[1];
},
trys: [],
ops: []
},
f,
y,
t,
g = Object.create((typeof Iterator === 'function' ? Iterator : Object).prototype);
return (
(g.next = verb(0)),
(g['throw'] = verb(1)),
(g['return'] = verb(2)),
typeof Symbol === 'function' &&
(g[Symbol.iterator] = function () {
return this;
}),
g
);
function verb(n) {
return function (v) {
return step([n, v]);
};
}
function step(op) {
if (f) throw new TypeError('Generator is already executing.');
while ((g && ((g = 0), op[0] && (_ = 0)), _))
try {
if (
((f = 1),
y &&
(t = op[0] & 2 ? y['return'] : op[0] ? y['throw'] || ((t = y['return']) && t.call(y), 0) : y.next) &&
!(t = t.call(y, op[1])).done)
)
return t;
if (((y = 0), t)) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0:
case 1:
t = op;
break;
case 4:
_.label++;
return { value: op[1], done: false };
case 5:
_.label++;
y = op[1];
op = [0];
continue;
case 7:
op = _.ops.pop();
_.trys.pop();
continue;
default:
if (!((t = _.trys), (t = t.length > 0 && t[t.length - 1])) && (op[0] === 6 || op[0] === 2)) {
_ = 0;
continue;
}
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) {
_.label = op[1];
break;
}
if (op[0] === 6 && _.label < t[1]) {
_.label = t[1];
t = op;
break;
}
if (t && _.label < t[2]) {
_.label = t[2];
_.ops.push(op);
break;
}
if (t[2]) _.ops.pop();
_.trys.pop();
continue;
}
op = body.call(thisArg, _);
} catch (e) {
op = [6, e];
y = 0;
} finally {
f = t = 0;
}
if (op[0] & 5) throw op[1];
return { value: op[0] ? op[1] : void 0, done: true };
}
};
Object.defineProperty(exports, '__esModule', { value: true });
exports.AudioFileTooLargeError = exports.WHISPER_MAX_FILE_BYTES = void 0;
exports.transcribeAudioFile = transcribeAudioFile;
var node_fs_1 = require('node:fs');
var promises_1 = require('node:fs/promises');
var openai_1 = require('openai');
/** OpenAI audio transcription file size limit (bytes). */
exports.WHISPER_MAX_FILE_BYTES = 25 * 1024 * 1024;
var AudioFileTooLargeError = /** @class */ (function (_super) {
__extends(AudioFileTooLargeError, _super);
function AudioFileTooLargeError(fileSizeBytes) {
var _this =
_super.call(
this,
'Audio file '.concat(fileSizeBytes, ' bytes exceeds Whisper limit of ').concat(exports.WHISPER_MAX_FILE_BYTES)
) || this;
_this.fileSizeBytes = fileSizeBytes;
_this.code = 'AUDIO_TOO_LARGE';
_this.name = 'AudioFileTooLargeError';
return _this;
}
return AudioFileTooLargeError;
})(Error);
exports.AudioFileTooLargeError = AudioFileTooLargeError;
/**
* Calls Whisper with verbose_json + segment timestamps. Omit `language` for
* auto-detection; the API returns the detected ISO 639-1 code.
*/
function transcribeAudioFile(params) {
return __awaiter(this, void 0, void 0, function () {
var fileStat, client, response, language, durationSeconds, fullText, rawSegments, segments;
var _a, _b, _c, _d, _e;
return __generator(this, function (_f) {
switch (_f.label) {
case 0:
return [4 /*yield*/, (0, promises_1.stat)(params.localPath)];
case 1:
fileStat = _f.sent();
if (fileStat.size > exports.WHISPER_MAX_FILE_BYTES) {
throw new AudioFileTooLargeError(fileStat.size);
}
client = new openai_1.default({ apiKey: params.apiKey });
return [
4 /*yield*/,
client.audio.transcriptions.create(
__assign(
{
file: (0, node_fs_1.createReadStream)(params.localPath),
model: 'whisper-1',
response_format: 'verbose_json',
timestamp_granularities: ['segment']
},
params.language ? { language: params.language } : {}
)
)
];
case 2:
response = _f.sent();
language = (_a = response.language) !== null && _a !== void 0 ? _a : 'und';
durationSeconds = (_b = response.duration) !== null && _b !== void 0 ? _b : 0;
fullText =
(_d = (_c = response.text) === null || _c === void 0 ? void 0 : _c.trim()) !== null && _d !== void 0
? _d
: '';
rawSegments = (_e = response.segments) !== null && _e !== void 0 ? _e : [];
segments = rawSegments.map(function (seg) {
return {
start: seg.start,
end: seg.end,
text: seg.text.trim()
};
});
return [
2 /*return*/,
{ language: language, durationSeconds: durationSeconds, segments: segments, fullText: fullText }
];
}
});
});
}
@@ -0,0 +1,67 @@
import { createReadStream } from 'node:fs';
import { stat } from 'node:fs/promises';
import OpenAI from 'openai';
/** OpenAI audio transcription file size limit (bytes). */
export const WHISPER_MAX_FILE_BYTES = 25 * 1024 * 1024;
export class AudioFileTooLargeError extends Error {
readonly code = 'AUDIO_TOO_LARGE';
constructor(readonly fileSizeBytes: number) {
super(`Audio file ${fileSizeBytes} bytes exceeds Whisper limit of ${WHISPER_MAX_FILE_BYTES}`);
this.name = 'AudioFileTooLargeError';
}
}
export interface TranscriptionSegment {
start: number;
end: number;
text: string;
}
export interface TranscribeAudioFileResult {
language: string;
durationSeconds: number;
segments: TranscriptionSegment[];
fullText: string;
}
/**
* Calls Whisper with verbose_json + segment timestamps. Omit `language` for
* auto-detection; the API returns the detected ISO 639-1 code.
*/
export async function transcribeAudioFile(params: {
apiKey: string;
localPath: string;
language?: string;
}): Promise<TranscribeAudioFileResult> {
const fileStat = await stat(params.localPath);
if (fileStat.size > WHISPER_MAX_FILE_BYTES) {
throw new AudioFileTooLargeError(fileStat.size);
}
const client = new OpenAI({ apiKey: params.apiKey });
const response = await client.audio.transcriptions.create({
file: createReadStream(params.localPath),
model: 'whisper-1',
response_format: 'verbose_json',
timestamp_granularities: ['segment'],
...(params.language ? { language: params.language } : {})
});
const language = response.language ?? 'und';
const durationSeconds = response.duration ?? 0;
const fullText = response.text?.trim() ?? '';
const rawSegments = response.segments ?? [];
const segments: TranscriptionSegment[] = rawSegments.map((seg) => ({
start: seg.start,
end: seg.end,
text: seg.text.trim()
}));
return { language, durationSeconds, segments, fullText };
}
@@ -0,0 +1,37 @@
'use strict';
Object.defineProperty(exports, '__esModule', { value: true });
exports.segmentsToWebVtt = segmentsToWebVtt;
function formatVttTime(seconds) {
if (!Number.isFinite(seconds) || seconds < 0) {
seconds = 0;
}
var totalMs = Math.round(seconds * 1000);
var hours = Math.floor(totalMs / 3600000);
var minutes = Math.floor((totalMs % 3600000) / 60000);
var secs = Math.floor((totalMs % 60000) / 1000);
var ms = totalMs % 1000;
var pad = function (n, width) {
return n.toString().padStart(width, '0');
};
return ''.concat(pad(hours, 2), ':').concat(pad(minutes, 2), ':').concat(pad(secs, 2), '.').concat(pad(ms, 3));
}
/**
* Build a minimal WEBVTT caption file from Whisper segment timestamps.
*/
function segmentsToWebVtt(segments) {
var lines = ['WEBVTT', ''];
var cueIndex = 1;
for (var _i = 0, segments_1 = segments; _i < segments_1.length; _i++) {
var segment = segments_1[_i];
var text = segment.text.trim();
if (!text) continue;
var start = formatVttTime(segment.start);
var end = formatVttTime(Math.max(segment.end, segment.start + 0.001));
lines.push(String(cueIndex));
lines.push(''.concat(start, ' --> ').concat(end));
lines.push(text);
lines.push('');
cueIndex++;
}
return lines.join('\n');
}
@@ -0,0 +1,41 @@
import type { TranscriptionSegment } from './openai';
function formatVttTime(seconds: number): string {
if (!Number.isFinite(seconds) || seconds < 0) {
seconds = 0;
}
const totalMs = Math.round(seconds * 1000);
const hours = Math.floor(totalMs / 3_600_000);
const minutes = Math.floor((totalMs % 3_600_000) / 60_000);
const secs = Math.floor((totalMs % 60_000) / 1000);
const ms = totalMs % 1000;
const pad = (n: number, width: number) => n.toString().padStart(width, '0');
return `${pad(hours, 2)}:${pad(minutes, 2)}:${pad(secs, 2)}.${pad(ms, 3)}`;
}
/**
* Build a minimal WEBVTT caption file from Whisper segment timestamps.
*/
export function segmentsToWebVtt(segments: TranscriptionSegment[]): string {
const lines = ['WEBVTT', ''];
let cueIndex = 1;
for (const segment of segments) {
const text = segment.text.trim();
if (!text) continue;
const start = formatVttTime(segment.start);
const end = formatVttTime(Math.max(segment.end, segment.start + 0.001));
lines.push(String(cueIndex));
lines.push(`${start} --> ${end}`);
lines.push(text);
lines.push('');
cueIndex++;
}
return lines.join('\n');
}
+214
View File
@@ -0,0 +1,214 @@
'use strict';
var __extends =
(this && this.__extends) ||
(function () {
var extendStatics = function (d, b) {
extendStatics =
Object.setPrototypeOf ||
({ __proto__: [] } instanceof Array &&
function (d, b) {
d.__proto__ = b;
}) ||
function (d, b) {
for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p];
};
return extendStatics(d, b);
};
return function (d, b) {
if (typeof b !== 'function' && b !== null)
throw new TypeError('Class extends value ' + String(b) + ' is not a constructor or null');
extendStatics(d, b);
function __() {
this.constructor = d;
}
d.prototype = b === null ? Object.create(b) : ((__.prototype = b.prototype), new __());
};
})();
var __awaiter =
(this && this.__awaiter) ||
function (thisArg, _arguments, P, generator) {
function adopt(value) {
return value instanceof P
? value
: new P(function (resolve) {
resolve(value);
});
}
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) {
try {
step(generator.next(value));
} catch (e) {
reject(e);
}
}
function rejected(value) {
try {
step(generator['throw'](value));
} catch (e) {
reject(e);
}
}
function step(result) {
result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected);
}
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator =
(this && this.__generator) ||
function (thisArg, body) {
var _ = {
label: 0,
sent: function () {
if (t[0] & 1) throw t[1];
return t[1];
},
trys: [],
ops: []
},
f,
y,
t,
g = Object.create((typeof Iterator === 'function' ? Iterator : Object).prototype);
return (
(g.next = verb(0)),
(g['throw'] = verb(1)),
(g['return'] = verb(2)),
typeof Symbol === 'function' &&
(g[Symbol.iterator] = function () {
return this;
}),
g
);
function verb(n) {
return function (v) {
return step([n, v]);
};
}
function step(op) {
if (f) throw new TypeError('Generator is already executing.');
while ((g && ((g = 0), op[0] && (_ = 0)), _))
try {
if (
((f = 1),
y &&
(t = op[0] & 2 ? y['return'] : op[0] ? y['throw'] || ((t = y['return']) && t.call(y), 0) : y.next) &&
!(t = t.call(y, op[1])).done)
)
return t;
if (((y = 0), t)) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0:
case 1:
t = op;
break;
case 4:
_.label++;
return { value: op[1], done: false };
case 5:
_.label++;
y = op[1];
op = [0];
continue;
case 7:
op = _.ops.pop();
_.trys.pop();
continue;
default:
if (!((t = _.trys), (t = t.length > 0 && t[t.length - 1])) && (op[0] === 6 || op[0] === 2)) {
_ = 0;
continue;
}
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) {
_.label = op[1];
break;
}
if (op[0] === 6 && _.label < t[1]) {
_.label = t[1];
t = op;
break;
}
if (t && _.label < t[2]) {
_.label = t[2];
_.ops.push(op);
break;
}
if (t[2]) _.ops.pop();
_.trys.pop();
continue;
}
op = body.call(thisArg, _);
} catch (e) {
op = [6, e];
y = 0;
} finally {
f = t = 0;
}
if (op[0] & 5) throw op[1];
return { value: op[0] ? op[1] : void 0, done: true };
}
};
Object.defineProperty(exports, '__esModule', { value: true });
exports.JobCanceledError = void 0;
exports.isMediaJobCancelRequested = isMediaJobCancelRequested;
exports.throwIfCancelRequested = throwIfCancelRequested;
exports.errorMessage = errorMessage;
var jobs_1 = require('@cio/db/queries/jobs');
var logger_1 = require('./logger');
/**
* Lightweight cooperative cancellation. Workers check `isCancelRequested` at
* step boundaries and inside long ffmpeg/fetch loops. The plan calls for
* `cancelRequestedAt` on each run table, with the worker abandoning further
* work the moment it observes a request.
*/
var JobCanceledError = /** @class */ (function (_super) {
__extends(JobCanceledError, _super);
function JobCanceledError(mediaJobId) {
var _this = _super.call(this, 'Media job '.concat(mediaJobId, ' canceled by user request')) || this;
_this.mediaJobId = mediaJobId;
_this.name = 'JobCanceledError';
return _this;
}
return JobCanceledError;
})(Error);
exports.JobCanceledError = JobCanceledError;
function isMediaJobCancelRequested(mediaJobId) {
return __awaiter(this, void 0, void 0, function () {
var job, error_1;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
_a.trys.push([0, 2, , 3]);
return [4 /*yield*/, (0, jobs_1.getMediaJobById)(mediaJobId)];
case 1:
job = _a.sent();
return [2 /*return*/, Boolean(job === null || job === void 0 ? void 0 : job.cancelRequestedAt)];
case 2:
error_1 = _a.sent();
logger_1.log.warn('cancel-check-failed', { mediaJobId: mediaJobId, error: errorMessage(error_1) });
return [2 /*return*/, false];
case 3:
return [2 /*return*/];
}
});
});
}
function throwIfCancelRequested(mediaJobId) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
return [4 /*yield*/, isMediaJobCancelRequested(mediaJobId)];
case 1:
if (_a.sent()) {
throw new JobCanceledError(mediaJobId);
}
return [2 /*return*/];
}
});
});
}
function errorMessage(error) {
return error instanceof Error ? error.message : String(error);
}
+37
View File
@@ -0,0 +1,37 @@
import { getMediaJobById } from '@cio/db/queries/jobs';
import { log } from './logger';
/**
* Lightweight cooperative cancellation. Workers check `isCancelRequested` at
* step boundaries and inside long ffmpeg/fetch loops. The plan calls for
* `cancelRequestedAt` on each run table, with the worker abandoning further
* work the moment it observes a request.
*/
export class JobCanceledError extends Error {
constructor(public readonly mediaJobId: string) {
super(`Media job ${mediaJobId} canceled by user request`);
this.name = 'JobCanceledError';
}
}
export async function isMediaJobCancelRequested(mediaJobId: string): Promise<boolean> {
try {
const job = await getMediaJobById(mediaJobId);
return Boolean(job?.cancelRequestedAt);
} catch (error) {
log.warn('cancel-check-failed', { mediaJobId, error: errorMessage(error) });
return false;
}
}
export async function throwIfCancelRequested(mediaJobId: string): Promise<void> {
if (await isMediaJobCancelRequested(mediaJobId)) {
throw new JobCanceledError(mediaJobId);
}
}
export function errorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
+250
View File
@@ -0,0 +1,250 @@
'use strict';
var __awaiter =
(this && this.__awaiter) ||
function (thisArg, _arguments, P, generator) {
function adopt(value) {
return value instanceof P
? value
: new P(function (resolve) {
resolve(value);
});
}
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) {
try {
step(generator.next(value));
} catch (e) {
reject(e);
}
}
function rejected(value) {
try {
step(generator['throw'](value));
} catch (e) {
reject(e);
}
}
function step(result) {
result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected);
}
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator =
(this && this.__generator) ||
function (thisArg, body) {
var _ = {
label: 0,
sent: function () {
if (t[0] & 1) throw t[1];
return t[1];
},
trys: [],
ops: []
},
f,
y,
t,
g = Object.create((typeof Iterator === 'function' ? Iterator : Object).prototype);
return (
(g.next = verb(0)),
(g['throw'] = verb(1)),
(g['return'] = verb(2)),
typeof Symbol === 'function' &&
(g[Symbol.iterator] = function () {
return this;
}),
g
);
function verb(n) {
return function (v) {
return step([n, v]);
};
}
function step(op) {
if (f) throw new TypeError('Generator is already executing.');
while ((g && ((g = 0), op[0] && (_ = 0)), _))
try {
if (
((f = 1),
y &&
(t = op[0] & 2 ? y['return'] : op[0] ? y['throw'] || ((t = y['return']) && t.call(y), 0) : y.next) &&
!(t = t.call(y, op[1])).done)
)
return t;
if (((y = 0), t)) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0:
case 1:
t = op;
break;
case 4:
_.label++;
return { value: op[1], done: false };
case 5:
_.label++;
y = op[1];
op = [0];
continue;
case 7:
op = _.ops.pop();
_.trys.pop();
continue;
default:
if (!((t = _.trys), (t = t.length > 0 && t[t.length - 1])) && (op[0] === 6 || op[0] === 2)) {
_ = 0;
continue;
}
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) {
_.label = op[1];
break;
}
if (op[0] === 6 && _.label < t[1]) {
_.label = t[1];
t = op;
break;
}
if (t && _.label < t[2]) {
_.label = t[2];
_.ops.push(op);
break;
}
if (t[2]) _.ops.pop();
_.trys.pop();
continue;
}
op = body.call(thisArg, _);
} catch (e) {
op = [6, e];
y = 0;
} finally {
f = t = 0;
}
if (op[0] & 5) throw op[1];
return { value: op[0] ? op[1] : void 0, done: true };
}
};
var _a, _b;
Object.defineProperty(exports, '__esModule', { value: true });
exports.FFPROBE_BIN = exports.FFMPEG_BIN = void 0;
exports.ffprobeJson = ffprobeJson;
exports.ffmpegRun = ffmpegRun;
exports.warnIfFfmpegMissing = warnIfFfmpegMissing;
var node_child_process_1 = require('node:child_process');
var node_util_1 = require('node:util');
var logger_1 = require('./logger');
var execFileAsync = (0, node_util_1.promisify)(node_child_process_1.execFile);
/**
* Resolved at module load. Set `FFMPEG_PATH` / `FFPROBE_PATH` to point at
* non-default binary locations (e.g. when ffmpeg lives outside PATH for the
* worker process). When unset we fall through to `ffmpeg` / `ffprobe` on
* PATH, which is what most installs provide.
*/
exports.FFMPEG_BIN = (_a = process.env.FFMPEG_PATH) !== null && _a !== void 0 ? _a : 'ffmpeg';
exports.FFPROBE_BIN = (_b = process.env.FFPROBE_PATH) !== null && _b !== void 0 ? _b : 'ffprobe';
/** Cap subprocess stdout/stderr so a runaway log can't blow up the worker. */
var MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
/**
* Run `ffprobe -print_format json -show_format -show_streams <file>` and
* return the parsed JSON. Mirrors the shape we used to get from
* `fluent-ffmpeg.ffprobe()` but without the deprecated wrapper.
*/
function ffprobeJson(localPath) {
return __awaiter(this, void 0, void 0, function () {
var args, stdout;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
args = ['-v', 'error', '-print_format', 'json', '-show_format', '-show_streams', localPath];
return [4 /*yield*/, execFileAsync(exports.FFPROBE_BIN, args, { maxBuffer: MAX_OUTPUT_BYTES })];
case 1:
stdout = _a.sent().stdout;
return [2 /*return*/, JSON.parse(stdout)];
}
});
});
}
/**
* Spawn ffmpeg with the supplied argv. Captures stderr and surfaces a useful
* tail in the error message when ffmpeg exits non-zero, since ffmpeg writes
* progress and diagnostics to stderr by default.
*/
function ffmpegRun(args) {
return __awaiter(this, void 0, void 0, function () {
var error_1, stderr, tail;
var _a, _b;
return __generator(this, function (_c) {
switch (_c.label) {
case 0:
_c.trys.push([0, 2, , 3]);
return [4 /*yield*/, execFileAsync(exports.FFMPEG_BIN, args, { maxBuffer: MAX_OUTPUT_BYTES })];
case 1:
_c.sent();
return [3 /*break*/, 3];
case 2:
error_1 = _c.sent();
stderr =
(_b =
(_a = error_1 === null || error_1 === void 0 ? void 0 : error_1.stderr) === null || _a === void 0
? void 0
: _a.toString('utf8')) !== null && _b !== void 0
? _b
: '';
tail = stderr.split('\n').filter(Boolean).slice(-5).join(' | ');
throw new Error('ffmpeg exited with error'.concat(tail ? ': '.concat(tail) : ''), { cause: error_1 });
case 3:
return [2 /*return*/];
}
});
});
}
/**
* Probe for the binaries at worker startup. Logs a warning when missing so
* media jobs surface a clearer failure than a raw ENOENT from the first
* spawn. Non-media workers (emails, maintenance) keep booting either way.
*/
function warnIfFfmpegMissing() {
return __awaiter(this, void 0, void 0, function () {
var checks, missing;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
return [4 /*yield*/, Promise.all([isExecutable(exports.FFMPEG_BIN), isExecutable(exports.FFPROBE_BIN)])];
case 1:
checks = _a.sent();
missing = [];
if (!checks[0]) missing.push(exports.FFMPEG_BIN);
if (!checks[1]) missing.push(exports.FFPROBE_BIN);
if (missing.length === 0) {
logger_1.log.info('ffmpeg-binaries-resolved', { ffmpeg: exports.FFMPEG_BIN, ffprobe: exports.FFPROBE_BIN });
return [2 /*return*/];
}
logger_1.log.warn('ffmpeg-binaries-missing', {
missing: missing,
hint: 'Install ffmpeg (macOS: brew install ffmpeg, Debian/Ubuntu: apt install ffmpeg) or set FFMPEG_PATH / FFPROBE_PATH. Media jobs will fail until this is resolved; other workers are unaffected.'
});
return [2 /*return*/];
}
});
});
}
function isExecutable(bin) {
return __awaiter(this, void 0, void 0, function () {
var _a;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
_b.trys.push([0, 2, , 3]);
return [4 /*yield*/, execFileAsync(bin, ['-version'], { maxBuffer: MAX_OUTPUT_BYTES })];
case 1:
_b.sent();
return [2 /*return*/, true];
case 2:
_a = _b.sent();
return [2 /*return*/, false];
case 3:
return [2 /*return*/];
}
});
});
}
+95
View File
@@ -0,0 +1,95 @@
import { execFile } from 'node:child_process';
import { promisify } from 'node:util';
import { log } from './logger';
const execFileAsync = promisify(execFile);
/**
* Resolved at module load. Set `FFMPEG_PATH` / `FFPROBE_PATH` to point at
* non-default binary locations (e.g. when ffmpeg lives outside PATH for the
* worker process). When unset we fall through to `ffmpeg` / `ffprobe` on
* PATH, which is what most installs provide.
*/
export const FFMPEG_BIN = process.env.FFMPEG_PATH ?? 'ffmpeg';
export const FFPROBE_BIN = process.env.FFPROBE_PATH ?? 'ffprobe';
/** Cap subprocess stdout/stderr so a runaway log can't blow up the worker. */
const MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
export interface FfprobeStream {
codec_type?: 'video' | 'audio' | 'subtitle' | 'data';
codec_name?: string;
width?: number;
height?: number;
duration?: string;
}
export interface FfprobeFormat {
duration?: string;
size?: string;
bit_rate?: string;
}
export interface FfprobeData {
streams: FfprobeStream[];
format: FfprobeFormat;
}
/**
* Run `ffprobe -print_format json -show_format -show_streams <file>` and
* return the parsed JSON. Mirrors the shape we used to get from
* `fluent-ffmpeg.ffprobe()` but without the deprecated wrapper.
*/
export async function ffprobeJson(localPath: string): Promise<FfprobeData> {
const args = ['-v', 'error', '-print_format', 'json', '-show_format', '-show_streams', localPath];
const { stdout } = await execFileAsync(FFPROBE_BIN, args, { maxBuffer: MAX_OUTPUT_BYTES });
return JSON.parse(stdout) as FfprobeData;
}
/**
* Spawn ffmpeg with the supplied argv. Captures stderr and surfaces a useful
* tail in the error message when ffmpeg exits non-zero, since ffmpeg writes
* progress and diagnostics to stderr by default.
*/
export async function ffmpegRun(args: string[]): Promise<void> {
try {
await execFileAsync(FFMPEG_BIN, args, { maxBuffer: MAX_OUTPUT_BYTES });
} catch (error) {
const stderr = (error as { stderr?: string | Buffer })?.stderr?.toString('utf8') ?? '';
const tail = stderr.split('\n').filter(Boolean).slice(-5).join(' | ');
throw new Error(`ffmpeg exited with error${tail ? `: ${tail}` : ''}`, { cause: error as Error });
}
}
/**
* Probe for the binaries at worker startup. Logs a warning when missing so
* media jobs surface a clearer failure than a raw ENOENT from the first
* spawn. Non-media workers (emails, maintenance) keep booting either way.
*/
export async function warnIfFfmpegMissing(): Promise<void> {
const checks = await Promise.all([isExecutable(FFMPEG_BIN), isExecutable(FFPROBE_BIN)]);
const missing: string[] = [];
if (!checks[0]) missing.push(FFMPEG_BIN);
if (!checks[1]) missing.push(FFPROBE_BIN);
if (missing.length === 0) {
log.info('ffmpeg-binaries-resolved', { ffmpeg: FFMPEG_BIN, ffprobe: FFPROBE_BIN });
return;
}
log.warn('ffmpeg-binaries-missing', {
missing,
hint: 'Install ffmpeg (macOS: brew install ffmpeg, Debian/Ubuntu: apt install ffmpeg) or set FFMPEG_PATH / FFPROBE_PATH. Media jobs will fail until this is resolved; other workers are unaffected.'
});
}
async function isExecutable(bin: string): Promise<boolean> {
try {
await execFileAsync(bin, ['-version'], { maxBuffer: MAX_OUTPUT_BYTES });
return true;
} catch {
return false;
}
}
+39
View File
@@ -0,0 +1,39 @@
'use strict';
var __assign =
(this && this.__assign) ||
function () {
__assign =
Object.assign ||
function (t) {
for (var s, i = 1, n = arguments.length; i < n; i++) {
s = arguments[i];
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) t[p] = s[p];
}
return t;
};
return __assign.apply(this, arguments);
};
Object.defineProperty(exports, '__esModule', { value: true });
exports.log = void 0;
function emit(level, message, fields) {
if (fields === void 0) {
fields = {};
}
var payload = __assign({ ts: new Date().toISOString(), level: level, msg: message }, fields);
var stream = level === 'error' || level === 'warn' ? console.error : console.log;
stream(JSON.stringify(payload));
}
exports.log = {
debug: function (msg, fields) {
return emit('debug', msg, fields);
},
info: function (msg, fields) {
return emit('info', msg, fields);
},
warn: function (msg, fields) {
return emit('warn', msg, fields);
},
error: function (msg, fields) {
return emit('error', msg, fields);
}
};
+28
View File
@@ -0,0 +1,28 @@
/**
* Tiny structured logger for the worker. Keeps log output greppable across
* processors without pulling in a full logger dependency.
*/
type LogLevel = 'debug' | 'info' | 'warn' | 'error';
interface LogFields {
[key: string]: unknown;
}
function emit(level: LogLevel, message: string, fields: LogFields = {}): void {
const payload = {
ts: new Date().toISOString(),
level,
msg: message,
...fields
};
const stream = level === 'error' || level === 'warn' ? console.error : console.log;
stream(JSON.stringify(payload));
}
export const log = {
debug: (msg: string, fields?: LogFields) => emit('debug', msg, fields),
info: (msg: string, fields?: LogFields) => emit('info', msg, fields),
warn: (msg: string, fields?: LogFields) => emit('warn', msg, fields),
error: (msg: string, fields?: LogFields) => emit('error', msg, fields)
};
+266
View File
@@ -0,0 +1,266 @@
'use strict';
var __awaiter =
(this && this.__awaiter) ||
function (thisArg, _arguments, P, generator) {
function adopt(value) {
return value instanceof P
? value
: new P(function (resolve) {
resolve(value);
});
}
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) {
try {
step(generator.next(value));
} catch (e) {
reject(e);
}
}
function rejected(value) {
try {
step(generator['throw'](value));
} catch (e) {
reject(e);
}
}
function step(result) {
result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected);
}
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator =
(this && this.__generator) ||
function (thisArg, body) {
var _ = {
label: 0,
sent: function () {
if (t[0] & 1) throw t[1];
return t[1];
},
trys: [],
ops: []
},
f,
y,
t,
g = Object.create((typeof Iterator === 'function' ? Iterator : Object).prototype);
return (
(g.next = verb(0)),
(g['throw'] = verb(1)),
(g['return'] = verb(2)),
typeof Symbol === 'function' &&
(g[Symbol.iterator] = function () {
return this;
}),
g
);
function verb(n) {
return function (v) {
return step([n, v]);
};
}
function step(op) {
if (f) throw new TypeError('Generator is already executing.');
while ((g && ((g = 0), op[0] && (_ = 0)), _))
try {
if (
((f = 1),
y &&
(t = op[0] & 2 ? y['return'] : op[0] ? y['throw'] || ((t = y['return']) && t.call(y), 0) : y.next) &&
!(t = t.call(y, op[1])).done)
)
return t;
if (((y = 0), t)) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0:
case 1:
t = op;
break;
case 4:
_.label++;
return { value: op[1], done: false };
case 5:
_.label++;
y = op[1];
op = [0];
continue;
case 7:
op = _.ops.pop();
_.trys.pop();
continue;
default:
if (!((t = _.trys), (t = t.length > 0 && t[t.length - 1])) && (op[0] === 6 || op[0] === 2)) {
_ = 0;
continue;
}
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) {
_.label = op[1];
break;
}
if (op[0] === 6 && _.label < t[1]) {
_.label = t[1];
t = op;
break;
}
if (t && _.label < t[2]) {
_.label = t[2];
_.ops.push(op);
break;
}
if (t[2]) _.ops.pop();
_.trys.pop();
continue;
}
op = body.call(thisArg, _);
} catch (e) {
op = [6, e];
y = 0;
} finally {
f = t = 0;
}
if (op[0] & 5) throw op[1];
return { value: op[0] ? op[1] : void 0, done: true };
}
};
Object.defineProperty(exports, '__esModule', { value: true });
exports.downloadObjectToTempFile = downloadObjectToTempFile;
exports.uploadFileToBucket = uploadFileToBucket;
exports.uploadBufferToBucket = uploadBufferToBucket;
exports.safeUnlink = safeUnlink;
exports.videosBucket = videosBucket;
exports.mediaBucket = mediaBucket;
var client_s3_1 = require('@aws-sdk/client-s3');
var node_fs_1 = require('node:fs');
var promises_1 = require('node:fs/promises');
var node_os_1 = require('node:os');
var node_path_1 = require('node:path');
var promises_2 = require('node:stream/promises');
var storage_1 = require('../config/storage');
/**
* Streams an S3 object to a temp file on disk and returns the absolute path.
* Workers prefer disk over buffers so ffmpeg can seek without holding the
* whole video in RAM.
*/
function downloadObjectToTempFile(bucket, key, fileNameHint) {
return __awaiter(this, void 0, void 0, function () {
var dir, safeName, localPath, response, body;
var _a;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
dir = node_path_1.default.join((0, node_os_1.tmpdir)(), 'cio-jobs');
return [4 /*yield*/, (0, promises_1.mkdir)(dir, { recursive: true })];
case 1:
_b.sent();
safeName =
(_a =
fileNameHint === null || fileNameHint === void 0
? void 0
: fileNameHint.replace(/[^a-zA-Z0-9._-]/g, '_')) !== null && _a !== void 0
? _a
: 'media';
localPath = node_path_1.default.join(
dir,
''.concat(Date.now(), '_').concat(process.pid, '_').concat(safeName)
);
return [
4 /*yield*/,
(0, storage_1.getS3Client)().send(new client_s3_1.GetObjectCommand({ Bucket: bucket, Key: key }))
];
case 2:
response = _b.sent();
if (!response.Body) {
throw new Error('S3 object '.concat(bucket, '/').concat(key, ' returned no body'));
}
body = response.Body;
return [4 /*yield*/, (0, promises_2.pipeline)(body, (0, node_fs_1.createWriteStream)(localPath))];
case 3:
_b.sent();
return [2 /*return*/, localPath];
}
});
});
}
function uploadFileToBucket(bucket, key, localPath, contentType, cacheControl) {
return __awaiter(this, void 0, void 0, function () {
var buffer;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
return [4 /*yield*/, (0, promises_1.readFile)(localPath)];
case 1:
buffer = _a.sent();
return [
4 /*yield*/,
(0, storage_1.getS3Client)().send(
new client_s3_1.PutObjectCommand({
Bucket: bucket,
Key: key,
Body: buffer,
ContentType: contentType,
CacheControl: cacheControl
})
)
];
case 2:
_a.sent();
return [2 /*return*/];
}
});
});
}
function uploadBufferToBucket(bucket, key, buffer, contentType, cacheControl) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
return [
4 /*yield*/,
(0, storage_1.getS3Client)().send(
new client_s3_1.PutObjectCommand({
Bucket: bucket,
Key: key,
Body: buffer,
ContentType: contentType,
CacheControl: cacheControl
})
)
];
case 1:
_a.sent();
return [2 /*return*/];
}
});
});
}
function safeUnlink(filePath) {
return __awaiter(this, void 0, void 0, function () {
var _a;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
if (!filePath) return [2 /*return*/];
_b.label = 1;
case 1:
_b.trys.push([1, 3, , 4]);
return [4 /*yield*/, (0, promises_1.unlink)(filePath)];
case 2:
_b.sent();
return [3 /*break*/, 4];
case 3:
_a = _b.sent();
return [3 /*break*/, 4];
case 4:
return [2 /*return*/];
}
});
});
}
function videosBucket() {
return (0, storage_1.getStorageConfig)().bucketVideos;
}
function mediaBucket() {
return (0, storage_1.getStorageConfig)().bucketMedia;
}
+87
View File
@@ -0,0 +1,87 @@
import { GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3';
import { Readable } from 'node:stream';
import { createWriteStream } from 'node:fs';
import { mkdir, readFile, unlink } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { pipeline } from 'node:stream/promises';
import { getS3Client, getStorageConfig } from '../config/storage';
/**
* Streams an S3 object to a temp file on disk and returns the absolute path.
* Workers prefer disk over buffers so ffmpeg can seek without holding the
* whole video in RAM.
*/
export async function downloadObjectToTempFile(bucket: string, key: string, fileNameHint?: string): Promise<string> {
const dir = path.join(tmpdir(), 'cio-jobs');
await mkdir(dir, { recursive: true });
const safeName = fileNameHint?.replace(/[^a-zA-Z0-9._-]/g, '_') ?? 'media';
const localPath = path.join(dir, `${Date.now()}_${process.pid}_${safeName}`);
const response = await getS3Client().send(new GetObjectCommand({ Bucket: bucket, Key: key }));
if (!response.Body) {
throw new Error(`S3 object ${bucket}/${key} returned no body`);
}
const body = response.Body as Readable;
await pipeline(body, createWriteStream(localPath));
return localPath;
}
export async function uploadFileToBucket(
bucket: string,
key: string,
localPath: string,
contentType: string,
cacheControl?: string
): Promise<void> {
const buffer = await readFile(localPath);
await getS3Client().send(
new PutObjectCommand({
Bucket: bucket,
Key: key,
Body: buffer,
ContentType: contentType,
CacheControl: cacheControl
})
);
}
export async function uploadBufferToBucket(
bucket: string,
key: string,
buffer: Buffer,
contentType: string,
cacheControl?: string
): Promise<void> {
await getS3Client().send(
new PutObjectCommand({
Bucket: bucket,
Key: key,
Body: buffer,
ContentType: contentType,
CacheControl: cacheControl
})
);
}
export async function safeUnlink(filePath: string | undefined | null): Promise<void> {
if (!filePath) return;
try {
await unlink(filePath);
} catch {
// best-effort cleanup
}
}
export function videosBucket(): string {
return getStorageConfig().bucketVideos;
}
export function mediaBucket(): string {
return getStorageConfig().bucketMedia;
}
+232
View File
@@ -0,0 +1,232 @@
'use strict';
var __awaiter =
(this && this.__awaiter) ||
function (thisArg, _arguments, P, generator) {
function adopt(value) {
return value instanceof P
? value
: new P(function (resolve) {
resolve(value);
});
}
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) {
try {
step(generator.next(value));
} catch (e) {
reject(e);
}
}
function rejected(value) {
try {
step(generator['throw'](value));
} catch (e) {
reject(e);
}
}
function step(result) {
result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected);
}
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator =
(this && this.__generator) ||
function (thisArg, body) {
var _ = {
label: 0,
sent: function () {
if (t[0] & 1) throw t[1];
return t[1];
},
trys: [],
ops: []
},
f,
y,
t,
g = Object.create((typeof Iterator === 'function' ? Iterator : Object).prototype);
return (
(g.next = verb(0)),
(g['throw'] = verb(1)),
(g['return'] = verb(2)),
typeof Symbol === 'function' &&
(g[Symbol.iterator] = function () {
return this;
}),
g
);
function verb(n) {
return function (v) {
return step([n, v]);
};
}
function step(op) {
if (f) throw new TypeError('Generator is already executing.');
while ((g && ((g = 0), op[0] && (_ = 0)), _))
try {
if (
((f = 1),
y &&
(t = op[0] & 2 ? y['return'] : op[0] ? y['throw'] || ((t = y['return']) && t.call(y), 0) : y.next) &&
!(t = t.call(y, op[1])).done)
)
return t;
if (((y = 0), t)) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0:
case 1:
t = op;
break;
case 4:
_.label++;
return { value: op[1], done: false };
case 5:
_.label++;
y = op[1];
op = [0];
continue;
case 7:
op = _.ops.pop();
_.trys.pop();
continue;
default:
if (!((t = _.trys), (t = t.length > 0 && t[t.length - 1])) && (op[0] === 6 || op[0] === 2)) {
_ = 0;
continue;
}
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) {
_.label = op[1];
break;
}
if (op[0] === 6 && _.label < t[1]) {
_.label = t[1];
t = op;
break;
}
if (t && _.label < t[2]) {
_.label = t[2];
_.ops.push(op);
break;
}
if (t[2]) _.ops.pop();
_.trys.pop();
continue;
}
op = body.call(thisArg, _);
} catch (e) {
op = [6, e];
y = 0;
} finally {
f = t = 0;
}
if (op[0] & 5) throw op[1];
return { value: op[0] ? op[1] : void 0, done: true };
}
};
var _a;
Object.defineProperty(exports, '__esModule', { value: true });
require('./../bootstrap');
var bullmq_1 = require('bullmq');
var queries_1 = require('@cio/db/queries');
var jobs_1 = require('@cio/jobs');
var cancel_1 = require('../utils/cancel');
var env_1 = require('../config/env');
var logger_1 = require('../utils/logger');
var emails_1 = require('../processors/emails');
var concurrency =
Number.parseInt((_a = env_1.env.EMAIL_WORKER_CONCURRENCY) !== null && _a !== void 0 ? _a : '5', 10) || 5;
var connection = (0, jobs_1.createRedisConnection)();
var worker = new bullmq_1.Worker(
jobs_1.QUEUE_NAMES.emails,
function (job) {
return __awaiter(void 0, void 0, void 0, function () {
return __generator(this, function (_a) {
logger_1.log.info('email-job-start', {
jobName: job.name,
bullmqJobId: job.id,
attempt: job.attemptsMade + 1
});
switch (job.name) {
case jobs_1.JOB_NAMES.emails.send:
return [2 /*return*/, (0, emails_1.processSendEmail)(job.data)];
default:
throw new Error('Unknown emails job: '.concat(job.name));
}
return [2 /*return*/];
});
});
},
{ connection: connection, concurrency: concurrency }
);
worker.on('failed', function (job, err) {
return __awaiter(void 0, void 0, void 0, function () {
var isFinalAttempt;
var _a, _b;
return __generator(this, function (_c) {
switch (_c.label) {
case 0:
if (!job) return [2 /*return*/];
logger_1.log.error('email-job-failed', {
jobName: job.name,
bullmqJobId: job.id,
attempt: job.attemptsMade,
error: (0, cancel_1.errorMessage)(err)
});
isFinalAttempt = job.attemptsMade >= ((_a = job.opts.attempts) !== null && _a !== void 0 ? _a : 1);
if (!isFinalAttempt) return [2 /*return*/];
// BullMQ exhausted retries — write a dead-letter row so an operator can
// inspect the payload + error and replay if needed.
return [
4 /*yield*/,
(0, queries_1.recordDeadLetterJob)({
organizationId: null,
domain: 'emails',
runId: null,
queueName: jobs_1.QUEUE_NAMES.emails,
jobName: job.name,
bullmqJobId: (_b = job.id) !== null && _b !== void 0 ? _b : null,
payload: job.data,
error: { code: 'WORKER_EXHAUSTED_RETRIES', message: (0, cancel_1.errorMessage)(err), stack: err.stack },
attempts: job.attemptsMade
})
];
case 1:
// BullMQ exhausted retries — write a dead-letter row so an operator can
// inspect the payload + error and replay if needed.
_c.sent();
return [2 /*return*/];
}
});
});
});
worker.on('ready', function () {
return logger_1.log.info('email-worker-ready', { concurrency: concurrency, queue: jobs_1.QUEUE_NAMES.emails });
});
worker.on('error', function (err) {
return logger_1.log.error('email-worker-error', { error: (0, cancel_1.errorMessage)(err) });
});
var shutdown = function (signal) {
return __awaiter(void 0, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
logger_1.log.info('email-worker-shutdown', { signal: signal });
return [4 /*yield*/, worker.close()];
case 1:
_a.sent();
return [4 /*yield*/, connection.quit()];
case 2:
_a.sent();
process.exit(0);
return [2 /*return*/];
}
});
});
};
process.on('SIGTERM', function () {
return void shutdown('SIGTERM');
});
process.on('SIGINT', function () {
return void shutdown('SIGINT');
});
+74
View File
@@ -0,0 +1,74 @@
import './../bootstrap';
import { Worker } from 'bullmq';
import { recordDeadLetterJob } from '@cio/db/queries';
import { JOB_NAMES, QUEUE_NAMES, createRedisConnection } from '@cio/jobs';
import { errorMessage } from '../utils/cancel';
import { env } from '../config/env';
import { log } from '../utils/logger';
import { processSendEmail } from '../processors/emails';
const concurrency = Number.parseInt(env.EMAIL_WORKER_CONCURRENCY ?? '5', 10) || 5;
const connection = createRedisConnection();
const worker = new Worker(
QUEUE_NAMES.emails,
async (job) => {
log.info('email-job-start', {
jobName: job.name,
bullmqJobId: job.id,
attempt: job.attemptsMade + 1
});
switch (job.name) {
case JOB_NAMES.emails.send:
return processSendEmail(job.data);
default:
throw new Error(`Unknown emails job: ${job.name}`);
}
},
{ connection, concurrency }
);
worker.on('failed', async (job, err) => {
if (!job) return;
log.error('email-job-failed', {
jobName: job.name,
bullmqJobId: job.id,
attempt: job.attemptsMade,
error: errorMessage(err)
});
const isFinalAttempt = job.attemptsMade >= (job.opts.attempts ?? 1);
if (!isFinalAttempt) return;
// BullMQ exhausted retries — write a dead-letter row so an operator can
// inspect the payload + error and replay if needed.
await recordDeadLetterJob({
organizationId: null,
domain: 'emails',
runId: null,
queueName: QUEUE_NAMES.emails,
jobName: job.name,
bullmqJobId: job.id ?? null,
payload: job.data as Record<string, unknown>,
error: { code: 'WORKER_EXHAUSTED_RETRIES', message: errorMessage(err), stack: err.stack },
attempts: job.attemptsMade
});
});
worker.on('ready', () => log.info('email-worker-ready', { concurrency, queue: QUEUE_NAMES.emails }));
worker.on('error', (err) => log.error('email-worker-error', { error: errorMessage(err) }));
const shutdown = async (signal: string) => {
log.info('email-worker-shutdown', { signal });
await worker.close();
await connection.quit();
process.exit(0);
};
process.on('SIGTERM', () => void shutdown('SIGTERM'));
process.on('SIGINT', () => void shutdown('SIGINT'));
+203
View File
@@ -0,0 +1,203 @@
'use strict';
var __awaiter =
(this && this.__awaiter) ||
function (thisArg, _arguments, P, generator) {
function adopt(value) {
return value instanceof P
? value
: new P(function (resolve) {
resolve(value);
});
}
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) {
try {
step(generator.next(value));
} catch (e) {
reject(e);
}
}
function rejected(value) {
try {
step(generator['throw'](value));
} catch (e) {
reject(e);
}
}
function step(result) {
result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected);
}
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator =
(this && this.__generator) ||
function (thisArg, body) {
var _ = {
label: 0,
sent: function () {
if (t[0] & 1) throw t[1];
return t[1];
},
trys: [],
ops: []
},
f,
y,
t,
g = Object.create((typeof Iterator === 'function' ? Iterator : Object).prototype);
return (
(g.next = verb(0)),
(g['throw'] = verb(1)),
(g['return'] = verb(2)),
typeof Symbol === 'function' &&
(g[Symbol.iterator] = function () {
return this;
}),
g
);
function verb(n) {
return function (v) {
return step([n, v]);
};
}
function step(op) {
if (f) throw new TypeError('Generator is already executing.');
while ((g && ((g = 0), op[0] && (_ = 0)), _))
try {
if (
((f = 1),
y &&
(t = op[0] & 2 ? y['return'] : op[0] ? y['throw'] || ((t = y['return']) && t.call(y), 0) : y.next) &&
!(t = t.call(y, op[1])).done)
)
return t;
if (((y = 0), t)) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0:
case 1:
t = op;
break;
case 4:
_.label++;
return { value: op[1], done: false };
case 5:
_.label++;
y = op[1];
op = [0];
continue;
case 7:
op = _.ops.pop();
_.trys.pop();
continue;
default:
if (!((t = _.trys), (t = t.length > 0 && t[t.length - 1])) && (op[0] === 6 || op[0] === 2)) {
_ = 0;
continue;
}
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) {
_.label = op[1];
break;
}
if (op[0] === 6 && _.label < t[1]) {
_.label = t[1];
t = op;
break;
}
if (t && _.label < t[2]) {
_.label = t[2];
_.ops.push(op);
break;
}
if (t[2]) _.ops.pop();
_.trys.pop();
continue;
}
op = body.call(thisArg, _);
} catch (e) {
op = [6, e];
y = 0;
} finally {
f = t = 0;
}
if (op[0] & 5) throw op[1];
return { value: op[0] ? op[1] : void 0, done: true };
}
};
Object.defineProperty(exports, '__esModule', { value: true });
require('./../bootstrap');
var bullmq_1 = require('bullmq');
var queries_1 = require('@cio/db/queries');
var jobs_1 = require('@cio/jobs');
var cancel_1 = require('../utils/cancel');
var logger_1 = require('../utils/logger');
var connection = (0, jobs_1.createRedisConnection)();
var worker = new bullmq_1.Worker(
jobs_1.QUEUE_NAMES.maintenance,
function (job) {
return __awaiter(void 0, void 0, void 0, function () {
var data, cutoff, deleted;
var _a, _b;
return __generator(this, function (_c) {
switch (_c.label) {
case 0:
if (!(job.name === jobs_1.JOB_NAMES.maintenance.deadLetterCleanup)) return [3 /*break*/, 2];
data = jobs_1.ZDeadLetterCleanupPayload.parse((_a = job.data) !== null && _a !== void 0 ? _a : {});
cutoff = new Date(Date.now() - data.olderThanDays * 86400 * 1000).toISOString();
return [4 /*yield*/, (0, queries_1.pruneDeadLetterJobsOlderThan)(cutoff)];
case 1:
deleted = _c.sent();
logger_1.log.info('dead-letter-cleanup-done', { deleted: deleted, cutoff: cutoff });
return [2 /*return*/, { deleted: deleted }];
case 2:
if (job.name === jobs_1.JOB_NAMES.maintenance.retentionCompact) {
jobs_1.ZRetentionCompactPayload.parse((_b = job.data) !== null && _b !== void 0 ? _b : {});
// Retention windows for media_job, job_step, email_delivery to be wired
// alongside their respective queries. Logged for now so the schedule
// is observable without silently dropping data.
logger_1.log.info('retention-compact-noop', { reason: 'not-yet-implemented' });
return [2 /*return*/, { compacted: 0 }];
}
throw new Error('Unknown maintenance job: '.concat(job.name));
}
});
});
},
{ connection: connection, concurrency: 1 }
);
worker.on('ready', function () {
return logger_1.log.info('maintenance-worker-ready');
});
worker.on('failed', function (job, err) {
return logger_1.log.error('maintenance-job-failed', {
jobName: job === null || job === void 0 ? void 0 : job.name,
error: (0, cancel_1.errorMessage)(err)
});
});
worker.on('error', function (err) {
return logger_1.log.error('maintenance-worker-error', { error: (0, cancel_1.errorMessage)(err) });
});
var shutdown = function (signal) {
return __awaiter(void 0, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
logger_1.log.info('maintenance-worker-shutdown', { signal: signal });
return [4 /*yield*/, worker.close()];
case 1:
_a.sent();
return [4 /*yield*/, connection.quit()];
case 2:
_a.sent();
process.exit(0);
return [2 /*return*/];
}
});
});
};
process.on('SIGTERM', function () {
return void shutdown('SIGTERM');
});
process.on('SIGINT', function () {
return void shutdown('SIGINT');
});
+58
View File
@@ -0,0 +1,58 @@
import './../bootstrap';
import { Worker } from 'bullmq';
import { pruneDeadLetterJobsOlderThan } from '@cio/db/queries';
import {
JOB_NAMES,
QUEUE_NAMES,
ZDeadLetterCleanupPayload,
ZRetentionCompactPayload,
createRedisConnection
} from '@cio/jobs';
import { errorMessage } from '../utils/cancel';
import { log } from '../utils/logger';
const connection = createRedisConnection();
const worker = new Worker(
QUEUE_NAMES.maintenance,
async (job) => {
if (job.name === JOB_NAMES.maintenance.deadLetterCleanup) {
const data = ZDeadLetterCleanupPayload.parse(job.data ?? {});
const cutoff = new Date(Date.now() - data.olderThanDays * 86_400 * 1_000).toISOString();
const deleted = await pruneDeadLetterJobsOlderThan(cutoff);
log.info('dead-letter-cleanup-done', { deleted, cutoff });
return { deleted };
}
if (job.name === JOB_NAMES.maintenance.retentionCompact) {
ZRetentionCompactPayload.parse(job.data ?? {});
// Retention windows for media_job, job_step, email_delivery to be wired
// alongside their respective queries. Logged for now so the schedule
// is observable without silently dropping data.
log.info('retention-compact-noop', { reason: 'not-yet-implemented' });
return { compacted: 0 };
}
throw new Error(`Unknown maintenance job: ${job.name}`);
},
{ connection, concurrency: 1 }
);
worker.on('ready', () => log.info('maintenance-worker-ready'));
worker.on('failed', (job, err) =>
log.error('maintenance-job-failed', { jobName: job?.name, error: errorMessage(err) })
);
worker.on('error', (err) => log.error('maintenance-worker-error', { error: errorMessage(err) }));
const shutdown = async (signal: string) => {
log.info('maintenance-worker-shutdown', { signal });
await worker.close();
await connection.quit();
process.exit(0);
};
process.on('SIGTERM', () => void shutdown('SIGTERM'));
process.on('SIGINT', () => void shutdown('SIGINT'));
+265
View File
@@ -0,0 +1,265 @@
'use strict';
var __awaiter =
(this && this.__awaiter) ||
function (thisArg, _arguments, P, generator) {
function adopt(value) {
return value instanceof P
? value
: new P(function (resolve) {
resolve(value);
});
}
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) {
try {
step(generator.next(value));
} catch (e) {
reject(e);
}
}
function rejected(value) {
try {
step(generator['throw'](value));
} catch (e) {
reject(e);
}
}
function step(result) {
result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected);
}
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator =
(this && this.__generator) ||
function (thisArg, body) {
var _ = {
label: 0,
sent: function () {
if (t[0] & 1) throw t[1];
return t[1];
},
trys: [],
ops: []
},
f,
y,
t,
g = Object.create((typeof Iterator === 'function' ? Iterator : Object).prototype);
return (
(g.next = verb(0)),
(g['throw'] = verb(1)),
(g['return'] = verb(2)),
typeof Symbol === 'function' &&
(g[Symbol.iterator] = function () {
return this;
}),
g
);
function verb(n) {
return function (v) {
return step([n, v]);
};
}
function step(op) {
if (f) throw new TypeError('Generator is already executing.');
while ((g && ((g = 0), op[0] && (_ = 0)), _))
try {
if (
((f = 1),
y &&
(t = op[0] & 2 ? y['return'] : op[0] ? y['throw'] || ((t = y['return']) && t.call(y), 0) : y.next) &&
!(t = t.call(y, op[1])).done)
)
return t;
if (((y = 0), t)) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0:
case 1:
t = op;
break;
case 4:
_.label++;
return { value: op[1], done: false };
case 5:
_.label++;
y = op[1];
op = [0];
continue;
case 7:
op = _.ops.pop();
_.trys.pop();
continue;
default:
if (!((t = _.trys), (t = t.length > 0 && t[t.length - 1])) && (op[0] === 6 || op[0] === 2)) {
_ = 0;
continue;
}
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) {
_.label = op[1];
break;
}
if (op[0] === 6 && _.label < t[1]) {
_.label = t[1];
t = op;
break;
}
if (t && _.label < t[2]) {
_.label = t[2];
_.ops.push(op);
break;
}
if (t[2]) _.ops.pop();
_.trys.pop();
continue;
}
op = body.call(thisArg, _);
} catch (e) {
op = [6, e];
y = 0;
} finally {
f = t = 0;
}
if (op[0] & 5) throw op[1];
return { value: op[0] ? op[1] : void 0, done: true };
}
};
var _a;
Object.defineProperty(exports, '__esModule', { value: true });
require('./../bootstrap');
var bullmq_1 = require('bullmq');
var queries_1 = require('@cio/db/queries');
var jobs_1 = require('@cio/jobs');
var cancel_1 = require('../utils/cancel');
var env_1 = require('../config/env');
var logger_1 = require('../utils/logger');
var media_1 = require('../processors/media');
var concurrency =
Number.parseInt((_a = env_1.env.TRANSCRIBE_WORKER_CONCURRENCY) !== null && _a !== void 0 ? _a : '1', 10) || 1;
var connection = (0, jobs_1.createRedisConnection)();
var worker = new bullmq_1.Worker(
jobs_1.QUEUE_NAMES.mediaTranscribe,
function (job) {
return __awaiter(void 0, void 0, void 0, function () {
return __generator(this, function (_a) {
logger_1.log.info('transcribe-job-start', {
jobName: job.name,
bullmqJobId: job.id,
attempt: job.attemptsMade + 1
});
if (job.name === jobs_1.JOB_NAMES.mediaTranscribe.transcribeAudio) {
return [2 /*return*/, (0, media_1.processTranscribeAudio)(job.data)];
}
throw new Error('Unknown transcribe job: '.concat(job.name));
});
});
},
{ connection: connection, concurrency: concurrency }
);
worker.on('failed', function (job, err) {
return __awaiter(void 0, void 0, void 0, function () {
var isFinalAttempt, data;
var _a, _b, _c, _d, _e;
return __generator(this, function (_f) {
switch (_f.label) {
case 0:
if (!job) return [2 /*return*/];
if (!(err instanceof cancel_1.JobCanceledError)) return [3 /*break*/, 2];
logger_1.log.warn('transcribe-job-canceled', { jobName: job.name, bullmqJobId: job.id });
return [
4 /*yield*/,
(0, queries_1.updateMediaJob)(err.mediaJobId, {
status: 'canceled',
stage: 'canceled',
error: { code: 'CANCELED', message: 'Run canceled by user' }
})
];
case 1:
_f.sent();
return [2 /*return*/];
case 2:
logger_1.log.error('transcribe-job-failed', {
jobName: job.name,
bullmqJobId: job.id,
attempt: job.attemptsMade,
error: (0, cancel_1.errorMessage)(err)
});
isFinalAttempt = job.attemptsMade >= ((_a = job.opts.attempts) !== null && _a !== void 0 ? _a : 1);
if (!isFinalAttempt) return [2 /*return*/];
data = job.data;
if (!(data === null || data === void 0 ? void 0 : data.mediaJobId)) return [3 /*break*/, 4];
return [
4 /*yield*/,
(0, queries_1.updateMediaJob)(data.mediaJobId, {
status: 'failed',
stage: 'failed',
error: { code: 'WORKER_EXHAUSTED_RETRIES', message: (0, cancel_1.errorMessage)(err) }
})
];
case 3:
_f.sent();
_f.label = 4;
case 4:
return [
4 /*yield*/,
(0, queries_1.recordDeadLetterJob)({
organizationId:
(_c =
(_b = data === null || data === void 0 ? void 0 : data.actorContext) === null || _b === void 0
? void 0
: _b.organizationId) !== null && _c !== void 0
? _c
: null,
domain: 'media',
runId:
(_d = data === null || data === void 0 ? void 0 : data.mediaJobId) !== null && _d !== void 0
? _d
: null,
queueName: jobs_1.QUEUE_NAMES.mediaTranscribe,
jobName: job.name,
bullmqJobId: (_e = job.id) !== null && _e !== void 0 ? _e : null,
payload: job.data,
error: { code: 'WORKER_EXHAUSTED_RETRIES', message: (0, cancel_1.errorMessage)(err), stack: err.stack },
attempts: job.attemptsMade
})
];
case 5:
_f.sent();
return [2 /*return*/];
}
});
});
});
worker.on('ready', function () {
return logger_1.log.info('transcribe-worker-ready', {
concurrency: concurrency,
queue: jobs_1.QUEUE_NAMES.mediaTranscribe
});
});
worker.on('error', function (err) {
return logger_1.log.error('transcribe-worker-error', { error: (0, cancel_1.errorMessage)(err) });
});
var shutdown = function (signal) {
return __awaiter(void 0, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
logger_1.log.info('transcribe-worker-shutdown', { signal: signal });
return [4 /*yield*/, worker.close()];
case 1:
_a.sent();
return [4 /*yield*/, connection.quit()];
case 2:
_a.sent();
process.exit(0);
return [2 /*return*/];
}
});
});
};
process.on('SIGTERM', function () {
return void shutdown('SIGTERM');
});
process.on('SIGINT', function () {
return void shutdown('SIGINT');
});
+86
View File
@@ -0,0 +1,86 @@
import './../bootstrap';
import { Worker } from 'bullmq';
import { recordDeadLetterJob, updateMediaJob } from '@cio/db/queries';
import { JOB_NAMES, QUEUE_NAMES, createRedisConnection } from '@cio/jobs';
import { JobCanceledError, errorMessage } from '../utils/cancel';
import { env } from '../config/env';
import { log } from '../utils/logger';
import { processTranscribeAudio } from '../processors/media';
const concurrency = Number.parseInt(env.TRANSCRIBE_WORKER_CONCURRENCY ?? '1', 10) || 1;
const connection = createRedisConnection();
const worker = new Worker(
QUEUE_NAMES.mediaTranscribe,
async (job) => {
log.info('transcribe-job-start', { jobName: job.name, bullmqJobId: job.id, attempt: job.attemptsMade + 1 });
if (job.name === JOB_NAMES.mediaTranscribe.transcribeAudio) {
return processTranscribeAudio(job.data);
}
throw new Error(`Unknown transcribe job: ${job.name}`);
},
{ connection, concurrency }
);
worker.on('failed', async (job, err) => {
if (!job) return;
if (err instanceof JobCanceledError) {
log.warn('transcribe-job-canceled', { jobName: job.name, bullmqJobId: job.id });
await updateMediaJob(err.mediaJobId, {
status: 'canceled',
stage: 'canceled',
error: { code: 'CANCELED', message: 'Run canceled by user' }
});
return;
}
log.error('transcribe-job-failed', {
jobName: job.name,
bullmqJobId: job.id,
attempt: job.attemptsMade,
error: errorMessage(err)
});
const isFinalAttempt = job.attemptsMade >= (job.opts.attempts ?? 1);
if (!isFinalAttempt) return;
const data = job.data as { mediaJobId?: string; actorContext?: { organizationId?: string } };
if (data?.mediaJobId) {
await updateMediaJob(data.mediaJobId, {
status: 'failed',
stage: 'failed',
error: { code: 'WORKER_EXHAUSTED_RETRIES', message: errorMessage(err) }
});
}
await recordDeadLetterJob({
organizationId: data?.actorContext?.organizationId ?? null,
domain: 'media',
runId: data?.mediaJobId ?? null,
queueName: QUEUE_NAMES.mediaTranscribe,
jobName: job.name,
bullmqJobId: job.id ?? null,
payload: job.data as Record<string, unknown>,
error: { code: 'WORKER_EXHAUSTED_RETRIES', message: errorMessage(err), stack: err.stack },
attempts: job.attemptsMade
});
});
worker.on('ready', () => log.info('transcribe-worker-ready', { concurrency, queue: QUEUE_NAMES.mediaTranscribe }));
worker.on('error', (err) => log.error('transcribe-worker-error', { error: errorMessage(err) }));
const shutdown = async (signal: string) => {
log.info('transcribe-worker-shutdown', { signal });
await worker.close();
await connection.quit();
process.exit(0);
};
process.on('SIGTERM', () => void shutdown('SIGTERM'));
process.on('SIGINT', () => void shutdown('SIGINT'));
+265
View File
@@ -0,0 +1,265 @@
'use strict';
var __awaiter =
(this && this.__awaiter) ||
function (thisArg, _arguments, P, generator) {
function adopt(value) {
return value instanceof P
? value
: new P(function (resolve) {
resolve(value);
});
}
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) {
try {
step(generator.next(value));
} catch (e) {
reject(e);
}
}
function rejected(value) {
try {
step(generator['throw'](value));
} catch (e) {
reject(e);
}
}
function step(result) {
result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected);
}
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator =
(this && this.__generator) ||
function (thisArg, body) {
var _ = {
label: 0,
sent: function () {
if (t[0] & 1) throw t[1];
return t[1];
},
trys: [],
ops: []
},
f,
y,
t,
g = Object.create((typeof Iterator === 'function' ? Iterator : Object).prototype);
return (
(g.next = verb(0)),
(g['throw'] = verb(1)),
(g['return'] = verb(2)),
typeof Symbol === 'function' &&
(g[Symbol.iterator] = function () {
return this;
}),
g
);
function verb(n) {
return function (v) {
return step([n, v]);
};
}
function step(op) {
if (f) throw new TypeError('Generator is already executing.');
while ((g && ((g = 0), op[0] && (_ = 0)), _))
try {
if (
((f = 1),
y &&
(t = op[0] & 2 ? y['return'] : op[0] ? y['throw'] || ((t = y['return']) && t.call(y), 0) : y.next) &&
!(t = t.call(y, op[1])).done)
)
return t;
if (((y = 0), t)) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0:
case 1:
t = op;
break;
case 4:
_.label++;
return { value: op[1], done: false };
case 5:
_.label++;
y = op[1];
op = [0];
continue;
case 7:
op = _.ops.pop();
_.trys.pop();
continue;
default:
if (!((t = _.trys), (t = t.length > 0 && t[t.length - 1])) && (op[0] === 6 || op[0] === 2)) {
_ = 0;
continue;
}
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) {
_.label = op[1];
break;
}
if (op[0] === 6 && _.label < t[1]) {
_.label = t[1];
t = op;
break;
}
if (t && _.label < t[2]) {
_.label = t[2];
_.ops.push(op);
break;
}
if (t[2]) _.ops.pop();
_.trys.pop();
continue;
}
op = body.call(thisArg, _);
} catch (e) {
op = [6, e];
y = 0;
} finally {
f = t = 0;
}
if (op[0] & 5) throw op[1];
return { value: op[0] ? op[1] : void 0, done: true };
}
};
var _a;
Object.defineProperty(exports, '__esModule', { value: true });
require('./../bootstrap');
var bullmq_1 = require('bullmq');
var queries_1 = require('@cio/db/queries');
var jobs_1 = require('@cio/jobs');
var cancel_1 = require('../utils/cancel');
var env_1 = require('../config/env');
var logger_1 = require('../utils/logger');
var media_1 = require('../processors/media');
var concurrency =
Number.parseInt((_a = env_1.env.MEDIA_WORKER_CONCURRENCY) !== null && _a !== void 0 ? _a : '2', 10) || 2;
var connection = (0, jobs_1.createRedisConnection)();
var worker = new bullmq_1.Worker(
jobs_1.QUEUE_NAMES.media,
function (job) {
return __awaiter(void 0, void 0, void 0, function () {
return __generator(this, function (_a) {
logger_1.log.info('media-job-start', { jobName: job.name, bullmqJobId: job.id, attempt: job.attemptsMade + 1 });
switch (job.name) {
case jobs_1.JOB_NAMES.media.probeMetadata:
return [2 /*return*/, (0, media_1.processProbeMetadata)(job.data)];
case jobs_1.JOB_NAMES.media.generateThumbnail:
return [2 /*return*/, (0, media_1.processGenerateThumbnail)(job.data)];
case jobs_1.JOB_NAMES.media.extractAudio:
return [2 /*return*/, (0, media_1.processExtractAudio)(job.data)];
default:
throw new Error('Unknown media job: '.concat(job.name));
}
return [2 /*return*/];
});
});
},
{ connection: connection, concurrency: concurrency }
);
worker.on('failed', function (job, err) {
return __awaiter(void 0, void 0, void 0, function () {
var isFinalAttempt, data;
var _a, _b, _c, _d, _e;
return __generator(this, function (_f) {
switch (_f.label) {
case 0:
if (!job) return [2 /*return*/];
if (!(err instanceof cancel_1.JobCanceledError)) return [3 /*break*/, 2];
logger_1.log.warn('media-job-canceled', { jobName: job.name, bullmqJobId: job.id });
return [
4 /*yield*/,
(0, queries_1.updateMediaJob)(err.mediaJobId, {
status: 'canceled',
stage: 'canceled',
error: { code: 'CANCELED', message: 'Run canceled by user' }
})
];
case 1:
_f.sent();
return [2 /*return*/];
case 2:
logger_1.log.error('media-job-failed', {
jobName: job.name,
bullmqJobId: job.id,
attempt: job.attemptsMade,
error: (0, cancel_1.errorMessage)(err)
});
isFinalAttempt = job.attemptsMade >= ((_a = job.opts.attempts) !== null && _a !== void 0 ? _a : 1);
if (!isFinalAttempt) return [2 /*return*/];
data = job.data;
if (!(data === null || data === void 0 ? void 0 : data.mediaJobId)) return [3 /*break*/, 4];
return [
4 /*yield*/,
(0, queries_1.updateMediaJob)(data.mediaJobId, {
status: 'failed',
stage: 'failed',
error: { code: 'WORKER_EXHAUSTED_RETRIES', message: (0, cancel_1.errorMessage)(err) }
})
];
case 3:
_f.sent();
_f.label = 4;
case 4:
return [
4 /*yield*/,
(0, queries_1.recordDeadLetterJob)({
organizationId:
(_c =
(_b = data === null || data === void 0 ? void 0 : data.actorContext) === null || _b === void 0
? void 0
: _b.organizationId) !== null && _c !== void 0
? _c
: null,
domain: 'media',
runId:
(_d = data === null || data === void 0 ? void 0 : data.mediaJobId) !== null && _d !== void 0
? _d
: null,
queueName: jobs_1.QUEUE_NAMES.media,
jobName: job.name,
bullmqJobId: (_e = job.id) !== null && _e !== void 0 ? _e : null,
payload: job.data,
error: { code: 'WORKER_EXHAUSTED_RETRIES', message: (0, cancel_1.errorMessage)(err), stack: err.stack },
attempts: job.attemptsMade
})
];
case 5:
_f.sent();
return [2 /*return*/];
}
});
});
});
worker.on('ready', function () {
return logger_1.log.info('media-worker-ready', { concurrency: concurrency, queue: jobs_1.QUEUE_NAMES.media });
});
worker.on('error', function (err) {
return logger_1.log.error('media-worker-error', { error: (0, cancel_1.errorMessage)(err) });
});
var shutdown = function (signal) {
return __awaiter(void 0, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
logger_1.log.info('media-worker-shutdown', { signal: signal });
return [4 /*yield*/, worker.close()];
case 1:
_a.sent();
return [4 /*yield*/, connection.quit()];
case 2:
_a.sent();
process.exit(0);
return [2 /*return*/];
}
});
});
};
process.on('SIGTERM', function () {
return void shutdown('SIGTERM');
});
process.on('SIGINT', function () {
return void shutdown('SIGINT');
});
+91
View File
@@ -0,0 +1,91 @@
import './../bootstrap';
import { Worker } from 'bullmq';
import { recordDeadLetterJob, updateMediaJob } from '@cio/db/queries';
import { JOB_NAMES, QUEUE_NAMES, createRedisConnection } from '@cio/jobs';
import { JobCanceledError, errorMessage } from '../utils/cancel';
import { env } from '../config/env';
import { log } from '../utils/logger';
import { processExtractAudio, processGenerateThumbnail, processProbeMetadata } from '../processors/media';
const concurrency = Number.parseInt(env.MEDIA_WORKER_CONCURRENCY ?? '2', 10) || 2;
const connection = createRedisConnection();
const worker = new Worker(
QUEUE_NAMES.media,
async (job) => {
log.info('media-job-start', { jobName: job.name, bullmqJobId: job.id, attempt: job.attemptsMade + 1 });
switch (job.name) {
case JOB_NAMES.media.probeMetadata:
return processProbeMetadata(job.data);
case JOB_NAMES.media.generateThumbnail:
return processGenerateThumbnail(job.data);
case JOB_NAMES.media.extractAudio:
return processExtractAudio(job.data);
default:
throw new Error(`Unknown media job: ${job.name}`);
}
},
{ connection, concurrency }
);
worker.on('failed', async (job, err) => {
if (!job) return;
if (err instanceof JobCanceledError) {
log.warn('media-job-canceled', { jobName: job.name, bullmqJobId: job.id });
await updateMediaJob(err.mediaJobId, {
status: 'canceled',
stage: 'canceled',
error: { code: 'CANCELED', message: 'Run canceled by user' }
});
return;
}
log.error('media-job-failed', {
jobName: job.name,
bullmqJobId: job.id,
attempt: job.attemptsMade,
error: errorMessage(err)
});
const isFinalAttempt = job.attemptsMade >= (job.opts.attempts ?? 1);
if (!isFinalAttempt) return;
const data = job.data as { mediaJobId?: string; assetId?: string; actorContext?: { organizationId?: string } };
if (data?.mediaJobId) {
await updateMediaJob(data.mediaJobId, {
status: 'failed',
stage: 'failed',
error: { code: 'WORKER_EXHAUSTED_RETRIES', message: errorMessage(err) }
});
}
await recordDeadLetterJob({
organizationId: data?.actorContext?.organizationId ?? null,
domain: 'media',
runId: data?.mediaJobId ?? null,
queueName: QUEUE_NAMES.media,
jobName: job.name,
bullmqJobId: job.id ?? null,
payload: job.data as Record<string, unknown>,
error: { code: 'WORKER_EXHAUSTED_RETRIES', message: errorMessage(err), stack: err.stack },
attempts: job.attemptsMade
});
});
worker.on('ready', () => log.info('media-worker-ready', { concurrency, queue: QUEUE_NAMES.media }));
worker.on('error', (err) => log.error('media-worker-error', { error: errorMessage(err) }));
const shutdown = async (signal: string) => {
log.info('media-worker-shutdown', { signal });
await worker.close();
await connection.quit();
process.exit(0);
};
process.on('SIGTERM', () => void shutdown('SIGTERM'));
process.on('SIGINT', () => void shutdown('SIGINT'));